From 5d68e4396dd2971113f84d627510c11b356b9f8d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 7 Mar 2019 10:47:42 +0000 Subject: [PATCH 1/4] HADOOP-15691 Add PathCapabilities to FS and FC to complement StreamCapabilities. This contains all the work to date, reapplied to trunk so that Yetus will take up the PR again. Change-Id: Icfab71d177c88c75fd651d7eea56d3d1e8618f61 --- .../apache/hadoop/fs/AbstractFileSystem.java | 30 +++- .../apache/hadoop/fs/ChecksumFileSystem.java | 21 +++ .../hadoop/fs/CommonPathCapabilities.java | 137 +++++++++++++++++ .../hadoop/fs/DelegateToFileSystem.java | 7 + .../org/apache/hadoop/fs/FileContext.java | 21 ++- .../java/org/apache/hadoop/fs/FileSystem.java | 30 +++- .../apache/hadoop/fs/FilterFileSystem.java | 6 + .../java/org/apache/hadoop/fs/FilterFs.java | 5 + .../org/apache/hadoop/fs/HarFileSystem.java | 19 ++- .../apache/hadoop/fs/PathCapabilities.java | 54 +++++++ .../apache/hadoop/fs/RawLocalFileSystem.java | 20 +++ .../fs/http/AbstractHttpFileSystem.java | 20 +++ .../hadoop/fs/impl/FsLinkResolution.java | 98 ++++++++++++ .../fs/impl/PathCapabilitiesSupport.java | 43 ++++++ .../hadoop/fs/viewfs/ViewFileSystem.java | 37 +++++ .../site/markdown/filesystem/filesystem.md | 145 +++++++++++++++++- .../contract/AbstractContractAppendTest.java | 10 ++ .../contract/AbstractContractConcatTest.java | 7 + .../hadoop/fs/contract/ContractTestUtils.java | 29 +++- .../hadoop/hdfs/DistributedFileSystem.java | 35 +++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 38 +++++ .../fs/http/client/HttpFSFileSystem.java | 29 ++++ .../apache/hadoop/fs/s3a/S3AFileSystem.java | 38 ++++- .../hadoop/fs/s3a/commit/CommitConstants.java | 2 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 3 +- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 5 + .../apache/hadoop/fs/s3a/S3ATestUtils.java | 7 +- .../ITestSessionDelegationTokens.java | 8 + .../fs/s3a/commit/ITestCommitOperations.java | 5 +- .../s3guard/AbstractS3GuardToolTestBase.java | 2 +- .../apache/hadoop/fs/adl/AdlFileSystem.java | 19 +++ .../fs/azure/NativeAzureFileSystem.java | 18 +++ .../fs/azurebfs/AzureBlobFileSystem.java | 22 +++ 33 files changed, 943 insertions(+), 27 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index 6e82543ca850a8..68d451f39765d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -29,6 +29,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; @@ -45,6 +46,7 @@ import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -72,7 +74,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Stable -public abstract class AbstractFileSystem { +public abstract class AbstractFileSystem implements PathCapabilities { static final Logger LOG = LoggerFactory.getLogger(AbstractFileSystem.class); /** Recording statistics per a file system class. */ @@ -1371,4 +1373,30 @@ public CompletableFuture openFileWithOptions(Path path, new CompletableFuture<>(), () -> open(path, bufferSize)); } + /** + * Return the base capabilities of the filesystems + * may override to declare different behavior. + * @param path path to query the capability of. + * @param capability string to query the stream support for. + * @return true if the capability is supported under that part of the FS. + * @throws IOException on failure + */ + public boolean hasPathCapability(final Path path, + final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + makeQualified(path); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_SYMLINKS: + // delegate to the existing supportsSymlinks() call. + return supportsSymlinks(); + case CommonPathCapabilities.FS_DELEGATION_TOKENS: + // this is less efficient than it should be. + return getCanonicalServiceName() != null; + default: + // the feature is not implemented. + return false; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 99aa5d22babb45..e560b1e9ccd823 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -872,4 +873,24 @@ public FSDataOutputStreamBuilder createFile(Path path) { public FSDataOutputStreamBuilder appendFile(Path path) { return createDataOutputStreamBuilder(this, path).append(); } + + /** + * Disable those operations which are disabled so as to guarantee + * checksumming of all created files. + * {@inheritDoc} + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + // query the superclass, which triggers argument validation. + boolean superCapability = super.hasPathCapability(path, capability); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CONCAT: + return false; + default: + return superCapability; + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java new file mode 100644 index 00000000000000..4fa899d062b168 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import java.util.List; + +import org.apache.hadoop.fs.permission.FsPermission; + +/** + * Common path capabilities. + */ +public final class CommonPathCapabilities { + + public CommonPathCapabilities() { + } + + /** + * Does the Filesystem support + * {@link FileSystem#setAcl(Path, List)}, + * {@link FileSystem#getAclStatus(Path)} + * and related methods? + * Value: {@value}. + */ + public static final String FS_ACLS = "fs.paths.acls"; + + /** + * Does the Filesystem support {@link FileSystem#append(Path)}? + * Value: {@value}. + */ + public static final String FS_APPEND = "fs.paths.append"; + + /** + * Does the FS support {@link FileSystem#getFileChecksum(Path)}? + * Value: {@value}. + */ + public static final String FS_CHECKSUMS = "fs.paths.checksums"; + + /** + * Does the FS support {@link FileSystem#concat(Path, Path[])}? + * Value: {@value}. + */ + public static final String FS_CONCAT = "fs.paths.concat"; + + /** + * Does the filesystem support Delegation Tokens? + * Value: {@value}. + */ + public static final String FS_DELEGATION_TOKENS = + "fs.paths.delegation.tokens"; + + /** + * Does the FS support {@link FileSystem#listCorruptFileBlocks(Path)} ()}? + * Value: {@value}. + */ + public static final String FS_LIST_CORRUPT_FILE_BLOCKS = + "fs.paths.list-corrupt-file-blocks"; + + /** + * Does the FS support + * {@link FileSystem#createPathHandle(FileStatus, Options.HandleOpt...)} + * and related methods? + * Value: {@value}. + */ + public static final String FS_PATHHANDLES = "fs.paths.pathhandles"; + + /** + * Does the FS support {@link FileSystem#setPermission(Path, FsPermission)} + * and related methods? + * Value: {@value}. + */ + public static final String FS_PERMISSIONS = "fs.paths.permissions"; + + /** + * Does this filesystem connector only support filesystem read operations? + * For example, the {@code HttpFileSystem} is always read-only. + * This is different from "is the specific instance and path read only?", + * which must be determined by checking permissions (where supported), or + * attempting write operations under a path. + * Value: {@value}. + */ + public static final String FS_READ_ONLY_CONNECTOR = + "fs.paths.read-only-connector"; + + /** + * Does the FS support snapshots through + * {@link FileSystem#createSnapshot(Path)} and related methods?? + * Value: {@value}. + */ + public static final String FS_SNAPSHOTS = "fs.paths.snapshots"; + + /** + * Does the FS support {@link FileSystem#setStoragePolicy(Path, String)} + * and related methods? + * Value: {@value}. + */ + public static final String FS_STORAGEPOLICY = + "fs.paths.storagepolicy"; + + /** + * Does the FS support symlinks through + * {@link FileSystem#createSymlink(Path, Path, boolean)} and related methods? + * Value: {@value}. + */ + public static final String FS_SYMLINKS = + "fs.paths.symlinks"; + + /** + * Does the FS support {@link FileSystem#truncate(Path, long)} ? + * Value: {@value}. + */ + public static final String FS_TRUNCATE = + "fs.paths.truncate"; + + /** + * Does the Filesystem support XAttributes through + * {@link FileSystem#setXAttr(Path, String, byte[])} and related methods? + * Value: {@value}. + */ + public static final String FS_XATTRS = "fs.paths.xattrs"; + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 165c56c3d5c378..a8f294f3791589 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -281,4 +281,11 @@ public CompletableFuture openFileWithOptions(Path path, int bufferSize) throws IOException { return fsImpl.openFileWithOptions(path, mandatoryKeys, options, bufferSize); } + + @Override + public boolean hasPathCapability(final Path path, + final String capability) + throws IOException { + return fsImpl.hasPathCapability(path, capability); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index f65074856bf3e0..d43f20db91c469 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -46,6 +46,8 @@ import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl; +import org.apache.hadoop.fs.impl.FsLinkResolution; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -171,7 +173,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable -public class FileContext { +public class FileContext implements PathCapabilities { public static final Logger LOG = LoggerFactory.getLogger(FileContext.class); /** @@ -2934,4 +2936,21 @@ public CompletableFuture next( }.resolve(FileContext.this, absF); } } + + /** + * Return the path capabilities of the bonded {@code AbstractFileSystem}. + * @param path path to query the capability of. + * @param capability string to query the stream support for. + * @return true iff the capability is supported under that FS. + * @throws IOException path resolution or other IO failure + * @throws IllegalArgumentException invalid arguments + */ + public boolean hasPathCapability(Path path, String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + return FsLinkResolution.resolve(this, + fixRelativePart(path), + (fs, p) -> fs.hasPathCapability(p, capability)); + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index c3be2f21a57d99..d76c8b6303bdd1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -35,6 +35,7 @@ import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -58,6 +59,7 @@ import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -134,7 +136,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable public abstract class FileSystem extends Configured - implements Closeable, DelegationTokenIssuer { + implements Closeable, DelegationTokenIssuer, PathCapabilities { public static final String FS_DEFAULT_NAME_KEY = CommonConfigurationKeys.FS_DEFAULT_NAME_KEY; public static final String DEFAULT_FS = @@ -720,6 +722,7 @@ protected FileSystem() { * */ protected void checkPath(Path path) { + Preconditions.checkArgument(path != null, "null path"); URI uri = path.toUri(); String thatScheme = uri.getScheme(); if (thatScheme == null) // fs is relative @@ -3259,6 +3262,31 @@ public Collection getTrashRoots(boolean allUsers) { return ret; } + /** + * The base FileSystem implementation generally has no knowledge + * of the capabilities of actual implementations. + * Unless it has a way to explicitly determine the capabilities, + * this method returns false. + * {@inheritDoc} + */ + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + makeQualified(path); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_SYMLINKS: + // delegate to the existing supportsSymlinks() call. + return supportsSymlinks() && areSymlinksEnabled(); + case CommonPathCapabilities.FS_DELEGATION_TOKENS: + // this is less efficient than it should be. + return getCanonicalServiceName() != null; + default: + // the feature is not implemented. + return false; + } + } + // making it volatile to be able to do a double checked locking private volatile static boolean FILE_SYSTEMS_LOADED = false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index e05c574063f7ff..557b072cd9440f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -729,4 +729,10 @@ protected CompletableFuture openFileWithOptions( return fs.openFileWithOptions(pathHandle, mandatoryKeys, options, bufferSize); } + + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + return fs.hasPathCapability(path, capability); + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index f5430d60261609..731a52a7b41372 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -446,4 +446,9 @@ public CompletableFuture openFileWithOptions( return myFs.openFileWithOptions(path, mandatoryKeys, options, bufferSize); } + public boolean hasPathCapability(final Path path, + final String capability) + throws IOException { + return myFs.hasPathCapability(path, capability); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index f7da819ed6c173..1a3af41acb018b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.HandleOpt; import org.apache.hadoop.io.IOUtils; @@ -899,7 +900,23 @@ public void setPermission(Path p, FsPermission permission) throws IOException { throw new IOException("Har: setPermission not allowed"); } - + + /** + * Declare that this filesystem connector is always read only. + * {@inheritDoc} + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_READ_ONLY_CONNECTOR: + return true; + default: + return false; + } + } + /** * Hadoop archives input stream. This input stream fakes EOF * since archive files are part of bigger part files. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java new file mode 100644 index 00000000000000..8a3cf525397c94 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import java.io.IOException; + +/** + * The Path counterpoint to {@link StreamCapabilities}; a query to see if, + * a FileSystem/FileContext instance has a specific capability under the given + * path. + */ +public interface PathCapabilities { + + /** + * Probe for a filesystem instance offering a specific capability under the + * given path. + * If the function returns {@code true}, the filesystem is explicitly + * declaring that the capability is available. + * If the function returns {@code false}, it can mean one of: + *
    + *
  • The capability is not known.
    • + *
  • The capability is known but it is not supported.
    • + *
  • The capability is known but the filesystem does not know if it + * is supported under the supplied path it.
    • + *
+ * The core guarantee which a caller can rely on is: if the predicate + * returns true, then the specific operation/behavior can be expected to be + * supported. + * @param path path to query the capability of. + * @param capability non-null, non-empty string to query the path for support. + * @return true if the capability is supported under that part of the FS. + * @throws IOException this should not be raised, except on problems + * resolving paths or relaying the call. + * @throws IllegalArgumentException invalid arguments + */ + boolean hasPathCapability(Path path, String capability) + throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index bd003ae90ab4cf..fff266d2a9247a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -41,11 +41,13 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.Optional; +import java.util.Locale; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; @@ -1060,4 +1062,22 @@ public Path getLinkTarget(Path f) throws IOException { // return an unqualified symlink target return fi.getSymlink(); } + + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_PATHHANDLES: + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_TRUNCATE: + return true; + case CommonPathCapabilities.FS_SYMLINKS: + return FileSystem.areSymlinksEnabled(); + default: + return super.hasPathCapability(path, capability); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java index fa0b2cf6c314e0..5f02dab13d0da9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -27,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PositionedReadable; import org.apache.hadoop.fs.Seekable; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; @@ -35,6 +37,7 @@ import java.io.InputStream; import java.net.URI; import java.net.URLConnection; +import java.util.Locale; abstract class AbstractHttpFileSystem extends FileSystem { private static final long DEFAULT_BLOCK_SIZE = 4096; @@ -111,6 +114,23 @@ public FileStatus getFileStatus(Path path) throws IOException { return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0, path); } + /** + * Declare that this filesystem connector is always read only. + * {@inheritDoc} + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_READ_ONLY_CONNECTOR: + return true; + default: + return false; + } + } + private static class HttpDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java new file mode 100644 index 00000000000000..f5ef8c49233283 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FsLinkResolution.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.impl; + +import java.io.IOException; + +import com.google.common.base.Preconditions; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.FSLinkResolver; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnresolvedLinkException; + +/** + * Class to allow Lambda expressions to be used in {@link FileContext} + * link resolution. + * @param type of the returned value. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class FsLinkResolution extends FSLinkResolver { + + /** + * The function to invoke in the {@link #next(AbstractFileSystem, Path)} call. + */ + private final FsLinkResolutionFunction fn; + + /** + * Construct an instance with the given function. + * @param fn function to invoke. + */ + public FsLinkResolution(final FsLinkResolutionFunction fn) { + this.fn = Preconditions.checkNotNull(fn); + } + + @Override + public T next(final AbstractFileSystem fs, final Path p) + throws UnresolvedLinkException, IOException { + return fn.apply(fs, p); + } + + /** + * The signature of the function to invoke. + * @param type resolved to + */ + @FunctionalInterface + public interface FsLinkResolutionFunction { + + /** + * + * @param fs filesystem to resolve against. + * @param path path to resolve + * @return a result of type T + * @throws UnresolvedLinkException link resolution failure + * @throws IOException other IO failure. + */ + T apply(final AbstractFileSystem fs, final Path path) + throws IOException, UnresolvedLinkException; + } + + /** + * Apply the given function to the resolved path under the the supplied + * FileContext. + * @param fileContext file context to resolve under + * @param path path to resolve + * @param fn function to invoke + * @param return type. + * @return the return value of the function as revoked against the resolved + * path. + * @throws UnresolvedLinkException link resolution failure + * @throws IOException other IO failure. + */ + public static T resolve( + final FileContext fileContext, final Path path, + final FsLinkResolutionFunction fn) + throws UnresolvedLinkException, IOException { + return new FsLinkResolution<>(fn).resolve(fileContext, path); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java new file mode 100644 index 00000000000000..76be0e3bf49d30 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.impl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; + +import static com.google.common.base.Preconditions.checkArgument; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class PathCapabilitiesSupport { + + /** + * Validate the arguments to {@code PathCapabilities.hadCapability()}. + * @param path path to query the capability of. + * @param capability non-null, non-empty string to query the path for support. + * @throws IllegalArgumentException if a an argument is invalid. + */ + public static void validatehasPathCapabilityArgs( + final Path path, final String capability) { + checkArgument(path != null, "null path"); + checkArgument(capability != null && !capability.isEmpty(), + "null/empty capability"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 6bc469c67529cc..94e97f4233bfa3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -43,6 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStoragePolicySpi; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -56,10 +58,12 @@ import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.AclUtil; @@ -1026,6 +1030,39 @@ public Path getLinkTarget(Path path) throws IOException { return res.targetFileSystem.getLinkTarget(res.remainingPath); } + /** + * Fail fast on known write capabilities (append, concat), + * forward the rest to the viewed FS. + * @param path path to query the capability of. + * @param capability string to query the stream support for. + * @return the capability + * @throws IOException if there is no resolved FS, or it raises an IOE. + */ + @Override + public boolean hasPathCapability(Path path, String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + Path p = makeQualified(path); + + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_CONCAT: + // concat is not supported, as it may be invoked across filesystems. + return false; + default: + } + // otherwise, check capabilities of mounted FS. + try { + InodeTree.ResolveResult res + = fsState.resolve(getUriPath(p), true); + return res.targetFileSystem.hasPathCapability(res.remainingPath, + capability); + } catch (FileNotFoundException e) { + // no mount point, nothing will work. + throw new NotInMountpointException(path, "hasPathCapability"); + } + } + /** * An instance of this class represents an internal dir of the viewFs * that is internal dir of the mount table. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 0b09f02b93982a..950de824ff6a53 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -526,7 +526,7 @@ on the filesystem. `getFileStatus(P).getBlockSize()`. 1. By inference, it MUST be > 0 for any file of length > 0. -## State Changing Operations +## State Changing Operations ### `boolean mkdirs(Path p, FsPermission permission)` @@ -1479,7 +1479,7 @@ public interface StreamCapabilities { ### `boolean hasCapability(capability)` -Return true if the `OutputStream`, `InputStream`, or other FileSystem class +Return true iff the `OutputStream`, `InputStream`, or other FileSystem class has the desired capability. The caller can query the capabilities of a stream using a string value. @@ -1492,3 +1492,144 @@ hsync | HSYNC | Syncable | Flush out the data in client's us in:readahead | READAHEAD | CanSetReadahead | Set the readahead on the input stream. dropbehind | DROPBEHIND | CanSetDropBehind | Drop the cache. in:unbuffer | UNBUFFER | CanUnbuffer | Reduce the buffering on the input stream. + +## interface `PathCapabilities` + +The `PathCapabilities` provides a way to programmatically query the operations +offered by a FileSystem or FileContext instance under a given path. + +```java +public interface PathCapabilities { + boolean hasPathCapability(Path path, String capability) + throws IOException; +} +``` + +There are a number of goals here: + +1. Allow callers to probe for optional filesystem operations without actually +having to invoke them. +1. Allow filesystems with their own optional per-instance features to declare +whether or not they are active for the specific instance. +1. Allow for fileystem connectors which work with object stores to expose the +fundamental difference in semantics of these stores (e.g: files not visible +until closed, file rename being `O(data)`), directory rename being non-atomic, +etc. + +### Available Capabilities + +Capabilities are defined with a store prefix and an arbitrary (but hopefully +meaningful) string. + +All custom filesystem-specific capabilities MUST be given the prefix of that +filesystem schema. The standard schemas are: + +* `fs` : File system capabilities +* `object` : Object capabilities. + +The exact set of operations and their names are evolving. + +Consult the javadocs for `org.apache.hadoop.fs.PathCapabilities` for the +standard set of capabilities. + +Individual filesystems may offer their own set of capabilities which +can be probed for. These begin with the same prefix as the filesystem schema, +such as `hdfs:` or `s3a:`. + +### `boolean hasPathCapability(path, capability)` + +Probe for a filesystem instance offering a specific capability under the +given path. + +#### Postconditions + +```python +if fs_supports_the_feature(path, capability): + return True +else: + return False +``` + +Return: `True`, iff the specific capability is, the the best of the +knowledge of the client application, available. + +A filesystem instance *MUST NOT* return `True` for any capability unless it is +known to be supported by that specific instance. As a result, if a caller +probes for a capability then it can assume that the specific feature/semantics +are available. + +If the probe returns `False` then it can mean one of: + +1. The capability is unknown. +1. The capability is known, but known to be unavailable on this instance. + +This predicate is intended to be low cost. If it requires remote calls other +than path/link resolution, it SHOULD conclude that the availability +of the feature is unknown and return `False`. + +The predicate MUST also be side-effect free. + +*Validity of paths* +There is no requirement that the existence of the path must be checked; +the parameter exists so that any filesystem which relays operations to other +filesystems (e.g `viewfs`) can resolve and relay it to the nested filesystem. +Consider the call to be *relatively* lightweight. + +Because of this, it may be that while the filesystem declares that +it supports a capability under a path, the actual invocation of the operation +may fail for other reasons. + +As an example, while a filesystem may support `append()` under a path, +if invoked on a directory, the call may fail. + +That is for a path `root = new Path("/")`: the capabilities call may succeed + +```java +fs.hasCapabilities(root, "fs:append") == true +``` + +But a subsequent call to the operation on that specific path may fail, +because the root path is a directory +```java +fs.append(root) +``` + +The `hasCapabilities(path, capability)` probe is therefore declaring that +the operation will not be rejected as unsupported, not that a specific invocation +will be considered valid. + +*Duration of availability* + +As the state of a remote store changes,so may path capabilities. This +may be due to changes in the local state of the fileystem (e.g. symbolic links +or mount points changing), or changes in its functionality (e.g. a feature +becoming availaible/unavailable due to operational changes, system upgrades, etc.) + +*Capabilities which must be invoked to determine availablity* + +Some operations may be known by the filesystem client, and believed to be available, +but may actually fail when invoked due to the state of the underlying +filesystem —state which is cannot be determined except by attempting +side-effecting operations. + +A key example of this is symbolic links and the local filesystem. +The filesystem declares that it supports this unless symbolic links are explicitly +disabled —when invoked they may actually fail. + +### Implementors Notes + +Implementors MUST NOT return `true` for any capability which is not guaranteed +to be supported. To return `true` indicates that the implementation/deployment +of the filesystem does, to the best of the knowledge of the filesystem client, +offer the desired operations *and semantics* queried for. + +For performance reasons, implementations SHOULD NOT check the path for +existence, unless it needs to resolve symbolic links in parts of the path +to determine whether a feature is present.This required of `FileContext` +and `viewfs`. It may also be needed to + +Individual filesystems MUST NOT define new `fs:`-prefixed capabilities. +Instead they MUST do one of the following: + +* Define and stabilize new cross-filesystem capability flags (preferred). +* Use the schema of the filesystem to as a prefix for their own options. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java index a9fb1177397aec..9b92dacadd90e3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java @@ -18,12 +18,15 @@ package org.apache.hadoop.fs.contract; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; @@ -155,4 +158,11 @@ public void testRenameFileBeingAppended() throws Throwable { dataset.length); ContractTestUtils.compareByteArrays(dataset, bytes, dataset.length); } + + @Test + public void testFileSystemDeclaresCapability() throws Throwable { + assertHasPathCapabilities(getFileSystem(), target, + CommonPathCapabilities.FS_APPEND); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java index d30e0d66eff4f1..d712369de3b97a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java @@ -24,7 +24,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CONCAT; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; @@ -93,4 +95,9 @@ public void testConcatOnSelf() throws Throwable { () -> getFileSystem().concat(target, new Path[]{target}))); } + @Test + public void testFileSystemDeclaresCapability() throws Throwable { + assertHasPathCapabilities(getFileSystem(), target, FS_CONCAT); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 789fb0acc5f067..61e0be734bbb7f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.io.IOUtils; @@ -1491,22 +1492,40 @@ public static void assertCapabilities( assertTrue("Stream should be instanceof StreamCapabilities", stream instanceof StreamCapabilities); - if (shouldHaveCapabilities!=null) { + StreamCapabilities source = (StreamCapabilities) stream; + if (shouldHaveCapabilities != null) { for (String shouldHaveCapability : shouldHaveCapabilities) { assertTrue("Should have capability: " + shouldHaveCapability, - ((StreamCapabilities) stream).hasCapability(shouldHaveCapability)); + source.hasCapability(shouldHaveCapability)); } } - if (shouldNotHaveCapabilities!=null) { + if (shouldNotHaveCapabilities != null) { for (String shouldNotHaveCapability : shouldNotHaveCapabilities) { assertFalse("Should not have capability: " + shouldNotHaveCapability, - ((StreamCapabilities) stream) - .hasCapability(shouldNotHaveCapability)); + source.hasCapability(shouldNotHaveCapability)); } } } + /** + * Custom assert to test {@link PathCapabilities}. + * + * @param source source (FS, FC, etc) + * @param path path to check + * @param capabilities The array of unexpected capabilities + */ + public static void assertHasPathCapabilities( + final PathCapabilities source, + final Path path, + final String...capabilities) throws IOException { + + for (String shouldHaveCapability : capabilities) { + assertTrue("Should have capability: " + shouldHaveCapability, + source.hasPathCapability(path, shouldHaveCapability)); + } + } + /** * Function which calls {@code InputStream.read()} and * downgrades an IOE to a runtime exception. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 73abb991f99e21..f38fb0967f4b7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -48,6 +49,7 @@ import org.apache.hadoop.fs.GlobalStorageStatistics; import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider; import org.apache.hadoop.fs.InvalidPathHandleException; +import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.PathHandle; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options; @@ -62,6 +64,7 @@ import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -117,6 +120,7 @@ import java.util.Collection; import java.util.EnumSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -3404,4 +3408,35 @@ public RemoteIterator listOpenFiles( public HdfsDataOutputStreamBuilder appendFile(Path path) { return new HdfsDataOutputStreamBuilder(this, path).append(); } + + /** + * HDFS client capabilities. + * Keep {@code WebHdfsFileSystem} in sync. + * {@inheritDoc} + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + makeQualified(path); + + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_ACLS: + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CHECKSUMS: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_LIST_CORRUPT_FILE_BLOCKS: + case CommonPathCapabilities.FS_PATHHANDLES: + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_SNAPSHOTS: + case CommonPathCapabilities.FS_STORAGEPOLICY: + case CommonPathCapabilities.FS_XATTRS: + return true; + case CommonPathCapabilities.FS_SYMLINKS: + return FileSystem.areSymlinksEnabled(); + default: + return super.hasPathCapability(path, capability); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index baebdc1654a30d..8d30dd31c48779 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -46,6 +46,7 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.StringTokenizer; @@ -61,6 +62,7 @@ import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.DelegationTokenRenewer; @@ -74,6 +76,7 @@ import org.apache.hadoop.fs.GlobalStorageStatistics; import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider; import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.permission.FsCreateModes; @@ -1125,6 +1128,11 @@ public boolean mkdirs(Path f, FsPermission permission) throws IOException { ).run(); } + @Override + public boolean supportsSymlinks() { + return true; + } + /** * Create a symlink pointing to the destination path. */ @@ -2079,6 +2087,36 @@ public void setTestProvider(KeyProvider kp) { testProvider = kp; } + /** + * This filesystem's capabilities must be in sync with that of HDFS. + * @param path path to query the capability of. + * @param capability string to query the stream support for. + * @return true if a capability is supported. + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + // query the superclass, which triggers argument validation. + boolean superCapability = super.hasPathCapability(path, capability); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_ACLS: + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CHECKSUMS: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_SNAPSHOTS: + case CommonPathCapabilities.FS_STORAGEPOLICY: + case CommonPathCapabilities.FS_XATTRS: + return true; + case CommonPathCapabilities.FS_SYMLINKS: + // there's no checking of the {symlinksEnabled} flag in this class,\ + // so always return true. + return true; + default: + return superCapability; + } + } + /** * This class is used for opening, reading, and seeking files while using the * WebHdfsFileSystem. This class will invoke the retry policy when performing diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index ac909ddba757c8..966902ca963a69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -26,6 +26,7 @@ import com.google.common.base.Charsets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.DelegationTokenRenewer; import org.apache.hadoop.fs.FSDataInputStream; @@ -85,6 +86,7 @@ import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.HashMap; +import java.util.Locale; import java.util.Map; /** @@ -1561,4 +1563,31 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() return JsonUtilClient.toSnapshottableDirectoryList(json); } + /** + * This filesystem's capabilities must be in sync with that of + * {@code DistributedFileSystem.hasPathCapability()} except + * where the feature is not exposed (e.g. symlinks). + * {@inheritDoc} + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + // query the superclass, which triggers argument validation. + boolean superCapability = super.hasPathCapability(path, capability); + + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_ACLS: + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_SNAPSHOTS: + case CommonPathCapabilities.FS_STORAGEPOLICY: + case CommonPathCapabilities.FS_XATTRS: + return true; + case CommonPathCapabilities.FS_SYMLINKS: + return false; + default: + return superCapability; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0747be2d7e419a..1962f81970dcd6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -91,6 +91,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -122,6 +123,7 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.fs.s3a.auth.delegation.AWSPolicyProvider; @@ -4084,14 +4086,12 @@ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() { return instrumentation.newCommitterStatistics(); } - /** - * Return the capabilities of this filesystem instance. - * @param capability string to query the stream support for. - * @return whether the FS instance has the capability. - */ @Override - public boolean hasCapability(String capability) { - + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + makeQualified(path); switch (capability.toLowerCase(Locale.ENGLISH)) { case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER: @@ -4102,7 +4102,31 @@ public boolean hasCapability(String capability) { // select is only supported if enabled return selectBinding.isEnabled(); + case CommonPathCapabilities.FS_CHECKSUMS: + // capability depends on FS configuration + return getConf().getBoolean(ETAG_CHECKSUM_ENABLED, + ETAG_CHECKSUM_ENABLED_DEFAULT); + default: + return super.hasPathCapability(path, capability); + } + } + + /** + * Return the capabilities of this filesystem instance. + * + * This has been supplanted by {@link #hasPathCapability(Path, String)}. + * @param capability string to query the stream support for. + * @return whether the FS instance has the capability. + */ + @Deprecated + @Override + public boolean hasCapability(String capability) { + try { + return hasPathCapability(workingDir, capability); + } catch (IOException ex) { + // should never happen, so log and downgrade. + LOG.debug("Ignoring exception on hasCapability({}})", capability, ex); return false; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java index 877433bab2a0e5..374ccd01350811 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java @@ -82,7 +82,7 @@ private CommitConstants() { /** * Flag to indicate that a store supports magic committers. - * returned in {@code StreamCapabilities} + * returned in {@code PathCapabilities} * Value: {@value}. */ public static final String STORE_CAPABILITY_MAGIC_COMMITTER diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 9cb1efe380fda1..bd834e0f2cbf82 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -1227,7 +1227,8 @@ public int run(String[] args, PrintStream out) } else { println(out, "Filesystem %s is not using S3Guard", fsUri); } - boolean magic = fs.hasCapability( + boolean magic = fs.hasPathCapability( + new Path(s3Path), CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER); println(out, "The \"magic\" committer %s supported", magic ? "is" : "is not"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index fc8d872463cafd..18052338a84e43 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -28,6 +28,7 @@ import org.junit.Assume; import org.junit.Test; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.Path; @@ -142,6 +143,8 @@ public void testEmptyFileChecksums() throws Throwable { Path file1 = touchFile("file1"); EtagChecksum checksum1 = fs.getFileChecksum(file1, 0); LOG.info("Checksum for {}: {}", file1, checksum1); + assertTrue("FS checksum support disabled", + fs.hasPathCapability(file1, CommonPathCapabilities.FS_CHECKSUMS)); assertNotNull("Null file 1 checksum", checksum1); assertNotEquals("file 1 checksum", 0, checksum1.getLength()); assertEquals("checksums", checksum1, @@ -159,6 +162,8 @@ public void testChecksumDisabled() throws Throwable { final S3AFileSystem fs = getFileSystem(); Path file1 = touchFile("file1"); EtagChecksum checksum1 = fs.getFileChecksum(file1, 0); + assertFalse("FS checksum support enabled", + fs.hasPathCapability(file1, CommonPathCapabilities.FS_CHECKSUMS)); assertNull("Checksums are being generated", checksum1); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index b9743858b211bc..1889c05431066a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -1236,9 +1236,12 @@ public static void skipDuringFaultInjection(S3AFileSystem fs) { * Skip a test if the FS isn't marked as supporting magic commits. * @param fs filesystem */ - public static void assumeMagicCommitEnabled(S3AFileSystem fs) { + public static void assumeMagicCommitEnabled(S3AFileSystem fs) + throws IOException { assume("Magic commit option disabled on " + fs, - fs.hasCapability(CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER)); + fs.hasPathCapability( + fs.getWorkingDirectory(), + CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER)); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index d24373acf0561d..155f5d42ac6858 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -41,6 +41,8 @@ import org.apache.hadoop.security.token.TokenIdentifier; import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.CommonPathCapabilities.FS_DELEGATION_TOKENS; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeSessionTestsEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.roundTrip; import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders; @@ -112,6 +114,12 @@ public void testCanonicalization() throws Throwable { uri, new URI(service)); } + @Test + public void testFSDeclaresDTSupport() throws Throwable { + S3AFileSystem fs = getFileSystem(); + assertHasPathCapabilities(fs, fs.getHomeDirectory(), FS_DELEGATION_TOKENS); + } + @Test public void testSaveLoadTokens() throws Throwable { File tokenFile = File.createTempFile("token", "bin"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java index b0e2b8e4bcab0b..455a8a3ebd1c67 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java @@ -550,10 +550,7 @@ public void testFailuresInAbortListing() throws Throwable { @Test public void testWriteNormalStream() throws Throwable { S3AFileSystem fs = getFileSystem(); - Assume.assumeTrue( - "Filesystem does not have magic support enabled: " + fs, - fs.hasCapability(STORE_CAPABILITY_MAGIC_COMMITTER)); - + assumeMagicCommitEnabled(fs); Path destFile = path("normal"); try (FSDataOutputStream out = fs.create(destFile, true)) { out.writeChars("data"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 2d17ca54249578..97fcdc5e20c11c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -517,7 +517,7 @@ public void testProbeForMagic() throws Throwable { String name = fs.getUri().toString(); S3GuardTool.BucketInfo cmd = new S3GuardTool.BucketInfo( getConfiguration()); - if (fs.hasCapability( + if (fs.hasPathCapability(fs.getWorkingDirectory(), CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER)) { // if the FS is magic, expect this to work exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name); diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 3955721a765f13..70d9c6c713fae3 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -25,6 +25,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Locale; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -46,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.ContentSummary.Builder; import org.apache.hadoop.fs.CreateFlag; @@ -1033,4 +1035,21 @@ public static Configuration propagateAccountOptions(Configuration source, } return dest; } + + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + // qualify the path to make sure that it refers to the current FS. + // query the superclass, which triggers argument validation. + boolean superCapability = super.hasPathCapability(path, capability); + switch (capability.toLowerCase(Locale.ENGLISH)) { + case CommonPathCapabilities.FS_ACLS: + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_PERMISSIONS: + return true; + default: + return superCapability; + } + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index f8962d9b170df4..0dff40e1575613 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -51,6 +51,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BufferedFSInputStream; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -68,6 +69,7 @@ import org.apache.hadoop.fs.azure.security.Constants; import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager; import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -3866,4 +3868,20 @@ void updateChmodAllowedUsers(List chmodAllowedUsers) { void updateDaemonUsers(List daemonUsers) { this.daemonUsers = daemonUsers; } + + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + + switch (capability) { + case CommonPathCapabilities.FS_PERMISSIONS: + return true; + // Append support is dynamic + case CommonPathCapabilities.FS_APPEND: + return appendSupportEnabled; + default: + return super.hasPathCapability(path, capability); + } + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index d93822f33d5ba6..60976c1461e741 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -47,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -66,6 +67,7 @@ import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizationException; import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizer; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; +import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -1129,4 +1131,24 @@ private void performAbfsAuthCheck(FsAction action, Path... paths) } } } + + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + // qualify the path to make sure that it refers to the current FS. + makeQualified(path); + + switch (capability) { + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_APPEND: + return true; + case CommonPathCapabilities.FS_ACLS: + return getIsNamespaceEnabled(); + case CommonPathCapabilities.FS_DELEGATION_TOKENS: + return delegationTokenEnabled; + default: + return super.hasPathCapability(path, capability); + } + } } From fda80f5f088668debb25e732cd797155fc240d0d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 24 Apr 2019 00:36:14 +0100 Subject: [PATCH 2/4] HADOOP-15691 path capabilities * Cut the Delegation Token probe as it was too complicated to get right; that can be added later. * By having validatePathCapability(path, string) return the string of the locale-english-lower-case capability, it can be used directlly in the switch() statement, reduce complexity/duplication in all uses, and avoid uses forgetting to do the conversion. * Javadocs for CommonPathCapabilities remove backlinks to FileSystem APIs: they're generic to FS and FC after all. * ChRootedFileSystem calls the capability test on the full path. * ContractTestUtils adds assertLacksPathCapabilities() assertion. Change-Id: Ie5fdec9bb1e64d14880bbe8704ec0417d9073e2b Testing: none yet, waiting for Yetus. ADLs, Azure, S3A to follow --- .../apache/hadoop/fs/AbstractFileSystem.java | 12 ++--- .../apache/hadoop/fs/ChecksumFileSystem.java | 11 +++-- .../hadoop/fs/CommonPathCapabilities.java | 47 +++++++------------ .../org/apache/hadoop/fs/FileContext.java | 4 +- .../java/org/apache/hadoop/fs/FileSystem.java | 11 +---- .../org/apache/hadoop/fs/HarFileSystem.java | 6 +-- .../apache/hadoop/fs/RawLocalFileSystem.java | 7 ++- .../fs/http/AbstractHttpFileSystem.java | 8 ++-- .../fs/impl/PathCapabilitiesSupport.java | 6 ++- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 +++ .../hadoop/fs/viewfs/ViewFileSystem.java | 13 ++--- .../hadoop/fs/contract/ContractTestUtils.java | 25 +++++++++- .../hadoop/hdfs/DistributedFileSystem.java | 12 ++--- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 10 ++-- .../fs/http/client/HttpFSFileSystem.java | 9 ++-- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 11 ++--- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 10 ++-- .../ITestSessionDelegationTokens.java | 8 ---- .../apache/hadoop/fs/adl/AdlFileSystem.java | 11 ++--- .../fs/azure/NativeAzureFileSystem.java | 5 +- .../fs/azurebfs/AzureBlobFileSystem.java | 13 ++--- 21 files changed, 117 insertions(+), 128 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index 68d451f39765d6..ec71e13b9762e0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -29,7 +29,6 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; @@ -46,7 +45,6 @@ import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -62,6 +60,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** * This class provides an interface for implementors of a Hadoop file system * (analogous to the VFS of Unix). Applications do not access this class; @@ -1384,16 +1384,10 @@ public CompletableFuture openFileWithOptions(Path path, public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - // qualify the path to make sure that it refers to the current FS. - makeQualified(path); - switch (capability.toLowerCase(Locale.ENGLISH)) { + switch (validatePathCapabilityArgs(makeQualified(path), capability)) { case CommonPathCapabilities.FS_SYMLINKS: // delegate to the existing supportsSymlinks() call. return supportsSymlinks(); - case CommonPathCapabilities.FS_DELEGATION_TOKENS: - // this is less efficient than it should be. - return getCanonicalServiceName() != null; default: // the feature is not implemented. return false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index e560b1e9ccd823..5e5d29a28bfcee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -43,6 +43,8 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /**************************************************************** * Abstract Checksumed FileSystem. * It provide a basic implementation of a Checksumed FileSystem, @@ -875,21 +877,20 @@ public FSDataOutputStreamBuilder appendFile(Path path) { } /** - * Disable those operations which are disabled so as to guarantee - * checksumming of all created files. + * Disable those operations which the checksummed FS blocks. * {@inheritDoc} */ @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { // query the superclass, which triggers argument validation. - boolean superCapability = super.hasPathCapability(path, capability); - switch (capability.toLowerCase(Locale.ENGLISH)) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CONCAT: return false; default: - return superCapability; + return super.hasPathCapability(p, capability); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java index 4fa899d062b168..d3493b791bb2be 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java @@ -18,10 +18,6 @@ package org.apache.hadoop.fs; -import java.util.List; - -import org.apache.hadoop.fs.permission.FsPermission; - /** * Common path capabilities. */ @@ -31,56 +27,49 @@ public CommonPathCapabilities() { } /** - * Does the Filesystem support - * {@link FileSystem#setAcl(Path, List)}, - * {@link FileSystem#getAclStatus(Path)} + * Does the store support + * {@code FileSystem.setAcl(Path, List)}, + * {@code FileSystem.getAclStatus(Path)} * and related methods? * Value: {@value}. */ public static final String FS_ACLS = "fs.paths.acls"; /** - * Does the Filesystem support {@link FileSystem#append(Path)}? + * Does the store support {@code FileSystem.append(Path)}? * Value: {@value}. */ public static final String FS_APPEND = "fs.paths.append"; /** - * Does the FS support {@link FileSystem#getFileChecksum(Path)}? + * Does the store support {@code FileSystem.getFileChecksum(Path)}? * Value: {@value}. */ public static final String FS_CHECKSUMS = "fs.paths.checksums"; /** - * Does the FS support {@link FileSystem#concat(Path, Path[])}? + * Does the store support {@code FileSystem.concat(Path, Path[])}? * Value: {@value}. */ public static final String FS_CONCAT = "fs.paths.concat"; /** - * Does the filesystem support Delegation Tokens? - * Value: {@value}. - */ - public static final String FS_DELEGATION_TOKENS = - "fs.paths.delegation.tokens"; - - /** - * Does the FS support {@link FileSystem#listCorruptFileBlocks(Path)} ()}? + * Does the store support {@code FileSystem.listCorruptFileBlocks(Path)} ()}? * Value: {@value}. */ public static final String FS_LIST_CORRUPT_FILE_BLOCKS = "fs.paths.list-corrupt-file-blocks"; /** - * Does the FS support - * {@link FileSystem#createPathHandle(FileStatus, Options.HandleOpt...)} + * Does the store support + * {@code FileSystem.createPathHandle(FileStatus, Options.HandleOpt...)} * and related methods? * Value: {@value}. */ public static final String FS_PATHHANDLES = "fs.paths.pathhandles"; /** - * Does the FS support {@link FileSystem#setPermission(Path, FsPermission)} + * Does the store support {@code FileSystem.setPermission(Path, FsPermission)} * and related methods? * Value: {@value}. */ @@ -98,14 +87,14 @@ public CommonPathCapabilities() { "fs.paths.read-only-connector"; /** - * Does the FS support snapshots through - * {@link FileSystem#createSnapshot(Path)} and related methods?? + * Does the store support snapshots through + * {@code FileSystem.createSnapshot(Path)} and related methods?? * Value: {@value}. */ public static final String FS_SNAPSHOTS = "fs.paths.snapshots"; /** - * Does the FS support {@link FileSystem#setStoragePolicy(Path, String)} + * Does the store support {@code FileSystem.setStoragePolicy(Path, String)} * and related methods? * Value: {@value}. */ @@ -113,23 +102,23 @@ public CommonPathCapabilities() { "fs.paths.storagepolicy"; /** - * Does the FS support symlinks through - * {@link FileSystem#createSymlink(Path, Path, boolean)} and related methods? + * Does the store support symlinks through + * {@code FileSystem.createSymlink(Path, Path, boolean)} and related methods? * Value: {@value}. */ public static final String FS_SYMLINKS = "fs.paths.symlinks"; /** - * Does the FS support {@link FileSystem#truncate(Path, long)} ? + * Does the store support {@code FileSystem#truncate(Path, long)} ? * Value: {@value}. */ public static final String FS_TRUNCATE = "fs.paths.truncate"; /** - * Does the Filesystem support XAttributes through - * {@link FileSystem#setXAttr(Path, String, byte[])} and related methods? + * Does the store support XAttributes through + * {@code FileSystem#.setXAttr()} and related methods? * Value: {@value}. */ public static final String FS_XATTRS = "fs.paths.xattrs"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index d43f20db91c469..b2c1369a9c1fe1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -70,6 +70,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** * The FileContext class provides an interface for users of the Hadoop * file system. It exposes a number of file system operations, e.g. create, @@ -2947,7 +2949,7 @@ public CompletableFuture next( */ public boolean hasPathCapability(Path path, String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + validatePathCapabilityArgs(path, capability); return FsLinkResolution.resolve(this, fixRelativePart(path), (fs, p) -> fs.hasPathCapability(p, capability)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index d76c8b6303bdd1..4e9f172a4c79f8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -35,7 +35,6 @@ import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -59,7 +58,6 @@ import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -90,6 +88,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; /**************************************************************** * An abstract base class for a fairly generic filesystem. It @@ -3271,16 +3270,10 @@ public Collection getTrashRoots(boolean allUsers) { */ public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - // qualify the path to make sure that it refers to the current FS. - makeQualified(path); - switch (capability.toLowerCase(Locale.ENGLISH)) { + switch (validatePathCapabilityArgs(makeQualified(path), capability)) { case CommonPathCapabilities.FS_SYMLINKS: // delegate to the existing supportsSymlinks() call. return supportsSymlinks() && areSymlinksEnabled(); - case CommonPathCapabilities.FS_DELEGATION_TOKENS: - // this is less efficient than it should be. - return getCanonicalServiceName() != null; default: // the feature is not implemented. return false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 1a3af41acb018b..5f4c4a236e96c0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.HandleOpt; import org.apache.hadoop.io.IOUtils; @@ -37,6 +36,8 @@ import java.net.URLDecoder; import java.util.*; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** * This is an implementation of the Hadoop Archive * Filesystem. This archive Filesystem has index files @@ -908,8 +909,7 @@ public void setPermission(Path p, FsPermission permission) @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - switch (capability.toLowerCase(Locale.ENGLISH)) { + switch (validatePathCapabilityArgs(path, capability)) { case CommonPathCapabilities.FS_READ_ONLY_CONNECTOR: return true; default: diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index fff266d2a9247a..cf2210575da158 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -41,13 +41,11 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.Optional; -import java.util.Locale; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; @@ -55,6 +53,8 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /**************************************************************** * Implement the FileSystem API for the raw local filesystem. * @@ -1066,8 +1066,7 @@ public Path getLinkTarget(Path f) throws IOException { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - switch (capability.toLowerCase(Locale.ENGLISH)) { + switch (validatePathCapabilityArgs(makeQualified(path), capability)) { case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CONCAT: case CommonPathCapabilities.FS_PATHHANDLES: diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java index 5f02dab13d0da9..987833760215ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PositionedReadable; import org.apache.hadoop.fs.Seekable; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; @@ -37,7 +36,8 @@ import java.io.InputStream; import java.net.URI; import java.net.URLConnection; -import java.util.Locale; + +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; abstract class AbstractHttpFileSystem extends FileSystem { private static final long DEFAULT_BLOCK_SIZE = 4096; @@ -121,9 +121,7 @@ public FileStatus getFileStatus(Path path) throws IOException { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - - switch (capability.toLowerCase(Locale.ENGLISH)) { + switch (validatePathCapabilityArgs(path, capability)) { case CommonPathCapabilities.FS_READ_ONLY_CONNECTOR: return true; default: diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java index 76be0e3bf49d30..ec9a6d68973814 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.impl; +import java.util.Locale; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; @@ -32,12 +34,14 @@ public class PathCapabilitiesSupport { * Validate the arguments to {@code PathCapabilities.hadCapability()}. * @param path path to query the capability of. * @param capability non-null, non-empty string to query the path for support. + * @return the string to use in a switch statement. * @throws IllegalArgumentException if a an argument is invalid. */ - public static void validatehasPathCapabilityArgs( + public static String validatePathCapabilityArgs( final Path path, final String capability) { checkArgument(path != null, "null path"); checkArgument(capability != null && !capability.isEmpty(), "null/empty capability"); + return capability.toLowerCase(Locale.ENGLISH); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index c93225f8fd03d1..773a7b2def265d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -491,4 +491,10 @@ public FutureDataInputStreamBuilder openFile(final Path path) throws IOException, UnsupportedOperationException { return super.openFile(fullPath(path)); } + + @Override + public boolean hasPathCapability(final Path path, final String capability) + throws IOException { + return super.hasPathCapability(fullPath(path), capability); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 94e97f4233bfa3..f7b3a94e4f180c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; @@ -33,7 +34,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -58,12 +58,10 @@ import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.XAttrSetFlag; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.AclUtil; @@ -1041,11 +1039,8 @@ public Path getLinkTarget(Path path) throws IOException { @Override public boolean hasPathCapability(Path path, String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - // qualify the path to make sure that it refers to the current FS. - Path p = makeQualified(path); - - switch (capability.toLowerCase(Locale.ENGLISH)) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { case CommonPathCapabilities.FS_CONCAT: // concat is not supported, as it may be invoked across filesystems. return false; @@ -1059,7 +1054,7 @@ public boolean hasPathCapability(Path path, String capability) capability); } catch (FileNotFoundException e) { // no mount point, nothing will work. - throw new NotInMountpointException(path, "hasPathCapability"); + throw new NotInMountpointException(p, "hasPathCapability"); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 61e0be734bbb7f..f61634943bb7f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -1520,8 +1520,29 @@ public static void assertHasPathCapabilities( final Path path, final String...capabilities) throws IOException { - for (String shouldHaveCapability : capabilities) { - assertTrue("Should have capability: " + shouldHaveCapability, + for (String shouldHaveCapability: capabilities) { + assertTrue("Should have capability: " + shouldHaveCapability + + " under " + path, + source.hasPathCapability(path, shouldHaveCapability)); + } + } + + /** + * Custom assert to test that the named {@link PathCapabilities} + * are not supported. + * + * @param source source (FS, FC, etc) + * @param path path to check + * @param capabilities The array of unexpected capabilities + */ + public static void assertLacksPathCapabilities( + final PathCapabilities source, + final Path path, + final String...capabilities) throws IOException { + + for (String shouldHaveCapability: capabilities) { + assertFalse("Path must not support capability: " + shouldHaveCapability + + " under " + path, source.hasPathCapability(path, shouldHaveCapability)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index f38fb0967f4b7b..3b2bb47d92f805 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -49,7 +49,6 @@ import org.apache.hadoop.fs.GlobalStorageStatistics; import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider; import org.apache.hadoop.fs.InvalidPathHandleException; -import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.PathHandle; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options; @@ -64,7 +63,6 @@ import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.XAttrSetFlag; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -120,10 +118,11 @@ import java.util.Collection; import java.util.EnumSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /**************************************************************** * Implementation of the abstract FileSystem for the DFS system. * This object is the way end-user code interacts with a Hadoop @@ -3417,11 +3416,10 @@ public HdfsDataOutputStreamBuilder appendFile(Path path) { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); // qualify the path to make sure that it refers to the current FS. - makeQualified(path); + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { - switch (capability.toLowerCase(Locale.ENGLISH)) { case CommonPathCapabilities.FS_ACLS: case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CHECKSUMS: @@ -3436,7 +3434,7 @@ public boolean hasPathCapability(final Path path, final String capability) case CommonPathCapabilities.FS_SYMLINKS: return FileSystem.areSymlinksEnabled(); default: - return super.hasPathCapability(path, capability); + return super.hasPathCapability(p, capability); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 8d30dd31c48779..db78574eabb90b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -135,6 +135,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** A FileSystem for HDFS over the web. */ public class WebHdfsFileSystem extends FileSystem implements DelegationTokenRenewer.Renewable, @@ -2096,9 +2098,9 @@ public void setTestProvider(KeyProvider kp) { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - // query the superclass, which triggers argument validation. - boolean superCapability = super.hasPathCapability(path, capability); - switch (capability.toLowerCase(Locale.ENGLISH)) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { + case CommonPathCapabilities.FS_ACLS: case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CHECKSUMS: @@ -2113,7 +2115,7 @@ public boolean hasPathCapability(final Path path, final String capability) // so always return true. return true; default: - return superCapability; + return super.hasPathCapability(p, capability); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 966902ca963a69..0e5aae894d47a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -89,6 +89,8 @@ import java.util.Locale; import java.util.Map; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** * HttpFSServer implementation of the FileSystemAccess FileSystem. *

@@ -1573,9 +1575,8 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() public boolean hasPathCapability(final Path path, final String capability) throws IOException { // query the superclass, which triggers argument validation. - boolean superCapability = super.hasPathCapability(path, capability); - - switch (capability.toLowerCase(Locale.ENGLISH)) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { case CommonPathCapabilities.FS_ACLS: case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CONCAT: @@ -1587,7 +1588,7 @@ public boolean hasPathCapability(final Path path, final String capability) case CommonPathCapabilities.FS_SYMLINKS: return false; default: - return superCapability; + return super.hasPathCapability(p, capability); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 1962f81970dcd6..7f4fc38dffc028 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -36,7 +36,6 @@ import java.util.Date; import java.util.EnumSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -123,7 +122,6 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StreamCapabilities; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.fs.s3a.auth.delegation.AWSPolicyProvider; @@ -154,6 +152,7 @@ import org.apache.hadoop.util.SemaphoredDelegatingExecutor; import static org.apache.hadoop.fs.impl.AbstractFSBuilderImpl.rejectUnknownMandatoryKeys; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.Invoker.*; import static org.apache.hadoop.fs.s3a.S3AUtils.*; @@ -4089,10 +4088,8 @@ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); - // qualify the path to make sure that it refers to the current FS. - makeQualified(path); - switch (capability.toLowerCase(Locale.ENGLISH)) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER: // capability depends on FS configuration @@ -4108,7 +4105,7 @@ public boolean hasPathCapability(final Path path, final String capability) ETAG_CHECKSUM_ENABLED_DEFAULT); default: - return super.hasPathCapability(path, capability); + return super.hasPathCapability(p, capability); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index 18052338a84e43..8f7f1beb4444c9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -35,6 +35,8 @@ import org.apache.hadoop.fs.store.EtagChecksum; import org.apache.hadoop.test.LambdaTestUtils; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertLacksPathCapabilities; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; @@ -143,8 +145,8 @@ public void testEmptyFileChecksums() throws Throwable { Path file1 = touchFile("file1"); EtagChecksum checksum1 = fs.getFileChecksum(file1, 0); LOG.info("Checksum for {}: {}", file1, checksum1); - assertTrue("FS checksum support disabled", - fs.hasPathCapability(file1, CommonPathCapabilities.FS_CHECKSUMS)); + assertHasPathCapabilities(fs, file1, + CommonPathCapabilities.FS_CHECKSUMS); assertNotNull("Null file 1 checksum", checksum1); assertNotEquals("file 1 checksum", 0, checksum1.getLength()); assertEquals("checksums", checksum1, @@ -162,8 +164,8 @@ public void testChecksumDisabled() throws Throwable { final S3AFileSystem fs = getFileSystem(); Path file1 = touchFile("file1"); EtagChecksum checksum1 = fs.getFileChecksum(file1, 0); - assertFalse("FS checksum support enabled", - fs.hasPathCapability(file1, CommonPathCapabilities.FS_CHECKSUMS)); + assertLacksPathCapabilities(fs, file1, + CommonPathCapabilities.FS_CHECKSUMS); assertNull("Checksums are being generated", checksum1); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index 155f5d42ac6858..d24373acf0561d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -41,8 +41,6 @@ import org.apache.hadoop.security.token.TokenIdentifier; import static java.util.Objects.requireNonNull; -import static org.apache.hadoop.fs.CommonPathCapabilities.FS_DELEGATION_TOKENS; -import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeSessionTestsEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.roundTrip; import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders; @@ -114,12 +112,6 @@ public void testCanonicalization() throws Throwable { uri, new URI(service)); } - @Test - public void testFSDeclaresDTSupport() throws Throwable { - S3AFileSystem fs = getFileSystem(); - assertHasPathCapabilities(fs, fs.getHomeDirectory(), FS_DELEGATION_TOKENS); - } - @Test public void testSaveLoadTokens() throws Throwable { File tokenFile = File.createTempFile("token", "bin"); diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 70d9c6c713fae3..278b815782aa86 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -25,7 +25,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Locale; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -72,6 +71,7 @@ import org.apache.hadoop.util.VersionInfo; import static org.apache.hadoop.fs.adl.AdlConfKeys.*; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; /** * A FileSystem to access Azure Data Lake Store. @@ -1039,17 +1039,16 @@ public static Configuration propagateAccountOptions(Configuration source, @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - // qualify the path to make sure that it refers to the current FS. - // query the superclass, which triggers argument validation. - boolean superCapability = super.hasPathCapability(path, capability); - switch (capability.toLowerCase(Locale.ENGLISH)) { + + switch (validatePathCapabilityArgs(makeQualified(path), capability)) { + case CommonPathCapabilities.FS_ACLS: case CommonPathCapabilities.FS_APPEND: case CommonPathCapabilities.FS_CONCAT: case CommonPathCapabilities.FS_PERMISSIONS: return true; default: - return superCapability; + return super.hasPathCapability(path, capability); } } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 0dff40e1575613..a990b60b1c9e53 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -69,7 +69,6 @@ import org.apache.hadoop.fs.azure.security.Constants; import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager; import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -86,6 +85,7 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -3872,9 +3872,8 @@ void updateDaemonUsers(List daemonUsers) { @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); + switch (validatePathCapabilityArgs(path, capability)) { - switch (capability) { case CommonPathCapabilities.FS_PERMISSIONS: return true; // Append support is dynamic diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 60976c1461e741..2a4e76b28bb621 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -67,7 +67,6 @@ import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizationException; import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizer; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; -import org.apache.hadoop.fs.impl.PathCapabilitiesSupport; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -78,6 +77,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + /** * A {@link org.apache.hadoop.fs.FileSystem} for reading and writing files stored on Windows Azure @@ -1135,20 +1136,16 @@ private void performAbfsAuthCheck(FsAction action, Path... paths) @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { - PathCapabilitiesSupport.validatehasPathCapabilityArgs(path, capability); // qualify the path to make sure that it refers to the current FS. - makeQualified(path); - - switch (capability) { + final Path p = makeQualified(path); + switch (validatePathCapabilityArgs(p, capability)) { case CommonPathCapabilities.FS_PERMISSIONS: case CommonPathCapabilities.FS_APPEND: return true; case CommonPathCapabilities.FS_ACLS: return getIsNamespaceEnabled(); - case CommonPathCapabilities.FS_DELEGATION_TOKENS: - return delegationTokenEnabled; default: - return super.hasPathCapability(path, capability); + return super.hasPathCapability(p, capability); } } } From 600d11cc103f43bc006bac41811f1ad8481b79fe Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 23 Aug 2019 22:01:33 +0100 Subject: [PATCH 3/4] HADOOP-15691. Address comments also move capabilties off schemas into fs.schema.capability. why? -consistent with config options. -may make it easier to pass these around (e.g in job confs) -allows for more generic use than just FS/FC Change-Id: Idef1162006aee664b16cefdb99e303fa2f3407dd --- .../apache/hadoop/fs/AbstractFileSystem.java | 8 - .../hadoop/fs/CommonPathCapabilities.java | 28 ++-- .../apache/hadoop/fs/FilterFileSystem.java | 1 + .../apache/hadoop/fs/PathCapabilities.java | 17 +- .../fs/http/AbstractHttpFileSystem.java | 2 +- .../fs/impl/PathCapabilitiesSupport.java | 9 +- .../hadoop/fs/viewfs/ViewFileSystem.java | 4 +- .../site/markdown/filesystem/filesystem.md | 140 ---------------- .../src/site/markdown/filesystem/index.md | 1 + .../markdown/filesystem/pathcapabilities.md | 158 ++++++++++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 25 +-- .../hdfs/client/DfsPathCapabilities.java | 62 +++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 32 ++-- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 6 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 2 + .../hadoop/fs/s3a/commit/CommitConstants.java | 20 ++- .../hadoop/fs/s3a/select/SelectConstants.java | 2 +- .../hadoop/fs/s3a/select/SelectTool.java | 2 +- .../hadoop/fs/s3a/select/ITestS3Select.java | 4 +- 19 files changed, 304 insertions(+), 219 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/DfsPathCapabilities.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index ec71e13b9762e0..0453ca14537c3c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1373,14 +1373,6 @@ public CompletableFuture openFileWithOptions(Path path, new CompletableFuture<>(), () -> open(path, bufferSize)); } - /** - * Return the base capabilities of the filesystems - * may override to declare different behavior. - * @param path path to query the capability of. - * @param capability string to query the stream support for. - * @return true if the capability is supported under that part of the FS. - * @throws IOException on failure - */ public boolean hasPathCapability(final Path path, final String capability) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java index d3493b791bb2be..31e6bac0ccee5b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java @@ -23,7 +23,7 @@ */ public final class CommonPathCapabilities { - public CommonPathCapabilities() { + private CommonPathCapabilities() { } /** @@ -33,32 +33,32 @@ public CommonPathCapabilities() { * and related methods? * Value: {@value}. */ - public static final String FS_ACLS = "fs.paths.acls"; + public static final String FS_ACLS = "fs.capability.paths.acls"; /** * Does the store support {@code FileSystem.append(Path)}? * Value: {@value}. */ - public static final String FS_APPEND = "fs.paths.append"; + public static final String FS_APPEND = "fs.capability.paths.append"; /** * Does the store support {@code FileSystem.getFileChecksum(Path)}? * Value: {@value}. */ - public static final String FS_CHECKSUMS = "fs.paths.checksums"; + public static final String FS_CHECKSUMS = "fs.capability.paths.checksums"; /** * Does the store support {@code FileSystem.concat(Path, Path[])}? * Value: {@value}. */ - public static final String FS_CONCAT = "fs.paths.concat"; + public static final String FS_CONCAT = "fs.capability.paths.concat"; /** * Does the store support {@code FileSystem.listCorruptFileBlocks(Path)} ()}? * Value: {@value}. */ public static final String FS_LIST_CORRUPT_FILE_BLOCKS = - "fs.paths.list-corrupt-file-blocks"; + "fs.capability.paths.list-corrupt-file-blocks"; /** * Does the store support @@ -66,14 +66,14 @@ public CommonPathCapabilities() { * and related methods? * Value: {@value}. */ - public static final String FS_PATHHANDLES = "fs.paths.pathhandles"; + public static final String FS_PATHHANDLES = "fs.capability.paths.pathhandles"; /** * Does the store support {@code FileSystem.setPermission(Path, FsPermission)} * and related methods? * Value: {@value}. */ - public static final String FS_PERMISSIONS = "fs.paths.permissions"; + public static final String FS_PERMISSIONS = "fs.capability.paths.permissions"; /** * Does this filesystem connector only support filesystem read operations? @@ -84,14 +84,14 @@ public CommonPathCapabilities() { * Value: {@value}. */ public static final String FS_READ_ONLY_CONNECTOR = - "fs.paths.read-only-connector"; + "fs.capability.paths.read-only-connector"; /** * Does the store support snapshots through * {@code FileSystem.createSnapshot(Path)} and related methods?? * Value: {@value}. */ - public static final String FS_SNAPSHOTS = "fs.paths.snapshots"; + public static final String FS_SNAPSHOTS = "fs.capability.paths.snapshots"; /** * Does the store support {@code FileSystem.setStoragePolicy(Path, String)} @@ -99,7 +99,7 @@ public CommonPathCapabilities() { * Value: {@value}. */ public static final String FS_STORAGEPOLICY = - "fs.paths.storagepolicy"; + "fs.capability.paths.storagepolicy"; /** * Does the store support symlinks through @@ -107,20 +107,20 @@ public CommonPathCapabilities() { * Value: {@value}. */ public static final String FS_SYMLINKS = - "fs.paths.symlinks"; + "fs.capability.paths.symlinks"; /** * Does the store support {@code FileSystem#truncate(Path, long)} ? * Value: {@value}. */ public static final String FS_TRUNCATE = - "fs.paths.truncate"; + "fs.capability.paths.truncate"; /** * Does the store support XAttributes through * {@code FileSystem#.setXAttr()} and related methods? * Value: {@value}. */ - public static final String FS_XATTRS = "fs.paths.xattrs"; + public static final String FS_XATTRS = "fs.capability.paths.xattrs"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 557b072cd9440f..3bc3cb2e9b07a7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -730,6 +730,7 @@ protected CompletableFuture openFileWithOptions( bufferSize); } + @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { return fs.hasPathCapability(path, capability); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java index 8a3cf525397c94..d3492568f468f8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathCapabilities.java @@ -24,24 +24,31 @@ * The Path counterpoint to {@link StreamCapabilities}; a query to see if, * a FileSystem/FileContext instance has a specific capability under the given * path. + * Other classes may also implement the interface, as desired. + * + * See {@link CommonPathCapabilities} for the well-known capabilities. */ public interface PathCapabilities { /** - * Probe for a filesystem instance offering a specific capability under the - * given path. - * If the function returns {@code true}, the filesystem is explicitly + * Probe for a specific capability under the given path. + * If the function returns {@code true}, this instance is explicitly * declaring that the capability is available. * If the function returns {@code false}, it can mean one of: *

    *
  • The capability is not known.
    • *
  • The capability is known but it is not supported.
    • *
  • The capability is known but the filesystem does not know if it - * is supported under the supplied path it.
    • + * is supported under the supplied path. *
* The core guarantee which a caller can rely on is: if the predicate * returns true, then the specific operation/behavior can be expected to be - * supported. + * supported. However a specific call may be rejected for permission reasons, + * the actual file/directory not being present, or some other failure during + * the attempted execution of the operation. + *

+ * Implementors: {@link org.apache.hadoop.fs.impl.PathCapabilitiesSupport} + * can be used to help implement this method. * @param path path to query the capability of. * @param capability non-null, non-empty string to query the path for support. * @return true if the capability is supported under that part of the FS. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java index 987833760215ce..baf0a8187efd08 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java @@ -125,7 +125,7 @@ public boolean hasPathCapability(final Path path, final String capability) case CommonPathCapabilities.FS_READ_ONLY_CONNECTOR: return true; default: - return false; + return super.hasPathCapability(path, capability); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java index ec9a6d68973814..9332ac6e7eedba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/PathCapabilitiesSupport.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathCapabilities; import static com.google.common.base.Preconditions.checkArgument; @@ -31,7 +32,8 @@ public class PathCapabilitiesSupport { /** - * Validate the arguments to {@code PathCapabilities.hadCapability()}. + * Validate the arguments to + * {@link PathCapabilities#hasPathCapability(Path, String)}. * @param path path to query the capability of. * @param capability non-null, non-empty string to query the path for support. * @return the string to use in a switch statement. @@ -40,8 +42,9 @@ public class PathCapabilitiesSupport { public static String validatePathCapabilityArgs( final Path path, final String capability) { checkArgument(path != null, "null path"); - checkArgument(capability != null && !capability.isEmpty(), - "null/empty capability"); + checkArgument(capability != null, "capability parameter is null"); + checkArgument(!capability.isEmpty(), + "capability parameter is empty string"); return capability.toLowerCase(Locale.ENGLISH); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index f7b3a94e4f180c..faa374a39789bd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1029,8 +1029,7 @@ public Path getLinkTarget(Path path) throws IOException { } /** - * Fail fast on known write capabilities (append, concat), - * forward the rest to the viewed FS. + * Reject the concat operation; forward the rest to the viewed FS. * @param path path to query the capability of. * @param capability string to query the stream support for. * @return the capability @@ -1045,6 +1044,7 @@ public boolean hasPathCapability(Path path, String capability) // concat is not supported, as it may be invoked across filesystems. return false; default: + // no break } // otherwise, check capabilities of mounted FS. try { diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 950de824ff6a53..a2458ee891448d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -1493,143 +1493,3 @@ in:readahead | READAHEAD | CanSetReadahead | Set the readahead on the input st dropbehind | DROPBEHIND | CanSetDropBehind | Drop the cache. in:unbuffer | UNBUFFER | CanUnbuffer | Reduce the buffering on the input stream. -## interface `PathCapabilities` - -The `PathCapabilities` provides a way to programmatically query the operations -offered by a FileSystem or FileContext instance under a given path. - -```java -public interface PathCapabilities { - boolean hasPathCapability(Path path, String capability) - throws IOException; -} -``` - -There are a number of goals here: - -1. Allow callers to probe for optional filesystem operations without actually -having to invoke them. -1. Allow filesystems with their own optional per-instance features to declare -whether or not they are active for the specific instance. -1. Allow for fileystem connectors which work with object stores to expose the -fundamental difference in semantics of these stores (e.g: files not visible -until closed, file rename being `O(data)`), directory rename being non-atomic, -etc. - -### Available Capabilities - -Capabilities are defined with a store prefix and an arbitrary (but hopefully -meaningful) string. - -All custom filesystem-specific capabilities MUST be given the prefix of that -filesystem schema. The standard schemas are: - -* `fs` : File system capabilities -* `object` : Object capabilities. - -The exact set of operations and their names are evolving. - -Consult the javadocs for `org.apache.hadoop.fs.PathCapabilities` for the -standard set of capabilities. - -Individual filesystems may offer their own set of capabilities which -can be probed for. These begin with the same prefix as the filesystem schema, -such as `hdfs:` or `s3a:`. - -### `boolean hasPathCapability(path, capability)` - -Probe for a filesystem instance offering a specific capability under the -given path. - -#### Postconditions - -```python -if fs_supports_the_feature(path, capability): - return True -else: - return False -``` - -Return: `True`, iff the specific capability is, the the best of the -knowledge of the client application, available. - -A filesystem instance *MUST NOT* return `True` for any capability unless it is -known to be supported by that specific instance. As a result, if a caller -probes for a capability then it can assume that the specific feature/semantics -are available. - -If the probe returns `False` then it can mean one of: - -1. The capability is unknown. -1. The capability is known, but known to be unavailable on this instance. - -This predicate is intended to be low cost. If it requires remote calls other -than path/link resolution, it SHOULD conclude that the availability -of the feature is unknown and return `False`. - -The predicate MUST also be side-effect free. - -*Validity of paths* -There is no requirement that the existence of the path must be checked; -the parameter exists so that any filesystem which relays operations to other -filesystems (e.g `viewfs`) can resolve and relay it to the nested filesystem. -Consider the call to be *relatively* lightweight. - -Because of this, it may be that while the filesystem declares that -it supports a capability under a path, the actual invocation of the operation -may fail for other reasons. - -As an example, while a filesystem may support `append()` under a path, -if invoked on a directory, the call may fail. - -That is for a path `root = new Path("/")`: the capabilities call may succeed - -```java -fs.hasCapabilities(root, "fs:append") == true -``` - -But a subsequent call to the operation on that specific path may fail, -because the root path is a directory -```java -fs.append(root) -``` - -The `hasCapabilities(path, capability)` probe is therefore declaring that -the operation will not be rejected as unsupported, not that a specific invocation -will be considered valid. - -*Duration of availability* - -As the state of a remote store changes,so may path capabilities. This -may be due to changes in the local state of the fileystem (e.g. symbolic links -or mount points changing), or changes in its functionality (e.g. a feature -becoming availaible/unavailable due to operational changes, system upgrades, etc.) - -*Capabilities which must be invoked to determine availablity* - -Some operations may be known by the filesystem client, and believed to be available, -but may actually fail when invoked due to the state of the underlying -filesystem —state which is cannot be determined except by attempting -side-effecting operations. - -A key example of this is symbolic links and the local filesystem. -The filesystem declares that it supports this unless symbolic links are explicitly -disabled —when invoked they may actually fail. - -### Implementors Notes - -Implementors MUST NOT return `true` for any capability which is not guaranteed -to be supported. To return `true` indicates that the implementation/deployment -of the filesystem does, to the best of the knowledge of the filesystem client, -offer the desired operations *and semantics* queried for. - -For performance reasons, implementations SHOULD NOT check the path for -existence, unless it needs to resolve symbolic links in parts of the path -to determine whether a feature is present.This required of `FileContext` -and `viewfs`. It may also be needed to - -Individual filesystems MUST NOT define new `fs:`-prefixed capabilities. -Instead they MUST do one of the following: - -* Define and stabilize new cross-filesystem capability flags (preferred). -* Use the schema of the filesystem to as a prefix for their own options. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md index 6b4399ea2123ce..df538ee6cf96b8 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md @@ -33,6 +33,7 @@ HDFS as these are commonly expected by Hadoop client applications. 1. [Model](model.html) 1. [FileSystem class](filesystem.html) 1. [FSDataInputStream class](fsdatainputstream.html) +1. [PathCapabilities interface](pathcapabilities.html) 1. [FSDataOutputStreamBuilder class](fsdataoutputstreambuilder.html) 2. [Testing with the Filesystem specification](testing.html) 2. [Extending the specification and its tests](extending.html) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md new file mode 100644 index 00000000000000..9443baf0529a76 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md @@ -0,0 +1,158 @@ + + +# interface `PathCapabilities` + +The `PathCapabilities` interface provides a way to programmatically query the +operations offered under a given path by an instance of `FileSystem`, `FileContext` +or other implementing class. + +```java +public interface PathCapabilities { + boolean hasPathCapability(Path path, String capability) + throws IOException; +} +``` + +There are a number of goals here: + +1. Allow callers to probe for optional filesystem operations without actually +having to invoke them. +1. Allow filesystems with their own optional per-instance features to declare +whether or not they are active for the specific instance. +1. Allow for fileystem connectors which work with object stores to expose the +fundamental difference in semantics of these stores (e.g: files not visible +until closed, file rename being `O(data)`), directory rename being non-atomic, +etc. + +### Available Capabilities + +Capabilities are defined as strings and split into "Common Capabilites" +and non-standard ones for a specific store. + +The common capabilities are all defined under the prefix `fs.capability.` + +Consult the javadocs for `org.apache.hadoop.fs.CommonPathCapabilities` for these. + + +Individual filesystems MAY offer their own set of capabilities which +can be probed for. These MUST begin with `fs.` + the filesystem schema + + `.capability`. For example `fs.s3a.capability.select.sql`; + +### `boolean hasPathCapability(path, capability)` + +Probe for the instance offering a specific capability under the +given path. + +#### Postconditions + +```python +if fs_supports_the_feature(path, capability): + return True +else: + return False +``` + +Return: `True`, iff the specific capability is available. + +A filesystem instance *MUST NOT* return `True` for any capability unless it is +known to be supported by that specific instance. As a result, if a caller +probes for a capability then it can assume that the specific feature/semantics +are available. + +If the probe returns `False` then it can mean one of: + +1. The capability is unknown. +1. The capability is known, and known to be unavailable on this instance. +1. The capability is known but this local class does not know if it is supported + under the supplied path. + +This predicate is intended to be low cost. If it requires remote calls other +than path/link resolution, it SHOULD conclude that the availability +of the feature is unknown and return `False`. + +The predicate MUST also be side-effect free. + +*Validity of paths* +There is no requirement that the existence of the path must be checked; +the parameter exists so that any filesystem which relays operations to other +filesystems (e.g `viewfs`) can resolve and relay it to the nested filesystem. +Consider the call to be *relatively* lightweight. + +Because of this, it may be that while the filesystem declares that +it supports a capability under a path, the actual invocation of the operation +may fail for other reasons. + +As an example, while a filesystem may support `append()` under a path, +if invoked on a directory, the call may fail. + +That is for a path `root = new Path("/")`: the capabilities call may succeed + +```java +fs.hasCapabilities(root, "fs.capability.append") == true +``` + +But a subsequent call to the operation on that specific path may fail, +because the root path is a directory: + +```java +fs.append(root) +``` + + +Similarly, there is no checking that the caller has the permission to +perform a specific operation: just because a feature is available on that +path does not mean that the caller can execute the operation. + +The `hasCapabilities(path, capability)` probe is therefore declaring that +the operation will not be rejected as unsupported, not that a specific invocation +will be permitted on that path by the caller. + +*Duration of availability* + +As the state of a remote store changes,so may path capabilities. This +may be due to changes in the local state of the fileystem (e.g. symbolic links +or mount points changing), or changes in its functionality (e.g. a feature +becoming availaible/unavailable due to operational changes, system upgrades, etc.) + +*Capabilities which must be invoked to determine availablity* + +Some operations may be known by the client connector, and believed to be available, +but may actually fail when invoked due to the state and permissons of the remote +store —state which is cannot be determined except by attempting +side-effecting operations. + +A key example of this is symbolic links and the local filesystem. +The filesystem declares that it supports this unless symbolic links are explicitly +disabled —when invoked they may actually fail. + +### Implementors Notes + +Implementors *MUST NOT* return `true` for any capability which is not guaranteed +to be supported. To return `true` indicates that the implementation/deployment +of the filesystem does, to the best of the knowledge of the filesystem client, +offer the desired operations *and semantics* queried for. + +For performance reasons, implementations *SHOULD NOT* check the path for +existence, unless it needs to resolve symbolic links in parts of the path +to determine whether a feature is present. This is required of `FileContext` +and `viewfs`. + +Individual filesystems *MUST NOT* unilaterally define new `fs.capability`-prefixed +capabilities. Instead they *MUST* do one of the following: + +* Define and stabilize new cross-filesystem capability flags (preferred), +and so formally add a new `fs.capability` value. +* Use the schema of the filesystem to as a prefix for their own options, +e.g `fs.hdfs.` diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 3b2bb47d92f805..5a05ffef3be970 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -68,6 +68,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType; +import org.apache.hadoop.hdfs.client.DfsPathCapabilities; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator; @@ -3410,7 +3411,7 @@ public HdfsDataOutputStreamBuilder appendFile(Path path) { /** * HDFS client capabilities. - * Keep {@code WebHdfsFileSystem} in sync. + * Uses {@link DfsPathCapabilities} to keep {@code WebHdfsFileSystem} in sync. * {@inheritDoc} */ @Override @@ -3418,23 +3419,11 @@ public boolean hasPathCapability(final Path path, final String capability) throws IOException { // qualify the path to make sure that it refers to the current FS. final Path p = makeQualified(path); - switch (validatePathCapabilityArgs(p, capability)) { - - case CommonPathCapabilities.FS_ACLS: - case CommonPathCapabilities.FS_APPEND: - case CommonPathCapabilities.FS_CHECKSUMS: - case CommonPathCapabilities.FS_CONCAT: - case CommonPathCapabilities.FS_LIST_CORRUPT_FILE_BLOCKS: - case CommonPathCapabilities.FS_PATHHANDLES: - case CommonPathCapabilities.FS_PERMISSIONS: - case CommonPathCapabilities.FS_SNAPSHOTS: - case CommonPathCapabilities.FS_STORAGEPOLICY: - case CommonPathCapabilities.FS_XATTRS: - return true; - case CommonPathCapabilities.FS_SYMLINKS: - return FileSystem.areSymlinksEnabled(); - default: - return super.hasPathCapability(p, capability); + Optional cap = DfsPathCapabilities.hasPathCapability(p, + capability); + if (cap.isPresent()) { + return cap.get(); } + return super.hasPathCapability(p, capability); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/DfsPathCapabilities.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/DfsPathCapabilities.java new file mode 100644 index 00000000000000..6cad69a46c4e86 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/DfsPathCapabilities.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.client; + +import java.util.Optional; + +import org.apache.hadoop.fs.CommonPathCapabilities; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; + +public final class DfsPathCapabilities { + + private DfsPathCapabilities() { + } + + /** + * Common implementation of {@code hasPathCapability} for DFS and webhdfs. + * @param path path to check + * @param capability capability + * @return either a value to return or, if empty, a cue for the FS to + * pass up to its superclass. + */ + public static Optional hasPathCapability(final Path path, + final String capability) { + switch (validatePathCapabilityArgs(path, capability)) { + + case CommonPathCapabilities.FS_ACLS: + case CommonPathCapabilities.FS_APPEND: + case CommonPathCapabilities.FS_CHECKSUMS: + case CommonPathCapabilities.FS_CONCAT: + case CommonPathCapabilities.FS_LIST_CORRUPT_FILE_BLOCKS: + case CommonPathCapabilities.FS_PATHHANDLES: + case CommonPathCapabilities.FS_PERMISSIONS: + case CommonPathCapabilities.FS_SNAPSHOTS: + case CommonPathCapabilities.FS_STORAGEPOLICY: + case CommonPathCapabilities.FS_XATTRS: + return Optional.of(true); + case CommonPathCapabilities.FS_SYMLINKS: + return Optional.of(FileSystem.areSymlinksEnabled()); + default: + return Optional.empty(); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index db78574eabb90b..d0b10cbbcf813e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; @@ -94,6 +95,7 @@ import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HAUtilClient; import org.apache.hadoop.hdfs.HdfsKMSUtil; +import org.apache.hadoop.hdfs.client.DfsPathCapabilities; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.DirectoryListing; @@ -2090,33 +2092,21 @@ public void setTestProvider(KeyProvider kp) { } /** - * This filesystem's capabilities must be in sync with that of HDFS. - * @param path path to query the capability of. - * @param capability string to query the stream support for. - * @return true if a capability is supported. + * HDFS client capabilities. + * Uses {@link DfsPathCapabilities} to keep in sync with HDFS. + * {@inheritDoc} */ @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { + // qualify the path to make sure that it refers to the current FS. final Path p = makeQualified(path); - switch (validatePathCapabilityArgs(p, capability)) { - - case CommonPathCapabilities.FS_ACLS: - case CommonPathCapabilities.FS_APPEND: - case CommonPathCapabilities.FS_CHECKSUMS: - case CommonPathCapabilities.FS_CONCAT: - case CommonPathCapabilities.FS_PERMISSIONS: - case CommonPathCapabilities.FS_SNAPSHOTS: - case CommonPathCapabilities.FS_STORAGEPOLICY: - case CommonPathCapabilities.FS_XATTRS: - return true; - case CommonPathCapabilities.FS_SYMLINKS: - // there's no checking of the {symlinksEnabled} flag in this class,\ - // so always return true. - return true; - default: - return super.hasPathCapability(p, capability); + Optional cap = DfsPathCapabilities.hasPathCapability(p, + capability); + if (cap.isPresent()) { + return cap.get(); } + return super.hasPathCapability(p, capability); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index bdffed4b254927..a60f9af2a938bd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -497,17 +497,19 @@ S3AInstrumentation.OutputStreamStatistics getStatistics() { * @param capability string to query the stream support for. * @return true if the capability is supported by this instance. */ + @SuppressWarnings("deprecation") @Override public boolean hasCapability(String capability) { switch (capability.toLowerCase(Locale.ENGLISH)) { // does the output stream have delayed visibility case CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT: + case CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT_OLD: return !putTracker.outputImmediatelyVisible(); // The flush/sync options are absolutely not supported - case "hflush": - case "hsync": + case StreamCapabilities.HFLUSH: + case StreamCapabilities.HSYNC: return false; default: diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 7f4fc38dffc028..159505b055bd55 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -4085,6 +4085,7 @@ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() { return instrumentation.newCommitterStatistics(); } + @SuppressWarnings("deprecation") @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { @@ -4092,6 +4093,7 @@ public boolean hasPathCapability(final Path path, final String capability) switch (validatePathCapabilityArgs(p, capability)) { case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER: + case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER_OLD: // capability depends on FS configuration return isMagicCommitEnabled(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java index 374ccd01350811..c9b0337bcb26a1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java @@ -78,7 +78,7 @@ private CommitConstants() { * Value: {@value}. */ public static final String STREAM_CAPABILITY_MAGIC_OUTPUT - = "s3a:magic.output.stream"; + = "fs.s3a.capability.magic.output.stream"; /** * Flag to indicate that a store supports magic committers. @@ -86,6 +86,24 @@ private CommitConstants() { * Value: {@value}. */ public static final String STORE_CAPABILITY_MAGIC_COMMITTER + = "fs.s3a.capability.magic.committer"; + + /** + * Flag to indicate whether a stream is a magic output stream; + * returned in {@code StreamCapabilities} + * Value: {@value}. + */ + @Deprecated + public static final String STREAM_CAPABILITY_MAGIC_OUTPUT_OLD + = "s3a:magic.output.stream"; + + /** + * Flag to indicate that a store supports magic committers. + * returned in {@code PathCapabilities} + * Value: {@value}. + */ + @Deprecated + public static final String STORE_CAPABILITY_MAGIC_COMMITTER_OLD = "s3a:magic.committer"; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectConstants.java index d74411d2f92ca1..0e2bf914f83c55 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectConstants.java @@ -50,7 +50,7 @@ private SelectConstants() { * Does the FS Support S3 Select? * Value: {@value}. */ - public static final String S3_SELECT_CAPABILITY = "s3a:fs.s3a.select.sql"; + public static final String S3_SELECT_CAPABILITY = "fs.s3a.capability.select.sql"; /** * Flag: is S3 select enabled? diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java index 61409f8ea12211..4b362c667ece61 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java @@ -234,7 +234,7 @@ public int run(String[] args, PrintStream out) } setFilesystem((S3AFileSystem) fs); - if (!getFilesystem().hasCapability(S3_SELECT_CAPABILITY)) { + if (!getFilesystem().hasPathCapability(path, S3_SELECT_CAPABILITY)) { // capability disabled throw new ExitUtil.ExitException(EXIT_SERVICE_UNAVAILABLE, SELECT_IS_DISABLED + " for " + file); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java index 1f2faa209a923e..d6058d19521bef 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java @@ -102,9 +102,9 @@ public class ITestS3Select extends AbstractS3SelectTest { @Override public void setup() throws Exception { super.setup(); - Assume.assumeTrue("S3 Select is not enabled", - getFileSystem().hasCapability(S3_SELECT_CAPABILITY)); csvPath = path(getMethodName() + ".csv"); + Assume.assumeTrue("S3 Select is not enabled", + getFileSystem().hasPathCapability(csvPath, S3_SELECT_CAPABILITY)); selectConf = new Configuration(false); selectConf.setBoolean(SELECT_ERRORS_INCLUDE_SQL, true); createStandardCsvFile(getFileSystem(), csvPath, ALL_QUOTES); From 958c40258d3dfc3577aa0fe99c067cbb1cc9d422 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 24 Sep 2019 18:44:12 +0100 Subject: [PATCH 4/4] HADOOP-15691 -fix final nits. Change-Id: I30918995e3eee6b9a7c5da8d90c19f9c9aed2496 --- .../src/site/markdown/filesystem/pathcapabilities.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md index 9443baf0529a76..e053bfbaede9b7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/pathcapabilities.md @@ -47,7 +47,7 @@ Consult the javadocs for `org.apache.hadoop.fs.CommonPathCapabilities` for these Individual filesystems MAY offer their own set of capabilities which -can be probed for. These MUST begin with `fs.` + the filesystem schema + +can be probed for. These MUST begin with `fs.` + the filesystem scheme + `.capability`. For example `fs.s3a.capability.select.sql`; ### `boolean hasPathCapability(path, capability)` @@ -154,5 +154,5 @@ capabilities. Instead they *MUST* do one of the following: * Define and stabilize new cross-filesystem capability flags (preferred), and so formally add a new `fs.capability` value. -* Use the schema of the filesystem to as a prefix for their own options, +* Use the scheme of the filesystem to as a prefix for their own options, e.g `fs.hdfs.`