sourceEntries = filteringRemoteIterator(
+ listStatusIterator(path),
+ (st) -> filter.accept(st.getPath()));
+ // and then map that to a remote iterator of located file status
+ // entries, propagating any etags.
+ return mappingRemoteIterator(sourceEntries,
+ st -> new AbfsLocatedFileStatus(st,
+ st.isFile()
+ ? getFileBlockLocations(st, 0, st.getLen())
+ : null));
+ }
+
private FileStatus tryGetFileStatus(final Path f, TracingContext tracingContext) {
try {
return getFileStatus(f, tracingContext);
@@ -1490,6 +1525,8 @@ public class AzureBlobFileSystem extends FileSystem
switch (validatePathCapabilityArgs(p, capability)) {
case CommonPathCapabilities.FS_PERMISSIONS:
case CommonPathCapabilities.FS_APPEND:
+ case CommonPathCapabilities.ETAGS_AVAILABLE:
+ case CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME:
return true;
case CommonPathCapabilities.FS_ACLS:
return getIsNamespaceEnabled(
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 27beb34fccd..d86a3d96846 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -65,6 +65,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.EtagSource;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -952,7 +953,7 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
final long blockSize = abfsConfiguration.getAzureBlockSize();
final AbfsHttpOperation result = op.getResult();
- final String eTag = result.getResponseHeader(HttpHeaderConfigurations.ETAG);
+ String eTag = extractEtagHeader(result);
final String lastModified = result.getResponseHeader(HttpHeaderConfigurations.LAST_MODIFIED);
final String permissions = result.getResponseHeader((HttpHeaderConfigurations.X_MS_PERMISSIONS));
final boolean hasAcl = AbfsPermission.isExtendedAcl(permissions);
@@ -1710,10 +1711,27 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
return new AbfsPerfInfo(abfsPerfTracker, callerName, calleeName);
}
- private static class VersionedFileStatus extends FileStatus {
- private final String version;
+ /**
+ * A File status with version info extracted from the etag value returned
+ * in a LIST or HEAD request.
+ * The etag is included in the java serialization.
+ */
+ private static final class VersionedFileStatus extends FileStatus
+ implements EtagSource {
- VersionedFileStatus(
+ /**
+ * The superclass is declared serializable; this subclass can also
+ * be serialized.
+ */
+ private static final long serialVersionUID = -2009013240419749458L;
+
+ /**
+ * The etag of an object.
+ * Not-final so that serialization via reflection will preserve the value.
+ */
+ private String version;
+
+ private VersionedFileStatus(
final String owner, final String group, final FsPermission fsPermission, final boolean hasAcl,
final long length, final boolean isdir, final int blockReplication,
final long blocksize, final long modificationTime, final Path path,
@@ -1774,6 +1792,11 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
return this.version;
}
+ @Override
+ public String getEtag() {
+ return getVersion();
+ }
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
@@ -1879,4 +1902,30 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
}
return true;
}
+
+ /**
+ * Get the etag header from a response, stripping any quotations.
+ * see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @param result response to process.
+ * @return the quote-unwrapped etag.
+ */
+ private static String extractEtagHeader(AbfsHttpOperation result) {
+ String etag = result.getResponseHeader(HttpHeaderConfigurations.ETAG);
+ if (etag != null) {
+ // strip out any wrapper "" quotes which come back, for consistency with
+ // list calls
+ if (etag.startsWith("W/\"")) {
+ // Weak etag
+ etag = etag.substring(3);
+ } else if (etag.startsWith("\"")) {
+ // strong etag
+ etag = etag.substring(1);
+ }
+ if (etag.endsWith("\"")) {
+ // trailing quote
+ etag = etag.substring(0, etag.length() - 1);
+ }
+ }
+ return etag;
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java
new file mode 100644
index 00000000000..29da2c50435
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * {@link LocatedFileStatus} extended to also carry an ETag.
+ */
+public class AbfsLocatedFileStatus extends LocatedFileStatus implements EtagSource {
+
+ private static final long serialVersionUID = -8185960773314341594L;
+
+ /**
+ * etag; may be null.
+ */
+ private final String etag;
+
+ public AbfsLocatedFileStatus(FileStatus status, BlockLocation[] locations) {
+ super(checkNotNull(status), locations);
+ if (status instanceof EtagSource) {
+ this.etag = ((EtagSource) status).getEtag();
+ } else {
+ this.etag = null;
+ }
+ }
+
+ @Override
+ public String getEtag() {
+ return etag;
+ }
+
+ @Override
+ public String toString() {
+ return "AbfsLocatedFileStatus{"
+ + "etag='" + etag + '\'' + "} "
+ + super.toString();
+ }
+ // equals() and hashCode() overridden to avoid FindBugs warning.
+ // Base implementation is equality on Path only, which is still appropriate.
+
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
new file mode 100644
index 00000000000..d498ae71a4b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractEtagTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for etag support.
+ */
+public class ITestAbfsFileSystemContractEtag extends AbstractContractEtagTest {
+ private final boolean isSecure;
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsFileSystemContractEtag() throws Exception {
+ binding = new ABFSContractTestBinding();
+ this.isSecure = binding.isSecureMode();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ // Base rename contract test class re-uses the test folder
+ // This leads to failures when the test is re-run as same ABFS test
+ // containers are re-used for test run and creation of source and
+ // destination test paths fail, as they are already present.
+ binding.getFileSystem().delete(binding.getTestPath(), true);
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return binding.getRawConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, isSecure);
+ }
+}