diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 91da02dea85..7dabb6eb4a7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -96,6 +96,17 @@ can be queried to find if the path has an ACL. `getFileStatus(Path p).isEncrypte can be queried to find if the path is encrypted. `getFileStatus(Path p).isErasureCoded()` will tell if the path is erasure coded or not. +YARN's distributed cache lets applications add paths to be cached across +containers and applications via `Job.addCacheFile()` and `Job.addCacheArchive()`. +The cache treats world-readable resources paths added as shareable across +applications, and downloads them differently, unless they are declared as encrypted. + +To avoid failures during container launching, especially when delegation tokens +are used, filesystems and object stores which not implement POSIX access permissions +for both files and directories, MUST always return `true` to the `isEncrypted()` +predicate. This can be done by setting the `encrypted` flag to true when creating +the `FileStatus` instance. + ### `Path getHomeDirectory()` The function `getHomeDirectory` returns the home directory for the FileSystem diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java index ccf188f1202..8931ce5cd6c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java @@ -73,10 +73,20 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas final Path path = path("file"); createFile(getFileSystem(), path, false, new byte[0]); final FileStatus stat = getFileSystem().getFileStatus(path); - assertFalse("Expecting false for stat.isEncrypted()", + assertEquals("Result wrong for for isEncrypted() in " + stat, + areZeroByteFilesEncrypted(), stat.isEncrypted()); } + /** + * Are zero byte files encrypted. This is implicitly + * false for filesystems which do not encrypt. + * @return true iff zero byte files are encrypted. + */ + protected boolean areZeroByteFilesEncrypted() { + return false; + } + @Test public void testOpenReadDir() throws Throwable { describe("create & read a directory"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java index be08afe4b10..ca6ca908bec 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java @@ -54,7 +54,9 @@ public class S3AFileStatus extends FileStatus { public S3AFileStatus(Tristate isemptydir, Path path, String owner) { - super(0, true, 1, 0, 0, path); + super(0, true, 1, 0, 0, 0, + null, null, null, null, + path, false, true, false); isEmptyDirectory = isemptydir; setOwner(owner); setGroup(owner); @@ -70,7 +72,9 @@ public class S3AFileStatus extends FileStatus { */ public S3AFileStatus(long length, long modification_time, Path path, long blockSize, String owner) { - super(length, false, 1, blockSize, modification_time, path); + super(length, false, 1, blockSize, modification_time, 0, + null, null, null, null, + path, false, true, false); isEmptyDirectory = Tristate.FALSE; setOwner(owner); setGroup(owner); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java index 8e338b71d11..d78273b1471 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java @@ -45,4 +45,13 @@ public class ITestS3AContractOpen extends AbstractContractOpenTest { protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } + + /** + * S3A always declares zero byte files as encrypted. + * @return true, always. + */ + @Override + protected boolean areZeroByteFilesEncrypted() { + return true; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/mapreduce/filecache/TestS3AResourceScope.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/mapreduce/filecache/TestS3AResourceScope.java new file mode 100644 index 00000000000..f1788ef5bdd --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/mapreduce/filecache/TestS3AResourceScope.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.filecache; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test how S3A resources are scoped in YARN caching. + * In this package to make use of package-private methods of + * {@link ClientDistributedCacheManager}. + */ +public class TestS3AResourceScope extends HadoopTestBase { + + private static final Path PATH = new Path("s3a://example/path"); + + @Test + public void testS3AFilesArePrivate() throws Throwable { + S3AFileStatus status = new S3AFileStatus(false, PATH, "self"); + assertTrue("Not encrypted: " + status, status.isEncrypted()); + assertNotExecutable(status); + } + + @Test + public void testS3AFilesArePrivateOtherContstructor() throws Throwable { + S3AFileStatus status = new S3AFileStatus(0, 0, PATH, 1, "self"); + assertTrue("Not encrypted: " + status, status.isEncrypted()); + assertNotExecutable(status); + } + + private void assertNotExecutable(final S3AFileStatus status) + throws IOException { + Map cache = new HashMap<>(); + cache.put(PATH.toUri(), status); + assertFalse("Should not have been executable " + status, + ClientDistributedCacheManager.ancestorsHaveExecutePermissions( + null, PATH, cache)); + } +}