From 39f07295fb813080e65d06cb47003feeef3453f0 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Tue, 12 Jan 2016 12:18:33 -0800 Subject: [PATCH] HADOOP-11262. Enable YARN to use S3A. (Pieter Reuse via lei) (cherry picked from commit 126705f67eaa6d866f7572fbddf133c5d7552353) --- .../hadoop-common/CHANGES.txt | 2 + .../src/main/resources/core-default.xml | 6 + .../fs/FileContextMainOperationsBaseTest.java | 2 +- hadoop-tools/hadoop-aws/pom.xml | 18 ++- .../org/apache/hadoop/fs/s3a/Constants.java | 2 + .../java/org/apache/hadoop/fs/s3a/S3A.java | 47 ++++++ .../apache/hadoop/fs/s3a/S3AFileStatus.java | 27 +++- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 17 ++- .../site/markdown/tools/hadoop-aws/index.md | 12 +- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 21 +++ .../s3a/fileContext/TestS3AFileContext.java | 23 +++ .../TestS3AFileContextCreateMkdir.java | 35 +++++ .../TestS3AFileContextMainOperations.java | 60 ++++++++ .../TestS3AFileContextStatistics.java | 60 ++++++++ .../fileContext/TestS3AFileContextURI.java | 43 ++++++ .../fileContext/TestS3AFileContextUtil.java | 34 +++++ .../apache/hadoop/fs/s3a/yarn/TestS3A.java | 83 ++++++++++ .../fs/s3a/yarn/TestS3AMiniYarnCluster.java | 144 ++++++++++++++++++ .../src/test/resources/core-site.xml | 41 +++-- 19 files changed, 648 insertions(+), 29 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 264c53d6e9a..5132265952d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -41,6 +41,8 @@ Release 2.8.0 - UNRELEASED HADOOP-12657. Add a option to skip newline on empty files with getMerge -nl. (Kanaka Kumar Avvaru via aajisaka) + HADOOP-11262. Enable YARN to use S3A. (Pieter Reuse via lei) + IMPROVEMENTS HADOOP-12458. Retries is typoed to spell Retires in parts of diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 379fe5d56d6..c3cc693cb83 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -943,6 +943,12 @@ for ldap providers in the same way as above does. The implementation class of the S3A Filesystem + + fs.AbstractFileSystem.s3a.impl + org.apache.hadoop.fs.s3a.S3A + The implementation class of the S3A AbstractFileSystem. + + io.seqfile.compress.blocksize 1000000 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 12ec3751c3e..0c736a6b0ed 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -1166,7 +1166,7 @@ public abstract class FileContextMainOperationsBaseTest { fc.createSymlink(file, link, false); Assert.fail("Created a symlink on a file system that "+ "does not support symlinks."); - } catch (IOException e) { + } catch (UnsupportedOperationException e) { // Expected } createFile(file); diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 30126acee3c..fed54bf2043 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -128,6 +128,22 @@ junit test - + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.apache.hadoop + hadoop-mapreduce-examples + test + jar + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index fa81d935222..60d4b9b875b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -118,4 +118,6 @@ public class Constants { public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; public static final String FS_S3A = "s3a"; + + public static final int S3A_DEFAULT_PORT = -1; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java new file mode 100644 index 00000000000..d856d802d5c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * S3A implementation of AbstractFileSystem. + * This impl delegates to the S3AFileSystem + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class S3A extends DelegateToFileSystem{ + + public S3A(URI theUri, Configuration conf) + throws IOException, URISyntaxException { + super(theUri, new S3AFileSystem(), conf, "s3a", false); + } + + @Override + public int getUriDefaultPort() { + return Constants.S3A_DEFAULT_PORT; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java index 85e0ef77d52..47caea8390d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.fs.s3a; import org.apache.hadoop.fs.FileStatus; @@ -40,7 +39,12 @@ public class S3AFileStatus extends FileStatus { public boolean isEmptyDirectory() { return isEmptyDirectory; } - + + @Override + public String getOwner() { + return System.getProperty("user.name"); + } + /** Compare if this object is equal to another object * @param o the object to be compared. * @return true if two file status has the same path name; false if not. @@ -60,4 +64,23 @@ public class S3AFileStatus extends FileStatus { public int hashCode() { return super.hashCode(); } + + /** Get the modification time of the file/directory. + * + * s3a uses objects as "fake" directories, which are not updated to + * reflect the accurate modification time. We choose to report the + * current time because some parts of the ecosystem (e.g. the + * HistoryServer) use modification time to ignore "old" directories. + * + * @return for files the modification time in milliseconds since January 1, + * 1970 UTC or for directories the current time. + */ + @Override + public long getModificationTime(){ + if(isDirectory()){ + return System.currentTimeMillis(); + } else { + return super.getModificationTime(); + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 83be18423fe..6ede9f2d623 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -323,15 +323,24 @@ public class S3AFileSystem extends FileSystem { * * @return "s3a" */ + @Override public String getScheme() { return "s3a"; } - /** Returns a URI whose scheme and authority identify this FileSystem.*/ + /** + * Returns a URI whose scheme and authority identify this FileSystem. + */ + @Override public URI getUri() { return uri; } + @Override + public int getDefaultPort() { + return Constants.S3A_DEFAULT_PORT; + } + /** * Returns the S3 client used by this filesystem. * @return AmazonS3Client @@ -397,8 +406,10 @@ public class S3AFileSystem extends FileSystem { * @throws IOException * @see #setPermission(Path, FsPermission) */ - public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, - int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { String key = pathToKey(f); if (!overwrite && exists(f)) { diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 6df15e68cf4..b27c0505163 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -47,10 +47,14 @@ recursive file-by-file operations. They take time at least proportional to the number of files, during which time partial updates may be visible. If the operations are interrupted, the filesystem is left in an intermediate state. +## Warning #2: Because Object stores don't track modification times of directories, +features of Hadoop relying on this can have unexpected behaviour. E.g. the +AggregatedLogDeletionService of YARN will not remove the appropriate logfiles. + For further discussion on these topics, please consult [The Hadoop FileSystem API Definition](../../../hadoop-project-dist/hadoop-common/filesystem/index.html). -## Warning #2: your AWS credentials are valuable +## Warning #3: your AWS credentials are valuable Your AWS credentials not only pay for services, they offer read and write access to the data. Anyone with the credentials can not only read your datasets @@ -307,6 +311,12 @@ If you do any of these: change your credentials immediately! The implementation class of the S3A Filesystem + + fs.AbstractFileSystem.s3a.impl + org.apache.hadoop.fs.s3a.S3A + The implementation class of the S3A AbstractFileSystem. + + ### S3AFastOutputStream **Warning: NEW in hadoop 2.7. UNSTABLE, EXPERIMENTAL: use at own risk** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index d9726db7021..2308dd48cb0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.s3a; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; import org.junit.internal.AssumptionViolatedException; import java.io.IOException; @@ -51,4 +52,24 @@ public class S3ATestUtils { fs1.initialize(testURI, conf); return fs1; } + + public static FileContext createTestFileContext(Configuration conf) throws + IOException { + String fsname = conf.getTrimmed(TestS3AFileSystemContract.TEST_FS_S3A_NAME, ""); + + boolean liveTest = !StringUtils.isEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(Constants.FS_S3A); + } + if (!liveTest) { + // This doesn't work with our JUnit 3 style test cases, so instead we'll + // make this whole class not run by default + throw new AssumptionViolatedException( + "No test filesystem in " + TestS3AFileSystemContract.TEST_FS_S3A_NAME); + } + FileContext fc = FileContext.getFileContext(testURI,conf); + return fc; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java new file mode 100644 index 00000000000..34d78a5d265 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java @@ -0,0 +1,23 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import org.apache.hadoop.fs.TestFileContext; + +/** + * Implementation of TestFileContext for S3a + */ +public class TestS3AFileContext extends TestFileContext{ + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java new file mode 100644 index 00000000000..b0c4d8424f9 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java @@ -0,0 +1,35 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; + +/** + * Extends FileContextCreateMkdirBaseTest for a S3a FileContext + */ +public class TestS3AFileContextCreateMkdir + extends FileContextCreateMkdirBaseTest { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java new file mode 100644 index 00000000000..4d200d14e23 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java @@ -0,0 +1,60 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextMainOperationsBaseTest; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +/** + * S3A implementation of FileContextMainOperationsBaseTest + */ +public class TestS3AFileContextMainOperations + extends FileContextMainOperationsBaseTest { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + + @Override + protected boolean listCorruptedBlocksSupported() { + return false; + } + + @Test + @Ignore + public void testCreateFlagAppendExistingFile() throws IOException { + //append not supported, so test removed + } + + @Test + @Ignore + public void testCreateFlagCreateAppendExistingFile() throws IOException { + //append not supported, so test removed + } + + @Test + @Ignore + public void testSetVerifyChecksum() throws IOException { + //checksums ignored, so test removed + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java new file mode 100644 index 00000000000..a9f48483679 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java @@ -0,0 +1,60 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.net.URI; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FCStatisticsBaseTest; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; + +/** + * S3a implementation of FCStatisticsBaseTest + */ +public class TestS3AFileContextStatistics extends FCStatisticsBaseTest { + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true); + } + + @After + public void tearDown() throws Exception { + fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true); + } + + @Override + protected void verifyReadBytes(FileSystem.Statistics stats) { + // one blockSize for read, one for pread + Assert.assertEquals(2 * blockSize, stats.getBytesRead()); + } + + @Override + protected void verifyWrittenBytes(FileSystem.Statistics stats) { + //No extra bytes are written + Assert.assertEquals(blockSize, stats.getBytesWritten()); + } + + @Override + protected URI getFsUri() { + return fc.getHomeDirectory().toUri(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java new file mode 100644 index 00000000000..3da7b19a84a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java @@ -0,0 +1,43 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextURIBase; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +/** + * S3a implementation of FileContextURIBase + */ +public class TestS3AFileContextURI extends FileContextURIBase { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc1 = S3ATestUtils.createTestFileContext(conf); + fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS + super.setUp(); + } + + @Test + @Ignore + public void testFileStatus() throws IOException { + //test disabled (the statistics tested with this method are not relevant for an S3FS) + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java new file mode 100644 index 00000000000..666f4c292a9 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java @@ -0,0 +1,34 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextUtilBase; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; + +/** + * S3A implementation of FileContextUtilBase + */ +public class TestS3AFileContextUtil extends FileContextUtilBase { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java new file mode 100644 index 00000000000..c6a5a39b8ae --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Path; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.EnumSet; +import org.apache.hadoop.fs.s3a.S3ATestUtils; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestS3A { + private FileContext fc; + + @Rule + public final Timeout testTimeout = new Timeout(90000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + } + + @After + public void tearDown() throws Exception { + fc.delete(getTestPath(),true); + } + + protected Path getTestPath() { + return new Path("/tests3afc"); + } + + @Test + public void testS3AStatus() throws Exception { + FsStatus fsStatus = fc.getFsStatus(null); + assertNotNull(fsStatus); + assertTrue("Used capacity should be positive: " + fsStatus.getUsed(), + fsStatus.getUsed() >= 0); + assertTrue("Remaining capacity should be positive: " + fsStatus + .getRemaining(), + fsStatus.getRemaining() >= 0); + assertTrue("Capacity should be positive: " + fsStatus.getCapacity(), + fsStatus.getCapacity() >= 0); + } + + @Test + public void testS3ACreateFileInSubDir() throws Exception { + Path dirPath = getTestPath(); + fc.mkdir(dirPath,FileContext.DIR_DEFAULT_PERM,true); + Path filePath = new Path(dirPath, "file"); + try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag + .CREATE))) { + file.write(666); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java new file mode 100644 index 00000000000..990d79f2054 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.fs.s3a.yarn; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.examples.WordCount; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; + +import org.junit.After; +import static org.junit.Assert.assertTrue; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +public class TestS3AMiniYarnCluster { + + private final Configuration conf = new YarnConfiguration(); + private S3AFileSystem fs; + private MiniYARNCluster yarnCluster; + private final String rootPath = "/tests/MiniClusterWordCount/"; + + @Before + public void beforeTest() throws IOException { + fs = S3ATestUtils.createTestFileSystem(conf); + fs.mkdirs(new Path(rootPath + "input/")); + + yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName + 1, // number of node managers + 1, // number of local log dirs per node manager + 1); // number of hdfs dirs per node manager + yarnCluster.init(conf); + yarnCluster.start(); + } + + @After + public void afterTest() throws IOException { + fs.delete(new Path(rootPath), true); + yarnCluster.stop(); + } + + @Test + public void testWithMiniCluster() throws Exception { + Path input = new Path(rootPath + "input/in.txt"); + input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + Path output = new Path(rootPath + "output/"); + output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + + writeStringToFile(input, "first line\nsecond line\nthird line"); + + Job job = Job.getInstance(conf, "word count"); + job.setJarByClass(WordCount.class); + job.setMapperClass(WordCount.TokenizerMapper.class); + job.setCombinerClass(WordCount.IntSumReducer.class); + job.setReducerClass(WordCount.IntSumReducer.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + FileInputFormat.addInputPath(job, input); + FileOutputFormat.setOutputPath(job, output); + + int exitCode = (job.waitForCompletion(true) ? 0 : 1); + assertEquals("Returned error code.", 0, exitCode); + + assertTrue(fs.exists(new Path(output, "_SUCCESS"))); + String outputAsStr = readStringFromFile(new Path(output, "part-r-00000")); + Map resAsMap = getResultAsMap(outputAsStr); + + assertEquals(4, resAsMap.size()); + assertEquals(1, (int) resAsMap.get("first")); + assertEquals(1, (int) resAsMap.get("second")); + assertEquals(1, (int) resAsMap.get("third")); + assertEquals(3, (int) resAsMap.get("line")); + } + + /** + * helper method + */ + private Map getResultAsMap(String outputAsStr) throws IOException { + Map result = new HashMap<>(); + for (String line : outputAsStr.split("\n")) { + String[] tokens = line.split("\t"); + result.put(tokens[0], Integer.parseInt(tokens[1])); + } + return result; + } + + /** + * helper method + */ + private void writeStringToFile(Path path, String string) throws IOException { + FileContext fc = S3ATestUtils.createTestFileContext(conf); + try (FSDataOutputStream file = fc.create(path, + EnumSet.of(CreateFlag.CREATE))) { + file.write(string.getBytes()); + } + } + + /** + * helper method + */ + private String readStringFromFile(Path path) { + try (FSDataInputStream in = fs.open(path)) { + long bytesLen = fs.getFileStatus(path).getLen(); + byte[] buffer = new byte[(int) bytesLen]; + IOUtils.readFully(in, buffer, 0, buffer.length); + return new String(buffer); + } catch (IOException e) { + throw new RuntimeException("Failed to read from [" + path + "]", e); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 3397769d3ac..7d2046b588d 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -23,29 +23,28 @@ - - hadoop.tmp.dir - target/build/test - A base for other temporary directories. - true - + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + - - - hadoop.security.authentication - simple - - - - + + + hadoop.security.authentication + simple + + + + +