From a359dc87d431573b2783dbe4af25e00c836096aa Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Tue, 19 Jan 2016 13:57:08 -0800 Subject: [PATCH] HADOOP-12696. Add tests for S3FileSystem Contract. Contributed by Matt Paduano (cherry picked from commit 1acc509b45d58c0eb7e83ea1ba13169410be0dbe) (cherry picked from commit 2cbb8bbd728aab6d438c3051c970e4b4475b8e68) --- .../hadoop-common/CHANGES.txt | 2 + .../src/site/markdown/filesystem/testing.md | 4 +- .../fs/contract/AbstractContractSeekTest.java | 8 +- .../hadoop/fs/contract/ContractOptions.java | 6 + .../org/apache/hadoop/fs/s3/S3FileSystem.java | 58 ++++++---- .../apache/hadoop/fs/s3/S3InputStream.java | 17 ++- .../hadoop/fs/contract/s3/S3Contract.java | 40 +++++++ .../fs/contract/s3/TestS3ContractCreate.java | 32 ++++++ .../fs/contract/s3/TestS3ContractDelete.java | 31 ++++++ .../fs/contract/s3/TestS3ContractMkdir.java | 32 ++++++ .../fs/contract/s3/TestS3ContractOpen.java | 32 ++++++ .../fs/contract/s3/TestS3ContractRename.java | 32 ++++++ .../fs/contract/s3/TestS3ContractRootDir.java | 34 ++++++ .../fs/contract/s3/TestS3ContractSeek.java | 32 ++++++ .../src/test/resources/contract/s3.xml | 104 ++++++++++++++++++ 15 files changed, 430 insertions(+), 34 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java create mode 100644 hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 0c2cf417ca8..7406ee7e7e7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -380,6 +380,8 @@ Release 2.8.0 - UNRELEASED HADOOP-12604. Exception may be swallowed in KMSClientProvider. (Yongjun Zhang) + HADOOP-12696. Add tests for S3Filesystem Contract (Matt Paduano via raviprak) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md index 444fb609850..99561cd1a6d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md @@ -190,7 +190,7 @@ tests against remote FileSystems that require login details require usernames/ID All these details MUST be required to be placed in the file `src/test/resources/contract-test-options.xml`, and your SCM tools configured to never commit this file to subversion, git or equivalent. Furthermore, the build MUST be configured to never bundle this file in any `-test` artifacts generated. The Hadoop build does this, excluding `src/test/**/*.xml` from the JAR files. - +In addition, `src/test/resources/auth-keys.xml` will need to be created. It can be a copy of `contract-test-options.xml`. The `AbstractFSContract` class automatically loads this resource file if present; specific keys for specific test cases can be added. As an example, here are what S3N test keys look like: @@ -214,7 +214,7 @@ As an example, here are what S3N test keys look like: The `AbstractBondedFSContract` automatically skips a test suite if the FileSystem URL is not defined in the property `fs.contract.test.fs.%s`, where `%s` matches the schema name of the FileSystem. - +When running the tests `maven.test.skip` will need to be turned off since it is true by default on these tests. This can be done with a command like `mvn test -Ptests-on`. ### Important: passing the tests does not guarantee compatibility diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java index 4a0560ec256..8f5651021b2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java @@ -116,15 +116,13 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas */ @Test public void testSeekReadClosedFile() throws Throwable { - boolean supportsSeekOnClosedFiles = isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE); - instream = getFileSystem().open(smallSeekFile); getLog().debug( "Stream is of type " + instream.getClass().getCanonicalName()); instream.close(); try { instream.seek(0); - if (!supportsSeekOnClosedFiles) { + if (!isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE)) { fail("seek succeeded on a closed stream"); } } catch (IOException e) { @@ -132,7 +130,9 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas } try { int data = instream.available(); - fail("read() succeeded on a closed stream, got " + data); + if (!isSupported(SUPPORTS_AVAILABLE_ON_CLOSED_FILE)) { + fail("available() succeeded on a closed stream, got " + data); + } } catch (IOException e) { //expected a closed file } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java index 56caced7b75..d8c259265fb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java @@ -147,6 +147,12 @@ public interface ContractOptions { */ String SUPPORTS_SEEK_ON_CLOSED_FILE = "supports-seek-on-closed-file"; + /** + * Is available() on a closed InputStream supported? + * @{value} + */ + String SUPPORTS_AVAILABLE_ON_CLOSED_FILE = "supports-available-on-closed-file"; + /** * Flag to indicate that this FS expects to throw the strictest * exceptions it can, not generic IOEs, which, if returned, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java index 5f0ad83a5da..10f67fdbf55 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3native.NativeS3FileSystem; @@ -62,7 +63,7 @@ public class S3FileSystem extends FileSystem { public S3FileSystem() { // set store in initialize() } - + public S3FileSystem(FileSystemStore store) { this.store = store; } @@ -90,14 +91,14 @@ public class S3FileSystem extends FileSystem { } store.initialize(uri, conf); setConf(conf); - this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); + this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this); - } + } private static FileSystemStore createDefaultStore(Configuration conf) { FileSystemStore store = new Jets3tFileSystemStore(); - + RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); @@ -105,13 +106,13 @@ public class S3FileSystem extends FileSystem { new HashMap, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); - + RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map methodNameToPolicyMap = new HashMap(); methodNameToPolicyMap.put("storeBlock", methodPolicy); methodNameToPolicyMap.put("retrieveBlock", methodPolicy); - + return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap); } @@ -144,21 +145,29 @@ public class S3FileSystem extends FileSystem { paths.add(0, absolutePath); absolutePath = absolutePath.getParent(); } while (absolutePath != null); - + boolean result = true; - for (Path p : paths) { - result &= mkdir(p); + for (int i = 0; i < paths.size(); i++) { + Path p = paths.get(i); + try { + result &= mkdir(p); + } catch(FileAlreadyExistsException e) { + if (i + 1 < paths.size()) { + throw new ParentNotDirectoryException(e.getMessage()); + } + throw e; + } } return result; } - + private boolean mkdir(Path path) throws IOException { Path absolutePath = makeAbsolute(path); INode inode = store.retrieveINode(absolutePath); if (inode == null) { store.storeINode(absolutePath, INode.DIRECTORY_INODE); } else if (inode.isFile()) { - throw new IOException(String.format( + throw new FileAlreadyExistsException(String.format( "Can't make directory for path %s since it is a file.", absolutePath)); } @@ -176,11 +185,12 @@ public class S3FileSystem extends FileSystem { private INode checkFile(Path path) throws IOException { INode inode = store.retrieveINode(makeAbsolute(path)); + String message = String.format("No such file: '%s'", path.toString()); if (inode == null) { - throw new IOException("No such file."); + throw new FileNotFoundException(message + " does not exist"); } if (inode.isDirectory()) { - throw new IOException("Path " + path + " is a directory."); + throw new FileNotFoundException(message + " is a directory"); } return inode; } @@ -222,10 +232,14 @@ public class S3FileSystem extends FileSystem { INode inode = store.retrieveINode(makeAbsolute(file)); if (inode != null) { - if (overwrite) { + if (overwrite && !inode.isDirectory()) { delete(file, true); } else { - throw new FileAlreadyExistsException("File already exists: " + file); + String message = String.format("File already exists: '%s'", file); + if (inode.isDirectory()) { + message = message + " is a directory"; + } + throw new FileAlreadyExistsException(message); } } else { Path parent = file.getParent(); @@ -233,7 +247,7 @@ public class S3FileSystem extends FileSystem { if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } - } + } } return new FSDataOutputStream (new S3OutputStream(getConf(), store, makeAbsolute(file), @@ -254,7 +268,7 @@ public class S3FileSystem extends FileSystem { INode srcINode = store.retrieveINode(absoluteSrc); if (srcINode == null) { // src path doesn't exist - return false; + return false; } Path absoluteDst = makeAbsolute(dst); INode dstINode = store.retrieveINode(absoluteDst); @@ -311,7 +325,7 @@ public class S3FileSystem extends FileSystem { store.deleteBlock(block); } } else { - FileStatus[] contents = null; + FileStatus[] contents = null; try { contents = listStatus(absolutePath); } catch(FileNotFoundException fnfe) { @@ -319,7 +333,7 @@ public class S3FileSystem extends FileSystem { } if ((contents.length !=0) && (!recursive)) { - throw new IOException("Directory " + path.toString() + throw new IOException("Directory " + path.toString() + " is not empty."); } for (FileStatus p:contents) { @@ -331,9 +345,9 @@ public class S3FileSystem extends FileSystem { } return true; } - + /** - * FileStatus for S3 file systems. + * FileStatus for S3 file systems. */ @Override public FileStatus getFileStatus(Path f) throws IOException { @@ -343,7 +357,7 @@ public class S3FileSystem extends FileSystem { } return new S3FileStatus(f.makeQualified(this), inode); } - + @Override public long getDefaultBlockSize() { return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java index 5af57e66a58..6f39f01c8ee 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java @@ -22,6 +22,7 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.EOFException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,14 +47,14 @@ class S3InputStream extends FSInputStream { private long pos = 0; private File blockFile; - + private DataInputStream blockStream; private long blockEnd = -1; - + private FileSystem.Statistics stats; - - private static final Log LOG = + + private static final Log LOG = LogFactory.getLog(S3InputStream.class.getName()); @@ -65,7 +66,7 @@ class S3InputStream extends FSInputStream { public S3InputStream(Configuration conf, FileSystemStore store, INode inode, FileSystem.Statistics stats) { - + this.store = store; this.stats = stats; this.blocks = inode.getBlocks(); @@ -86,8 +87,12 @@ class S3InputStream extends FSInputStream { @Override public synchronized void seek(long targetPos) throws IOException { + String message = String.format("Cannot seek to %d", targetPos); if (targetPos > fileLength) { - throw new IOException("Cannot seek after EOF"); + throw new EOFException(message + ": after EOF"); + } + if (targetPos < 0) { + throw new EOFException(message + ": negative"); } pos = targetPos; blockEnd = -1; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java new file mode 100644 index 00000000000..b388ce77afc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; + +public class S3Contract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/s3.xml"; + + + public S3Contract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + @Override + public String getScheme() { + return "s3"; + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java new file mode 100644 index 00000000000..b8c758cecec --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestS3ContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java new file mode 100644 index 00000000000..2d3cec70005 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3ContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java new file mode 100644 index 00000000000..992ce53a9db --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestS3ContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java new file mode 100644 index 00000000000..2a4ba31b938 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestS3ContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java new file mode 100644 index 00000000000..68bdbdadf96 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3ContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java new file mode 100644 index 00000000000..b96808129fe --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * root dir operations against an S3 bucket + */ +public class TestS3ContractRootDir extends AbstractContractRootDirectoryTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java new file mode 100644 index 00000000000..bfcd1636ffd --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestS3ContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml new file mode 100644 index 00000000000..4b742c10529 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml @@ -0,0 +1,104 @@ + + + + + + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-blobstore + true + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.rename-returns-false-if-source-missing + true + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + false + + + + fs.contract.supports-atomic-rename + false + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + fs.contract.supports-available-on-closed-file + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + +