diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 2376c051c99..d472591409f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -3391,9 +3392,22 @@ public abstract class FileSystem extends Configured
Tracer tracer = FsTracer.get(conf);
try(TraceScope scope = tracer.newScope("FileSystem#createFileSystem")) {
scope.addKVAnnotation("scheme", uri.getScheme());
- Class> clazz = getFileSystemClass(uri.getScheme(), conf);
- FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
- fs.initialize(uri, conf);
+ Class extends FileSystem> clazz =
+ getFileSystemClass(uri.getScheme(), conf);
+ FileSystem fs = ReflectionUtils.newInstance(clazz, conf);
+ try {
+ fs.initialize(uri, conf);
+ } catch (IOException | RuntimeException e) {
+ // exception raised during initialization.
+ // log summary at warn and full stack at debug
+ LOGGER.warn("Failed to initialize fileystem {}: {}",
+ uri, e.toString());
+ LOGGER.debug("Failed to initialize fileystem", e);
+ // then (robustly) close the FS, so as to invoke any
+ // cleanup code.
+ IOUtils.cleanupWithLogger(LOGGER, fs);
+ throw e;
+ }
return fs;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index 4d627a5e8e2..10ad8a14487 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -18,14 +18,24 @@
package org.apache.hadoop.fs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URI;
import java.net.URL;
import java.util.ServiceConfigurationError;
import org.junit.Test;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.*;
+/**
+ * Tests related to filesystem creation and lifecycle.
+ */
public class TestFileSystemInitialization {
/**
@@ -55,4 +65,119 @@ public class TestFileSystemInitialization {
} catch (Exception | ServiceConfigurationError expected) {
}
}
+
+ @Test
+ public void testNewInstanceFailure() throws Throwable {
+ intercept(IOException.class, FailingFileSystem.INITIALIZE, () ->
+ FileSystem.newInstance(new URI("failing://localhost"), FailingFileSystem
+ .failingConf()));
+ assertThat(FailingFileSystem.initCount).describedAs("init count")
+ .isEqualTo(1);
+ assertThat(FailingFileSystem.closeCount).describedAs("close count")
+ .isEqualTo(1);
+ }
+
+ /**
+ * An FS which will fail on both init and close, and update
+ * counters of invocations as it does so.
+ */
+ public static class FailingFileSystem extends FileSystem {
+
+ public static final String INITIALIZE = "initialize()";
+
+ public static final String CLOSE = "close()";
+
+ private static int initCount;
+
+ private static int closeCount;
+
+ private static Configuration failingConf() {
+ final Configuration conf = new Configuration(false);
+ conf.setClass("fs.failing.impl", FailingFileSystem.class,
+ FileSystem.class);
+ return conf;
+ }
+
+ @Override
+ public void initialize(final URI name, final Configuration conf)
+ throws IOException {
+ super.initialize(name, conf);
+ initCount++;
+ throw new IOException(INITIALIZE);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeCount++;
+ throw new IOException(CLOSE);
+ }
+
+ @Override
+ public URI getUri() {
+ return null;
+ }
+
+ @Override
+ public FSDataInputStream open(final Path f, final int bufferSize)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public FSDataOutputStream create(final Path f,
+ final FsPermission permission,
+ final boolean overwrite,
+ final int bufferSize,
+ final short replication,
+ final long blockSize,
+ final Progressable progress) throws IOException {
+ return null;
+ }
+
+ @Override
+ public FSDataOutputStream append(final Path f,
+ final int bufferSize,
+ final Progressable progress) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean rename(final Path src, final Path dst) throws IOException {
+ return false;
+ }
+
+ @Override
+ public boolean delete(final Path f, final boolean recursive)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStatus[] listStatus(final Path f)
+ throws FileNotFoundException, IOException {
+ return new FileStatus[0];
+ }
+
+ @Override
+ public void setWorkingDirectory(final Path new_dir) {
+
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return null;
+ }
+
+ @Override
+ public boolean mkdirs(final Path f, final FsPermission permission)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStatus getFileStatus(final Path f) throws IOException {
+ return null;
+ }
+ }
+
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AbstractHadoopTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AbstractHadoopTestBase.java
new file mode 100644
index 00000000000..e18119ccafc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AbstractHadoopTestBase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.test;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+/**
+ * A base class for JUnit5+ tests that sets a default timeout for all tests
+ * that subclass this test.
+ *
+ * Threads are named to the method being executed, for ease of diagnostics
+ * in logs and thread dumps.
+ *
+ * Unlike {@link HadoopTestBase} this class does not extend JUnit Assert
+ * so is easier to use with AssertJ.
+ */
+public abstract class AbstractHadoopTestBase {
+
+ /**
+ * System property name to set the test timeout: {@value}.
+ */
+ public static final String PROPERTY_TEST_DEFAULT_TIMEOUT =
+ "test.default.timeout";
+
+ /**
+ * The default timeout (in milliseconds) if the system property
+ * {@link #PROPERTY_TEST_DEFAULT_TIMEOUT}
+ * is not set: {@value}.
+ */
+ public static final int TEST_DEFAULT_TIMEOUT_VALUE = 100000;
+
+ /**
+ * The JUnit rule that sets the default timeout for tests.
+ */
+ @Rule
+ public Timeout defaultTimeout = retrieveTestTimeout();
+
+ /**
+ * Retrieve the test timeout from the system property
+ * {@link #PROPERTY_TEST_DEFAULT_TIMEOUT}, falling back to
+ * the value in {@link #TEST_DEFAULT_TIMEOUT_VALUE} if the
+ * property is not defined.
+ * @return the recommended timeout for tests
+ */
+ public static Timeout retrieveTestTimeout() {
+ String propval = System.getProperty(PROPERTY_TEST_DEFAULT_TIMEOUT,
+ Integer.toString(
+ TEST_DEFAULT_TIMEOUT_VALUE));
+ int millis;
+ try {
+ millis = Integer.parseInt(propval);
+ } catch (NumberFormatException e) {
+ //fall back to the default value, as the property cannot be parsed
+ millis = TEST_DEFAULT_TIMEOUT_VALUE;
+ }
+ return new Timeout(millis, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * The method name.
+ */
+ @Rule
+ public TestName methodName = new TestName();
+
+ /**
+ * Get the method name; defaults to the value of {@link #methodName}.
+ * Subclasses may wish to override it, which will tune the thread naming.
+ * @return the name of the method.
+ */
+ protected String getMethodName() {
+ return methodName.getMethodName();
+ }
+
+ /**
+ * Static initializer names this thread "JUnit".
+ */
+ @BeforeClass
+ public static void nameTestThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ /**
+ * Before each method, the thread is renamed to match the method name.
+ */
+ @Before
+ public void nameThreadToMethod() {
+ Thread.currentThread().setName("JUnit-" + getMethodName());
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
index cb7df4b011a..23f3531a41e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.test;
+import java.util.concurrent.TimeUnit;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -70,7 +72,7 @@ public abstract class HadoopTestBase extends Assert {
//fall back to the default value, as the property cannot be parsed
millis = TEST_DEFAULT_TIMEOUT_VALUE;
}
- return new Timeout(millis);
+ return new Timeout(millis, TimeUnit.MILLISECONDS);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index a60f9af2a93..66cac99de7b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.util.Progressable;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Upload files/parts directly via different buffering mechanisms:
@@ -396,9 +397,9 @@ class S3ABlockOutputStream extends OutputStream implements
writeOperationHelper.writeFailed(ioe);
throw ioe;
} finally {
- closeAll(LOG, block, blockFactory);
+ cleanupWithLogger(LOG, block, blockFactory);
LOG.debug("Statistics: {}", statistics);
- closeAll(LOG, statistics);
+ cleanupWithLogger(LOG, statistics);
clearActiveBlock();
}
// Note end of write. This does not change the state of the remote FS.
@@ -437,7 +438,7 @@ class S3ABlockOutputStream extends OutputStream implements
// stream afterwards.
return writeOperationHelper.putObject(putObjectRequest);
} finally {
- closeAll(LOG, uploadData, block);
+ cleanupWithLogger(LOG, uploadData, block);
}
});
clearActiveBlock();
@@ -614,7 +615,7 @@ class S3ABlockOutputStream extends OutputStream implements
return partETag;
} finally {
// close the stream and block
- closeAll(LOG, uploadData, block);
+ cleanupWithLogger(LOG, uploadData, block);
}
});
partETagsFutures.add(partETagFuture);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 0e3bca57b25..156defb7ca0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.util.DirectBufferPool;
import static org.apache.hadoop.fs.s3a.S3ADataBlocks.DataBlock.DestState.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.closeAll;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Set of classes to support output streaming into blocks which are then
@@ -155,7 +155,7 @@ final class S3ADataBlocks {
*/
@Override
public void close() throws IOException {
- closeAll(LOG, uploadStream);
+ cleanupWithLogger(LOG, uploadStream);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f0ddf447b3e..832e0662c81 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -306,22 +306,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
throws IOException {
// get the host; this is guaranteed to be non-null, non-empty
bucket = name.getHost();
- LOG.debug("Initializing S3AFileSystem for {}", bucket);
- // clone the configuration into one with propagated bucket options
- Configuration conf = propagateBucketOptions(originalConf, bucket);
- // patch the Hadoop security providers
- patchSecurityCredentialProviders(conf);
- // look for delegation token support early.
- boolean delegationTokensEnabled = hasDelegationTokenBinding(conf);
- if (delegationTokensEnabled) {
- LOG.debug("Using delegation tokens");
- }
- // set the URI, this will do any fixup of the URI to remove secrets,
- // canonicalize.
- setUri(name, delegationTokensEnabled);
- super.initialize(uri, conf);
- setConf(conf);
try {
+ LOG.debug("Initializing S3AFileSystem for {}", bucket);
+ // clone the configuration into one with propagated bucket options
+ Configuration conf = propagateBucketOptions(originalConf, bucket);
+ // patch the Hadoop security providers
+ patchSecurityCredentialProviders(conf);
+ // look for delegation token support early.
+ boolean delegationTokensEnabled = hasDelegationTokenBinding(conf);
+ if (delegationTokensEnabled) {
+ LOG.debug("Using delegation tokens");
+ }
+ // set the URI, this will do any fixup of the URI to remove secrets,
+ // canonicalize.
+ setUri(name, delegationTokensEnabled);
+ super.initialize(uri, conf);
+ setConf(conf);
// look for encryption data
// DT Bindings may override this
@@ -381,6 +381,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
initCannedAcls(conf);
+ // This initiates a probe against S3 for the bucket existing.
+ // It is where all network and authentication configuration issues
+ // surface, and is potentially slow.
verifyBucketExists();
inputPolicy = S3AInputPolicy.getPolicy(
@@ -436,7 +439,13 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
initMultipartUploads(conf);
} catch (AmazonClientException e) {
+ // amazon client exception: stop all services then throw the translation
+ stopAllServices();
throw translateException("initializing ", new Path(name), e);
+ } catch (IOException | RuntimeException e) {
+ // other exceptions: stop the services.
+ stopAllServices();
+ throw e;
}
}
@@ -3118,27 +3127,43 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
try {
super.close();
} finally {
- if (transfers != null) {
- transfers.shutdownNow(true);
- transfers = null;
- }
- HadoopExecutors.shutdown(boundedThreadPool, LOG,
- THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
- boundedThreadPool = null;
- HadoopExecutors.shutdown(unboundedThreadPool, LOG,
- THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
- unboundedThreadPool = null;
- S3AUtils.closeAll(LOG, metadataStore, instrumentation);
- metadataStore = null;
- instrumentation = null;
- closeAutocloseables(LOG, credentials);
- cleanupWithLogger(LOG, delegationTokens.orElse(null));
- cleanupWithLogger(LOG, signerManager);
- signerManager = null;
- credentials = null;
+ stopAllServices();
}
}
+ /**
+ * Stop all services.
+ * This is invoked in close() and during failures of initialize()
+ * -make sure that all operations here are robust to failures in
+ * both the expected state of this FS and of failures while being stopped.
+ */
+ protected synchronized void stopAllServices() {
+ if (transfers != null) {
+ try {
+ transfers.shutdownNow(true);
+ } catch (RuntimeException e) {
+ // catch and swallow for resilience.
+ LOG.debug("When shutting down", e);
+ }
+ transfers = null;
+ }
+ HadoopExecutors.shutdown(boundedThreadPool, LOG,
+ THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
+ boundedThreadPool = null;
+ HadoopExecutors.shutdown(unboundedThreadPool, LOG,
+ THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
+ unboundedThreadPool = null;
+ closeAutocloseables(LOG, credentials);
+ cleanupWithLogger(LOG,
+ metadataStore,
+ instrumentation,
+ delegationTokens.orElse(null),
+ signerManager);
+ delegationTokens = Optional.empty();
+ signerManager = null;
+ credentials = null;
+ }
+
/**
* Verify that the input stream is open. Non blocking; this gives
* the last state of the volatile {@link #closed} field.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 7e3c5e6b925..02508810339 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -86,6 +86,7 @@ import java.util.concurrent.ExecutionException;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Utility methods for S3A code.
@@ -1613,26 +1614,17 @@ public final class S3AUtils {
/**
* Close the Closeable objects and ignore any Exception or
* null pointers.
- * (This is the SLF4J equivalent of that in {@code IOUtils}).
+ * This is obsolete: use
+ * {@link org.apache.hadoop.io.IOUtils#cleanupWithLogger(Logger, Closeable...)}
* @param log the log to log at debug level. Can be null.
* @param closeables the objects to close
*/
+ @Deprecated
public static void closeAll(Logger log,
Closeable... closeables) {
- if (log == null) {
- log = LOG;
- }
- for (Closeable c : closeables) {
- if (c != null) {
- try {
- log.debug("Closing {}", c);
- c.close();
- } catch (Exception e) {
- log.debug("Exception in closing {}", c, e);
- }
- }
- }
+ cleanupWithLogger(log, closeables);
}
+
/**
* Close the Closeable objects and ignore any Exception or
* null pointers.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
index 8686e5413fe..e3026929811 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
import org.apache.hadoop.util.Progressable;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Relays FS calls to the mocked FS, allows for some extra logging with
@@ -148,6 +149,11 @@ public class MockS3AFileSystem extends S3AFileSystem {
writeHelper = new WriteOperationHelper(this, conf);
}
+ @Override
+ public void close() {
+ cleanupWithLogger(LOG, instrumentation);
+ }
+
@Override
public WriteOperationHelper getWriteOperationHelper() {
return writeHelper;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index f155aa03fbd..82589fa6aa0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.MultipartUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestConstants;
-import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.fs.s3a.commit.CommitOperations;
@@ -63,6 +62,7 @@ import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.forbidden;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.newAssumedRoleConfig;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.LambdaTestUtils.*;
@@ -106,7 +106,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
@Override
public void teardown() throws Exception {
- S3AUtils.closeAll(LOG, roleFS);
+ cleanupWithLogger(LOG, roleFS);
super.teardown();
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
index 6b55b1b4c32..853810602ba 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
@@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.commit.ITestCommitOperations;
import static org.apache.hadoop.fs.s3a.Constants.ASSUMED_ROLE_ARN;
@@ -34,6 +33,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.*;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.*;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Verify that the commit operations work with a restricted set of operations.
@@ -84,7 +84,7 @@ public class ITestAssumedRoleCommitOperations extends ITestCommitOperations {
@Override
public void teardown() throws Exception {
- S3AUtils.closeAll(LOG, roleFS);
+ cleanupWithLogger(LOG, roleFS);
// switches getFileSystem() back to the full FS.
roleFS = null;
super.teardown();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
index 0b032196adf..050bd6e89f1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore;
import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
@@ -71,6 +70,7 @@ import static org.apache.hadoop.fs.s3a.auth.RoleModel.statement;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.bindRolePolicyStatements;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.newAssumedRoleConfig;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.failif;
@@ -255,7 +255,7 @@ public class ITestRestrictedReadAccess extends AbstractS3ATestBase {
try {
super.teardown();
} finally {
- S3AUtils.closeAll(LOG, readonlyFS);
+ cleanupWithLogger(LOG, readonlyFS);
}
}