Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13599. s3a close() to be non-synchronized, so avoid risk of deadlock on shutdown. Contributed by Steve Loughran. 

(cherry picked from commit 47f80922dc)
(cherry picked from commit 1f1e47e411)
This commit is contained in:
Chris Nauroth 2016-09-28 15:53:17 -07:00
parent 7467628d43
commit b7b06a5f34
2 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -127,6 +127,7 @@ public class S3AFileSystem extends FileSystem {
private S3AInputPolicy inputPolicy; private S3AInputPolicy inputPolicy;
private static final AtomicBoolean warnedOfCoreThreadDeprecation = private static final AtomicBoolean warnedOfCoreThreadDeprecation =
new AtomicBoolean(false); new AtomicBoolean(false);
private final AtomicBoolean closed = new AtomicBoolean(false);
// The maximum number of entries that can be deleted in any call to s3 // The maximum number of entries that can be deleted in any call to s3
private static final int MAX_ENTRIES_TO_DELETE = 1000; private static final int MAX_ENTRIES_TO_DELETE = 1000;
@ -1514,7 +1515,11 @@ public class S3AFileSystem extends FileSystem {
* @throws IOException IO problem * @throws IOException IO problem
*/ */
@Override @Override
public synchronized void close() throws IOException { public void close() throws IOException {
if (closed.getAndSet(true)) {
// already closed
return;
}
try { try {
super.close(); super.close();
} finally { } finally {

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
View File

@ -401,6 +401,14 @@ public class TestS3AConfiguration {
awsConf.getUserAgent()); awsConf.getUserAgent());
} }
@Test
public void testCloseIdempotent() throws Throwable {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
fs.close();
fs.close();
}
/** /**
* Reads and returns a field from an object using reflection. If the field * Reads and returns a field from an object using reflection. If the field
* cannot be found, is null, or is not the expected type, then this method * cannot be found, is null, or is not the expected type, then this method