Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13599. s3a close() to be non-synchronized, so avoid risk of deadlock on shutdown. Contributed by Steve Loughran.

Browse Source 
(cherry picked from commit 47f80922dc7cb2fa6d084e6fb1f354c4ec1d4c69)
(cherry picked from commit 1f1e47e41198760b0ddddceb489f2de2b569f38a)
This commit is contained in:
Chris Nauroth 2016-09-28 15:53:17 -07:00
parent 7467628d43
commit b7b06a5f34
2 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -127,6 +127,7 @@ public class S3AFileSystem extends FileSystem {
private S3AInputPolicy inputPolicy;
private static final AtomicBoolean warnedOfCoreThreadDeprecation =
new AtomicBoolean(false);
private final AtomicBoolean closed = new AtomicBoolean(false);
// The maximum number of entries that can be deleted in any call to s3
private static final int MAX_ENTRIES_TO_DELETE = 1000;
@ -1514,7 +1515,11 @@ private void innerCopyFromLocalFile(boolean delSrc, boolean overwrite,
* @throws IOException IO problem
*/
@Override
public synchronized void close() throws IOException {
public void close() throws IOException {
if (closed.getAndSet(true)) {
// already closed
return;
}
try {
super.close();
} finally {

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
View File

@ -401,6 +401,14 @@ public void testCustomUserAgent() throws Exception {
awsConf.getUserAgent());
}
@Test
public void testCloseIdempotent() throws Throwable {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
fs.close();
fs.close();
}
/**
* Reads and returns a field from an object using reflection. If the field
* cannot be found, is null, or is not the expected type, then this method