From fe0009a2bd17e26f9e9364ec3f772e9a3f138de8 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 8 Mar 2016 10:43:17 +0900 Subject: [PATCH] HDFS-9812. Streamer threads leak if failure happens when closing DFSOutputStream. Contributed by Lin Yiqun. (cherry picked from commit 352d299cf8ebe330d24117df98d1e6a64ae38c26) --- .../java/org/apache/hadoop/hdfs/DFSOutputStream.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 58787560cca..6d29ec872db 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -774,14 +774,19 @@ public class DFSOutputStream extends FSOutputSummer flushInternal(); // flush all data to Datanodes // get last block before destroying the streamer ExtendedBlock lastBlock = getStreamer().getBlock(); - closeThreads(false); + try (TraceScope ignored = dfsClient.getTracer().newScope("completeFile")) { completeFile(lastBlock); } } catch (ClosedChannelException ignored) { } finally { - setClosed(); + // Failures may happen when flushing data. + // Streamers may keep waiting for the new block information. + // Thus need to force closing these threads. + // Don't need to call setClosed() because closeThreads(true) + // calls setClosed() in the finally block. + closeThreads(true); } }