From 7f6d150c9a50352ee810ed67d9e61e4ec99d856d Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 2 Feb 2016 19:30:45 +0000 Subject: [PATCH] MAPREDUCE-6621. Memory Leak in JobClient#submitJobInternal(). Contributed by Xuan Gong (cherry picked from commit 43e669b22d9ac0b81ab8cc397c5d6d92bac986c6) --- hadoop-mapreduce-project/CHANGES.txt | 6 ++++++ .../src/main/java/org/apache/hadoop/mapred/JobClient.java | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5a99ff07368..927d4d7f5a3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -37,6 +37,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. (Xuan Gong via jlowe) + MAPREDUCE-6621. Memory Leak in JobClient#submitJobInternal() (Xuan Gong + via jlowe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES @@ -340,6 +343,9 @@ Release 2.6.4 - UNRELEASED MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. (Xuan Gong via jlowe) + MAPREDUCE-6621. Memory Leak in JobClient#submitJobInternal() (Xuan Gong + via jlowe) + Release 2.6.3 - 2015-12-17 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index aae2e1b7f40..9228ff61bea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -576,10 +576,18 @@ public class JobClient extends CLI { return job; } }); + + Cluster prev = cluster; // update our Cluster instance with the one created by Job for submission // (we can't pass our Cluster instance to Job, since Job wraps the config // instance, and the two configs would then diverge) cluster = job.getCluster(); + + // It is important to close the previous cluster instance + // to cleanup resources. + if (prev != null) { + prev.close(); + } return new NetworkedJob(job); } catch (InterruptedException ie) { throw new IOException("interrupted", ie);