From b4c28a1aa912ad1c064999b6586c526a2f0e895e Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 1 Aug 2013 20:03:08 +0000 Subject: [PATCH] svn merge -c 1509401 FIXES: MAPREDUCE-5428. HistoryFileManager doesn't stop threads when service is stopped. Contributed by Karthik Kambatla git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1509404 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/hs/HistoryFileManager.java | 11 +++++++++-- .../hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java | 11 +++++++---- .../hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java | 6 ++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 81985265e57..d797de27ba3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -499,6 +499,9 @@ Release 2.1.0-beta - 2013-08-06 MAPREDUCE-5419. TestSlive is getting FileNotFound Exception (Robert Parker via jlowe) + MAPREDUCE-5428. HistoryFileManager doesn't stop threads when service is + stopped (Karthik Kambatla via jlowe) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 054f8e3e61b..fb6028451ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -64,6 +64,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ShutdownThreadsHelper; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import com.google.common.annotations.VisibleForTesting; @@ -473,8 +474,8 @@ public class HistoryFileManager extends AbstractService { private Path intermediateDoneDirPath = null; // Intermediate Done Dir Path private FileContext intermediateDoneDirFc; // Intermediate Done Dir // FileContext - - private ThreadPoolExecutor moveToDoneExecutor = null; + @VisibleForTesting + protected ThreadPoolExecutor moveToDoneExecutor = null; private long maxHistoryAge = 0; public HistoryFileManager() { @@ -543,6 +544,12 @@ public class HistoryFileManager extends AbstractService { super.serviceInit(conf); } + @Override + public void serviceStop() throws Exception { + ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor); + super.serviceStop(); + } + protected JobListCache createJobListCache() { return new JobListCache(conf.getInt( JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java index da852e3e90e..54a4a25db37 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java @@ -70,12 +70,15 @@ public class TestJobHistoryEvents { ((JobHistory)context).start(); Assert.assertTrue( context.getStartTime()>0); Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED); - - + + // get job before stopping JobHistory + Job parsedJob = context.getJob(jobId); + + // stop JobHistory ((JobHistory)context).stop(); Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED); - Job parsedJob = context.getJob(jobId); - + + Assert.assertEquals("CompletedMaps not correct", 2, parsedJob.getCompletedMaps()); Assert.assertEquals(System.getProperty("user.name"), parsedJob.getUserName()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index 42704fa87a5..87611ee99d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -535,7 +535,10 @@ public class TestJobHistoryParsing { Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0); fileInfo = hfm.getFileInfo(jobId); + hfm.stop(); Assert.assertNotNull("Unable to locate old job history", fileInfo); + Assert.assertTrue("HistoryFileManager not shutdown properly", + hfm.moveToDoneExecutor.isTerminated()); } finally { LOG.info("FINISHED testScanningOldDirs"); } @@ -636,6 +639,9 @@ public class TestJobHistoryParsing { // correct live time hfm.setMaxHistoryAge(-1); hfm.clean(); + hfm.stop(); + Assert.assertTrue("Thread pool shutdown", + hfm.moveToDoneExecutor.isTerminated()); // should be deleted ! Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());