MAPREDUCE-5428. HistoryFileManager doesn't stop threads when service is stopped. Contributed by Karthik Kambatla

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1509401 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -636,6 +636,9 @@ Release 2.1.0-beta - 2013-08-06
     MAPREDUCE-5419. TestSlive is getting FileNotFound Exception (Robert Parker
     via jlowe)
 
+    MAPREDUCE-5428.  HistoryFileManager doesn't stop threads when service is
+    stopped (Karthik Kambatla via jlowe)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS
 
     MAPREDUCE-4739. Some MapReduce tests fail to find winutils.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

@@ -64,6 +64,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ShutdownThreadsHelper;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -473,8 +474,8 @@ public class HistoryFileManager extends AbstractService {
   private Path intermediateDoneDirPath = null; // Intermediate Done Dir Path
   private FileContext intermediateDoneDirFc; // Intermediate Done Dir
                                              // FileContext
-
+  @VisibleForTesting
-  private ThreadPoolExecutor moveToDoneExecutor = null;
+  protected ThreadPoolExecutor moveToDoneExecutor = null;
   private long maxHistoryAge = 0;
   
   public HistoryFileManager() {
@@ -543,6 +544,12 @@ public class HistoryFileManager extends AbstractService {
     super.serviceInit(conf);
   }
 
+  @Override
+  public void serviceStop() throws Exception {
+    ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor);
+    super.serviceStop();
+  }
+
   protected JobListCache createJobListCache() {
     return new JobListCache(conf.getInt(
         JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java

@@ -70,12 +70,15 @@ public class TestJobHistoryEvents {
     ((JobHistory)context).start();
     Assert.assertTrue( context.getStartTime()>0);
     Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);
-    
+
-    
+    // get job before stopping JobHistory
+    Job parsedJob = context.getJob(jobId);
+
+    // stop JobHistory
     ((JobHistory)context).stop();
     Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);
-      Job parsedJob = context.getJob(jobId);
+
-    
+
     Assert.assertEquals("CompletedMaps not correct", 2,
         parsedJob.getCompletedMaps());
     Assert.assertEquals(System.getProperty("user.name"), parsedJob.getUserName());

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java

@@ -534,7 +534,10 @@ public class TestJobHistoryParsing {
       Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);
 
       fileInfo = hfm.getFileInfo(jobId);
+      hfm.stop();
       Assert.assertNotNull("Unable to locate old job history", fileInfo);
+      Assert.assertTrue("HistoryFileManager not shutdown properly",
+          hfm.moveToDoneExecutor.isTerminated());
     } finally {
       LOG.info("FINISHED testScanningOldDirs");
     }
@@ -637,6 +640,9 @@ public class TestJobHistoryParsing {
       // correct live time
       hfm.setMaxHistoryAge(-1);
       hfm.clean();
+      hfm.stop();
+      Assert.assertTrue("Thread pool shutdown",
+          hfm.moveToDoneExecutor.isTerminated());
       // should be deleted !
       Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());