From 868362c9b16397a2b491c06f920452e6233d0922 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 1 May 2013 00:44:13 +0000 Subject: [PATCH] MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs inline with YARN. Contributed by Zhijie Shen. svn merge --ignore-ancestry -c 1477889 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1477890 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/app/TestStagingCleanup.java | 9 +++++---- .../java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 +- .../src/main/resources/mapred-default.xml | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5519e82fb6d..7effb2ad91a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -53,6 +53,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5069. add concrete common implementations of CombineFileInputFormat (Sangjin Lee via bobby) + MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs + inline with YARN. (Zhijie Shen via vinodkv) + OPTIMIZATIONS MAPREDUCE-4974. Optimising the LineRecordReader initialize() method diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java index 10b79aba73e..0bd2215fa85 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java @@ -90,6 +90,7 @@ public void testDeletionofStaging() throws IOException { JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); + Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); appMaster.init(conf); @@ -116,8 +117,9 @@ public void testNoDeletionofStagingOnReboot() throws IOException { appId.setId(0); attemptId.setApplicationId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); + Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, - JobStateInternal.REBOOT, 4); + JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); appMaster.init(conf); appMaster.start(); //shutdown the job, not the lastRetry @@ -144,7 +146,7 @@ public void testDeletionofStagingOnReboot() throws IOException { attemptId.setApplicationId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, - JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); + JobStateInternal.REBOOT, 1); //no retry appMaster.init(conf); appMaster.start(); //shutdown the job, is lastRetry @@ -201,8 +203,7 @@ public void testDeletionofStagingOnKillLastTry() throws IOException { JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); - MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, - MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); + MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 1); //no retry appMaster.init(conf); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 0c1c6fd3e0a..110b18d5563 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -677,6 +677,6 @@ public interface MRJobConfig { */ public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts"; - public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1; + public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index aeac39fd9cd..8c720968b95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1250,10 +1250,11 @@ mapreduce.am.max-attempts - 1 + 2 The maximum number of application attempts. It is a application-specific setting. It should not be larger than the global number - set by resourcemanager. Otherwise, it will be override. + set by resourcemanager. Otherwise, it will be override. The default number is + set to 2, to allow at least one retry for AM.