From d6e3164d4a18271299c63377326ca56e8a980830 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 27 May 2015 16:40:56 -0700 Subject: [PATCH] YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/api/ApplicationConstants.java | 50 ++++++++----------- .../amlauncher/AMLauncher.java | 29 +++++------ .../TestApplicationMasterLauncher.java | 7 ++- 4 files changed, 41 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 14dbcc9b394..2ae923ddb14 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -48,6 +48,9 @@ Trunk - Unreleased YARN-2428. LCE default banned user list should have yarn (Varun Saxena via aw) + YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var + for a container (Darrell Taylor via aw) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 42464da1a43..d75297a4168 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -28,13 +28,13 @@ import org.apache.hadoop.util.Shell; /** * This is the API for the applications comprising of constants that YARN sets * up for the applications and the containers. - * + * * TODO: Investigate the semantics and security of each cross-boundary refs. */ @Public @Evolving public interface ApplicationConstants { - + /** * The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment * only @@ -48,11 +48,11 @@ public interface ApplicationConstants { UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; /** - * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in + * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in * ApplicationMaster's environment only. This states that for all non-relative * web URLs in the app masters web UI what base should they have. */ - public static final String APPLICATION_WEB_PROXY_BASE_ENV = + public static final String APPLICATION_WEB_PROXY_BASE_ENV = "APPLICATION_WEB_PROXY_BASE"; /** @@ -98,16 +98,10 @@ public interface ApplicationConstants { public static final String STDOUT = "stdout"; - /** - * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment - * only - */ - public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS"; - /** * Environment for Applications. - * - * Some of the environment variables for applications are final + * + * Some of the environment variables for applications are final * i.e. they cannot be modified by the applications. */ public enum Environment { @@ -116,76 +110,76 @@ public interface ApplicationConstants { * Final, non-modifiable. */ USER("USER"), - + /** * $LOGNAME * Final, non-modifiable. */ LOGNAME("LOGNAME"), - + /** * $HOME * Final, non-modifiable. */ HOME("HOME"), - + /** * $PWD * Final, non-modifiable. */ PWD("PWD"), - + /** * $PATH */ PATH("PATH"), - + /** * $SHELL */ SHELL("SHELL"), - + /** * $JAVA_HOME */ JAVA_HOME("JAVA_HOME"), - + /** * $CLASSPATH */ CLASSPATH("CLASSPATH"), - + /** * $APP_CLASSPATH */ APP_CLASSPATH("APP_CLASSPATH"), - + /** * $LD_LIBRARY_PATH */ LD_LIBRARY_PATH("LD_LIBRARY_PATH"), - + /** * $HADOOP_CONF_DIR * Final, non-modifiable. */ HADOOP_CONF_DIR("HADOOP_CONF_DIR"), - + /** * $HADOOP_COMMON_HOME */ HADOOP_COMMON_HOME("HADOOP_COMMON_HOME"), - + /** * $HADOOP_HDFS_HOME */ HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"), - + /** * $MALLOC_ARENA_MAX */ MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"), - + /** * $HADOOP_YARN_HOME */ @@ -240,11 +234,11 @@ public interface ApplicationConstants { private Environment(String variable) { this.variable = variable; } - + public String key() { return variable; } - + public String toString() { return variable; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index b44d13bd588..713e75fa5e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -79,10 +79,10 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; - + @SuppressWarnings("rawtypes") private final EventHandler handler; - + public AMLauncher(RMContext rmContext, RMAppAttempt application, AMLauncherEventType eventType, Configuration conf) { this.application = application; @@ -92,20 +92,20 @@ public class AMLauncher implements Runnable { this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); } - + private void connect() throws IOException { ContainerId masterContainerID = masterContainer.getId(); - + containerMgrProxy = getContainerMgrProxy(masterContainerID); } - + private void launch() throws IOException, YarnException { connect(); ContainerId masterContainerID = masterContainer.getId(); ApplicationSubmissionContext applicationContext = application.getSubmissionContext(); LOG.info("Setting up container " + masterContainer - + " for AM " + application.getAppAttemptId()); + + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); @@ -129,7 +129,7 @@ public class AMLauncher implements Runnable { + application.getAppAttemptId()); } } - + private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); @@ -187,17 +187,17 @@ public class AMLauncher implements Runnable { ContainerId containerID) throws IOException { // Construct the actual Container - ContainerLaunchContext container = + ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); LOG.info("Command to launch container " + containerID + " : " + StringUtils.arrayToString(container.getCommands().toArray( new String[0]))); - + // Finalize the container setupTokens(container, containerID); - + return container; } @@ -209,7 +209,7 @@ public class AMLauncher implements Runnable { Map environment = container.getEnvironment(); environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase()); - // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. + // Set AppSubmitTime to be consumable by the AM. ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); environment.put( @@ -217,9 +217,6 @@ public class AMLauncher implements Runnable { String.valueOf(rmContext.getRMApps() .get(applicationId) .getSubmitTime())); - environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV, - String.valueOf(rmContext.getRMApps().get( - applicationId).getMaxAppAttempts())); Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); @@ -249,7 +246,7 @@ public class AMLauncher implements Runnable { ((RMAppAttemptImpl)application).setAMRMToken(amrmToken); return amrmToken; } - + @SuppressWarnings("unchecked") public void run() { switch (eventType) { @@ -279,7 +276,7 @@ public class AMLauncher implements Runnable { sb.append(" is not handled by this NodeManager"); if (!e.getMessage().contains(sb.toString())) { // Ignoring if container is already killed by Node Manager. - LOG.info("Error cleaning master ", e); + LOG.info("Error cleaning master ", e); } } break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 9a4395e20fe..d4f8e93b88d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -107,8 +107,7 @@ public class TestApplicationMasterLauncher { nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); - maxAppAttempts = - Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); + maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS; return StartContainersResponse.newInstance( new HashMap(), new ArrayList(), new HashMap()); @@ -184,8 +183,8 @@ public class TestApplicationMasterLauncher { am.waitForState(RMAppAttemptState.FINISHED); rm.stop(); } - - + + @SuppressWarnings("unused") @Test(timeout = 100000) public void testallocateBeforeAMRegistration() throws Exception {