YARN-1912. ResourceLocalizer started without any jvm memory control.

Contributed by Masatake Iwasaki

(cherry picked from commit 6471d18bc7)

hadoop-yarn-project/CHANGES.txt

@@ -170,6 +170,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3271. FairScheduler: Move tests related to max-runnable-apps from 
     TestFairScheduler to TestAppRunnability. (nijel via kasha)
 
+    YARN-1912. ResourceLocalizer started without any jvm memory control.
+    (Masatake Iwasaki via xgong)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1004,6 +1004,13 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 
     NM_PREFIX + "health-checker.script.opts";
 
+  /** The JVM options used on forking ContainerLocalizer process
+      by container executor. */
+  public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY =
+      NM_PREFIX + "container-localizer.java.opts";
+  public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
+      "-Xmx256m";
+
   /** The Docker image name(For DockerContainerExecutor).*/
   public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
     NM_PREFIX + "docker-container-executor.image-name";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -246,6 +246,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     if (javaLibPath != null) {
       command.add("-Djava.library.path=" + javaLibPath);
     }
+    command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
     buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
     String[] commandArray = command.toArray(new String[command.size()]);
     ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java

@@ -697,6 +697,7 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
      if (javaLibPath != null) {
        command.add("-Djava.library.path=" + javaLibPath);
      }
+     command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
 
      ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, 
          localDirs);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java

@@ -55,6 +55,7 @@ import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.SerializedException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -313,6 +314,16 @@ public class ContainerLocalizer {
     return status;
   }
 
+  /**
+   * Returns the JVM options to to launch the resource localizer.
+   * @param conf the configuration properties to launch the resource localizer.
+   */
+  public static List<String> getJavaOpts(Configuration conf) {
+    String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
+        YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
+    return Arrays.asList(opts.split(" "));
+  }
+
   /**
    * Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor},
    * as expected by ContainerLocalizer.main

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -186,19 +186,20 @@ public class TestLinuxContainerExecutorWithMocks {
     try {
       mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
       List<String> result=readMockParams();
-      Assert.assertEquals(result.size(), 17);
+      Assert.assertEquals(result.size(), 18);
       Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
       Assert.assertEquals(result.get(1), "test");
       Assert.assertEquals(result.get(2), "0" );
       Assert.assertEquals(result.get(3),"application_0" );
       Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
       Assert.assertEquals(result.get(8), "-classpath" );
-      Assert.assertEquals(result.get(11),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
+      Assert.assertEquals(result.get(11), "-Xmx256m" );
-      Assert.assertEquals(result.get(12), "test");
+      Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
-      Assert.assertEquals(result.get(13), "application_0");
+      Assert.assertEquals(result.get(13), "test");
-      Assert.assertEquals(result.get(14),"12345" );
+      Assert.assertEquals(result.get(14), "application_0");
-      Assert.assertEquals(result.get(15),"localhost" );
+      Assert.assertEquals(result.get(15),"12345" );
-      Assert.assertEquals(result.get(16),"8040" );
+      Assert.assertEquals(result.get(16),"localhost" );
+      Assert.assertEquals(result.get(17),"8040" );
 
     } catch (InterruptedException e) {
       LOG.error("Error:"+e.getMessage(),e);