YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7. Contributed by Wei Yan.

(cherry-picked from commit 7e3ebc0fc59dbd1d7768b5a6c05af18b8c8fea4d)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -872,6 +872,10 @@ public class StringUtils {
     return sb.toString();
   }
 
+  public static String join(char separator, Iterable<?> strings) {
+    return join(separator + "", strings);
+  }
+
   /**
    * Concatenates strings, using a separator.
    *
@@ -894,6 +898,10 @@ public class StringUtils {
     return sb.toString();
   }
 
+  public static String join(char separator, String[] strings) {
+    return join(separator + "", strings);
+  }
+
   /**
    * Convert SOME_STUFF to SomeStuff
    *

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java

@@ -278,8 +278,12 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
     s.add("c");
     assertEquals("", StringUtils.join(":", s.subList(0, 0)));
     assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
+    assertEquals("", StringUtils.join(':', s.subList(0, 0)));
+    assertEquals("a", StringUtils.join(':', s.subList(0, 1)));
     assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
     assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
+    assertEquals("a:b", StringUtils.join(':', s.subList(0, 2)));
+    assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3)));
   }
   
   @Test (timeout = 30000)

hadoop-yarn-project/CHANGES.txt

@@ -547,6 +547,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
     prefix for auth options. (Bibin A Chundatt via devaraj)
 
+    YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7.
+    (Wei Yan via vvasudev)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -243,8 +243,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
                    Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
                    appId,
                    nmPrivateContainerTokensPath.toUri().getPath().toString(),
-                   StringUtils.join(",", localDirs),
+                   StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-                   StringUtils.join(",", logDirs)));
+                       localDirs),
+                   StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+                       logDirs)));
 
     File jvm =                                  // use same jvm as parent
       new File(new File(System.getProperty("java.home"), "bin"), "java");
@@ -365,8 +367,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
             nmPrivateContainerScriptPath.toUri().getPath().toString(),
             nmPrivateTokensPath.toUri().getPath().toString(),
             pidFilePath.toString(),
-            StringUtils.join(",", localDirs),
+            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-            StringUtils.join(",", logDirs),
+                localDirs),
+            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+                logDirs),
             resourcesOptions));
 
         if (tcCommandFile != null) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java

@@ -36,6 +36,7 @@ import java.util.List;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class PrivilegedOperation {
+  public final static char LINUX_FILE_PATH_SEPARATOR = '%';
 
   public enum OperationType {
     CHECK_SETUP("--checksetup"),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java

@@ -234,7 +234,7 @@ public class PrivilegedOperationExecutor {
 
       if (noneArgsOnly == false) {
         //We have already appended at least one tasks file.
-        finalOpArg.append(",");
+        finalOpArg.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
         finalOpArg.append(tasksFile);
       } else {
         finalOpArg.append(tasksFile);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java

@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.SystemClock;
@@ -409,10 +410,11 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
 
     if (isCpuWeightEnabled()) {
       sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
-      sb.append(",");
+      sb.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
     }
 
-    if (sb.charAt(sb.length() - 1) == ',') {
+    if (sb.charAt(sb.length() - 1) ==
+        PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR) {
       sb.deleteCharAt(sb.length() - 1);
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c

@@ -283,7 +283,7 @@ char * get_value(const char* key) {
 
 /**
  * Function to return an array of values for a key.
- * Value delimiter is assumed to be a comma.
+ * Value delimiter is assumed to be a '%'.
  */
 char ** get_values(const char * key) {
   char *value = get_value(key);
@@ -291,7 +291,7 @@ char ** get_values(const char * key) {
 }
 
 /**
- * Extracts array of values from the comma separated list of values.
+ * Extracts array of values from the '%' separated list of values.
  */
 char ** extract_values(char *value) {
   char ** toPass = NULL;
@@ -303,14 +303,14 @@ char ** extract_values(char *value) {
   //first allocate any array of 10
   if(value != NULL) {
     toPass = (char **) malloc(sizeof(char *) * toPassSize);
-    tempTok = strtok_r((char *)value, ",", &tempstr);
+    tempTok = strtok_r((char *)value, "%", &tempstr);
     while (tempTok != NULL) {
       toPass[size++] = tempTok;
       if(size == toPassSize) {
         toPassSize += MAX_SIZE;
         toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
       }
-      tempTok = strtok_r(NULL, ",", &tempstr);
+      tempTok = strtok_r(NULL, "%", &tempstr);
     }
   }
   if (toPass != NULL) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c

@@ -30,8 +30,8 @@
 
 #define TEST_ROOT "/tmp/test-container-executor"
 #define DONT_TOUCH_FILE "dont-touch-me"
-#define NM_LOCAL_DIRS       TEST_ROOT "/local-1," TEST_ROOT "/local-2," \
+#define NM_LOCAL_DIRS       TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
-               TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5"
+               TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
 #define NM_LOG_DIRS         TEST_ROOT "/logs/userlogs"
 #define ARRAY_SIZE 1000
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
@@ -148,8 +149,10 @@ public class TestLinuxContainerExecutorWithMocks {
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, appId, containerId,
         workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
-        StringUtils.join(",", dirsHandler.getLocalDirs()),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-        StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"),
+            dirsHandler.getLocalDirs()),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLogDirs()), "cgroups=none"),
         readMockParams());
     
   }
@@ -311,8 +314,10 @@ public class TestLinuxContainerExecutorWithMocks {
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, appId, containerId,
         workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
-        StringUtils.join(",", dirsHandler.getLocalDirs()),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-        StringUtils.join(",", dirsHandler.getLogDirs()),
+            dirsHandler.getLocalDirs()),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLogDirs()),
         "cgroups=none"), readMockParams());
 
   }