YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7. Contributed by Wei Yan.

(cherry-picked from commit 7e3ebc0fc59dbd1d7768b5a6c05af18b8c8fea4d)
This commit is contained in:
Varun Vasudev 2015-07-07 16:59:29 +05:30
parent c5c0577248
commit 16ca0fcf99
10 changed files with 44 additions and 17 deletions

View File

@ -872,6 +872,10 @@ public static String join(CharSequence separator, Iterable<?> strings) {
return sb.toString(); return sb.toString();
} }
public static String join(char separator, Iterable<?> strings) {
return join(separator + "", strings);
}
/** /**
* Concatenates strings, using a separator. * Concatenates strings, using a separator.
* *
@ -894,6 +898,10 @@ public static String join(CharSequence separator, String[] strings) {
return sb.toString(); return sb.toString();
} }
public static String join(char separator, String[] strings) {
return join(separator + "", strings);
}
/** /**
* Convert SOME_STUFF to SomeStuff * Convert SOME_STUFF to SomeStuff
* *

View File

@ -278,8 +278,12 @@ public void testJoin() {
s.add("c"); s.add("c");
assertEquals("", StringUtils.join(":", s.subList(0, 0))); assertEquals("", StringUtils.join(":", s.subList(0, 0)));
assertEquals("a", StringUtils.join(":", s.subList(0, 1))); assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
assertEquals("", StringUtils.join(':', s.subList(0, 0)));
assertEquals("a", StringUtils.join(':', s.subList(0, 1)));
assertEquals("a:b", StringUtils.join(":", s.subList(0, 2))); assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3))); assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
assertEquals("a:b", StringUtils.join(':', s.subList(0, 2)));
assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3)));
} }
@Test (timeout = 30000) @Test (timeout = 30000)

View File

@ -547,6 +547,9 @@ Release 2.8.0 - UNRELEASED
YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
prefix for auth options. (Bibin A Chundatt via devaraj) prefix for auth options. (Bibin A Chundatt via devaraj)
YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7.
(Wei Yan via vvasudev)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -243,8 +243,10 @@ public void startLocalizer(LocalizerStartContext ctx)
Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()), Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
appId, appId,
nmPrivateContainerTokensPath.toUri().getPath().toString(), nmPrivateContainerTokensPath.toUri().getPath().toString(),
StringUtils.join(",", localDirs), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
StringUtils.join(",", logDirs))); localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs)));
File jvm = // use same jvm as parent File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java"); new File(new File(System.getProperty("java.home"), "bin"), "java");
@ -365,8 +367,10 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
nmPrivateContainerScriptPath.toUri().getPath().toString(), nmPrivateContainerScriptPath.toUri().getPath().toString(),
nmPrivateTokensPath.toUri().getPath().toString(), nmPrivateTokensPath.toUri().getPath().toString(),
pidFilePath.toString(), pidFilePath.toString(),
StringUtils.join(",", localDirs), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
StringUtils.join(",", logDirs), localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs),
resourcesOptions)); resourcesOptions));
if (tcCommandFile != null) { if (tcCommandFile != null) {

View File

@ -36,6 +36,7 @@
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Unstable @InterfaceStability.Unstable
public class PrivilegedOperation { public class PrivilegedOperation {
public final static char LINUX_FILE_PATH_SEPARATOR = '%';
public enum OperationType { public enum OperationType {
CHECK_SETUP("--checksetup"), CHECK_SETUP("--checksetup"),

View File

@ -234,7 +234,7 @@ public String executePrivilegedOperation(PrivilegedOperation operation,
if (noneArgsOnly == false) { if (noneArgsOnly == false) {
//We have already appended at least one tasks file. //We have already appended at least one tasks file.
finalOpArg.append(","); finalOpArg.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
finalOpArg.append(tasksFile); finalOpArg.append(tasksFile);
} else { } else {
finalOpArg.append(tasksFile); finalOpArg.append(tasksFile);

View File

@ -49,6 +49,7 @@
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.SystemClock;
@ -409,10 +410,11 @@ public String getResourcesOption(ContainerId containerId) {
if (isCpuWeightEnabled()) { if (isCpuWeightEnabled()) {
sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks"); sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
sb.append(","); sb.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
} }
if (sb.charAt(sb.length() - 1) == ',') { if (sb.charAt(sb.length() - 1) ==
PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR) {
sb.deleteCharAt(sb.length() - 1); sb.deleteCharAt(sb.length() - 1);
} }

View File

@ -283,7 +283,7 @@ char * get_value(const char* key) {
/** /**
* Function to return an array of values for a key. * Function to return an array of values for a key.
* Value delimiter is assumed to be a comma. * Value delimiter is assumed to be a '%'.
*/ */
char ** get_values(const char * key) { char ** get_values(const char * key) {
char *value = get_value(key); char *value = get_value(key);
@ -291,7 +291,7 @@ char ** get_values(const char * key) {
} }
/** /**
* Extracts array of values from the comma separated list of values. * Extracts array of values from the '%' separated list of values.
*/ */
char ** extract_values(char *value) { char ** extract_values(char *value) {
char ** toPass = NULL; char ** toPass = NULL;
@ -303,14 +303,14 @@ char ** extract_values(char *value) {
//first allocate any array of 10 //first allocate any array of 10
if(value != NULL) { if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize); toPass = (char **) malloc(sizeof(char *) * toPassSize);
tempTok = strtok_r((char *)value, ",", &tempstr); tempTok = strtok_r((char *)value, "%", &tempstr);
while (tempTok != NULL) { while (tempTok != NULL) {
toPass[size++] = tempTok; toPass[size++] = tempTok;
if(size == toPassSize) { if(size == toPassSize) {
toPassSize += MAX_SIZE; toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize)); toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
} }
tempTok = strtok_r(NULL, ",", &tempstr); tempTok = strtok_r(NULL, "%", &tempstr);
} }
} }
if (toPass != NULL) { if (toPass != NULL) {

View File

@ -30,8 +30,8 @@
#define TEST_ROOT "/tmp/test-container-executor" #define TEST_ROOT "/tmp/test-container-executor"
#define DONT_TOUCH_FILE "dont-touch-me" #define DONT_TOUCH_FILE "dont-touch-me"
#define NM_LOCAL_DIRS TEST_ROOT "/local-1," TEST_ROOT "/local-2," \ #define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5" TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
#define NM_LOG_DIRS TEST_ROOT "/logs/userlogs" #define NM_LOG_DIRS TEST_ROOT "/logs/userlogs"
#define ARRAY_SIZE 1000 #define ARRAY_SIZE 1000

View File

@ -49,6 +49,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
@ -148,8 +149,10 @@ public void testContainerLaunch() throws IOException {
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId, appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
StringUtils.join(",", dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"), dirsHandler.getLocalDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLogDirs()), "cgroups=none"),
readMockParams()); readMockParams());
} }
@ -311,8 +314,10 @@ public Object answer(InvocationOnMock invocationOnMock)
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId, appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
StringUtils.join(",", dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
StringUtils.join(",", dirsHandler.getLogDirs()), dirsHandler.getLocalDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLogDirs()),
"cgroups=none"), readMockParams()); "cgroups=none"), readMockParams());
} }