diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index edeec9f2d6d..4820903f9e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1540,6 +1540,12 @@ public class YarnConfiguration extends Configuration { public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = "-Xmx256m"; + /** The log level of container localizer process. */ + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL= + NM_PREFIX + "container-localizer.log.level"; + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT = + "INFO"; + /** The Docker image name(For DockerContainerExecutor).*/ public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = NM_PREFIX + "docker-container-executor.image-name"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 71dd72aa149..e869e30bfae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1150,6 +1150,14 @@ -Xmx256m + + + The log level for container localizer while it is an independent process. + + yarn.nodemanager.container-localizer.log.level + INFO + + Where to store container logs. An application's localized log directory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index c1c3dc0c438..a75a32cd3ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ConfigurationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; @@ -381,6 +382,10 @@ public class LinuxContainerExecutor extends ContainerExecutor { List localizerArgs = new ArrayList<>(); buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs); + + Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId); + localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir); + initializeContainerOp.appendArgs(localizerArgs); try { @@ -401,6 +406,27 @@ public class LinuxContainerExecutor extends ContainerExecutor { } } + private List replaceWithContainerLogDir(List commands, + Path containerLogDir) { + List newCmds = new ArrayList<>(commands.size()); + + for (String item : commands) { + newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, + containerLogDir.toString())); + } + + return newCmds; + } + + private Path getContainerLogDir(LocalDirsHandlerService dirsHandler, + String appId, String containerId) throws IOException { + String relativeContainerLogDir = ContainerLaunch + .getRelativeContainerLogDir(appId, containerId); + + return dirsHandler.getLogPathForWrite(relativeContainerLogDir, + false); + } + /** * Set up the {@link ContainerLocalizer}. * @@ -416,7 +442,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { public void buildMainArgs(List command, String user, String appId, String locId, InetSocketAddress nmAddr, List localDirs) { ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 7c124c3700a..e9344727ccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -702,7 +702,7 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { command.addAll(ContainerLocalizer.getJavaOpts(getConf())); ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); String cmdLine = StringUtils.join(command, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 8c634790e0e..6e1ace4cb56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -60,6 +60,8 @@ import org.apache.hadoop.util.DiskValidatorFactory; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.SerializedException; @@ -407,8 +409,12 @@ public class ContainerLocalizer { */ public static void buildMainArgs(List command, String user, String appId, String locId, - InetSocketAddress nmAddr, List localDirs) { - + InetSocketAddress nmAddr, List localDirs, Configuration conf) { + + String logLevel = conf.get(YarnConfiguration. + NM_CONTAINER_LOCALIZER_LOG_LEVEL, + YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT); + addLog4jSystemProperties(logLevel, command); command.add(ContainerLocalizer.class.getName()); command.add(user); command.add(appId); @@ -420,6 +426,16 @@ public class ContainerLocalizer { } } + private static void addLog4jSystemProperties( + String logLevel, List command) { + command.add("-Dlog4j.configuration=container-log4j.properties"); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + + ApplicationConstants.LOG_DIR_EXPANSION_VAR); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0"); + command.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); + command.add("-Dhadoop.root.logfile=container-localizer-syslog"); + } + public static void main(String[] argv) throws Throwable { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); int nRet = 0; @@ -430,6 +446,7 @@ public class ContainerLocalizer { // MKDIR $x/$user/appcache/$appid/filecache // LOAD $x/$user/appcache/$appid/appTokens try { + createLogDir(); String user = argv[0]; String appId = argv[1]; String locId = argv[2]; @@ -465,6 +482,31 @@ public class ContainerLocalizer { } } + /** + * Create the log directory, if the directory exists, make sure its permission + * is 750. + */ + private static void createLogDir() { + FileContext localFs; + try { + localFs = FileContext.getLocalFSFileContext(new Configuration()); + + String logDir = System.getProperty( + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); + + if (logDir != null && !logDir.trim().isEmpty()) { + Path containerLogPath = new Path(logDir); + FsPermission containerLogDirPerm= new FsPermission((short)0750); + localFs.mkdir(containerLogPath, containerLogDirPerm, true); + // set permission again to make sure the permission is correct + // in case the directory is already there. + localFs.setPermission(containerLogPath, containerLogDirPerm); + } + } catch (IOException e) { + throw new YarnRuntimeException("Unable to create the log dir", e); + } + } + private static void initDirs(Configuration conf, String user, String appId, FileContext lfs, List localDirs) throws IOException { if (null == localDirs || 0 == localDirs.size()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 6eb1d9d6918..db36868906f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -269,7 +269,7 @@ public class TestLinuxContainerExecutorWithMocks { .build()); List result=readMockParams(); - Assert.assertEquals(result.size(), 19); + Assert.assertEquals(result.size(), 24); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); @@ -277,12 +277,17 @@ public class TestLinuxContainerExecutorWithMocks { Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); Assert.assertEquals(result.get(8), "-classpath" ); Assert.assertEquals(result.get(11), "-Xmx256m" ); - Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(13), "test"); - Assert.assertEquals(result.get(14), "application_0"); - Assert.assertEquals(result.get(15),"12345" ); - Assert.assertEquals(result.get(16),"localhost" ); - Assert.assertEquals(result.get(17),"8040" ); + Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" ); + Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345"); + Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0"); + Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA"); + Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog"); + Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(18), "test"); + Assert.assertEquals(result.get(19), "application_0"); + Assert.assertEquals(result.get(20),"12345" ); + Assert.assertEquals(result.get(21),"localhost" ); + Assert.assertEquals(result.get(22),"8040" ); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e);