YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux container executor. (Contributed by Yufei Gu)
This commit is contained in:
parent
fedabcad42
commit
d8923cdbf1
|
@ -1675,6 +1675,12 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
|
||||
"-Xmx256m";
|
||||
|
||||
/** The log level of container localizer process. */
|
||||
public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
|
||||
NM_PREFIX + "container-localizer.log.level";
|
||||
public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT =
|
||||
"INFO";
|
||||
|
||||
/** Prefix for runtime configuration constants. */
|
||||
public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
|
||||
"runtime.linux.";
|
||||
|
|
|
@ -1163,6 +1163,14 @@
|
|||
<value>-Xmx256m</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
The log level for container localizer while it is an independent process.
|
||||
</description>
|
||||
<name>yarn.nodemanager.container-localizer.log.level</name>
|
||||
<value>INFO</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
Where to store container logs. An application's localized log directory
|
||||
|
|
|
@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|||
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
|
||||
|
@ -382,6 +383,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
|||
List<String> localizerArgs = new ArrayList<>();
|
||||
|
||||
buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs);
|
||||
|
||||
Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId);
|
||||
localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir);
|
||||
|
||||
initializeContainerOp.appendArgs(localizerArgs);
|
||||
|
||||
try {
|
||||
|
@ -402,6 +407,27 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
|||
}
|
||||
}
|
||||
|
||||
private List<String> replaceWithContainerLogDir(List<String> commands,
|
||||
Path containerLogDir) {
|
||||
List<String> newCmds = new ArrayList<>(commands.size());
|
||||
|
||||
for (String item : commands) {
|
||||
newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
|
||||
containerLogDir.toString()));
|
||||
}
|
||||
|
||||
return newCmds;
|
||||
}
|
||||
|
||||
private Path getContainerLogDir(LocalDirsHandlerService dirsHandler,
|
||||
String appId, String containerId) throws IOException {
|
||||
String relativeContainerLogDir = ContainerLaunch
|
||||
.getRelativeContainerLogDir(appId, containerId);
|
||||
|
||||
return dirsHandler.getLogPathForWrite(relativeContainerLogDir,
|
||||
false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the {@link ContainerLocalizer}.
|
||||
*
|
||||
|
@ -417,7 +443,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
|||
public void buildMainArgs(List<String> command, String user, String appId,
|
||||
String locId, InetSocketAddress nmAddr, List<String> localDirs) {
|
||||
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
|
||||
localDirs);
|
||||
localDirs, super.getConf());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -702,7 +702,7 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
|
|||
command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
|
||||
|
||||
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
|
||||
localDirs);
|
||||
localDirs, super.getConf());
|
||||
|
||||
String cmdLine = StringUtils.join(command, " ");
|
||||
|
||||
|
|
|
@ -63,6 +63,8 @@ import org.apache.hadoop.util.DiskValidatorFactory;
|
|||
import org.apache.hadoop.util.Shell;
|
||||
import org.apache.hadoop.util.concurrent.HadoopExecutors;
|
||||
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||
import org.apache.hadoop.yarn.api.records.SerializedException;
|
||||
|
@ -408,8 +410,12 @@ public class ContainerLocalizer {
|
|||
*/
|
||||
public static void buildMainArgs(List<String> command,
|
||||
String user, String appId, String locId,
|
||||
InetSocketAddress nmAddr, List<String> localDirs) {
|
||||
|
||||
InetSocketAddress nmAddr, List<String> localDirs, Configuration conf) {
|
||||
|
||||
String logLevel = conf.get(YarnConfiguration.
|
||||
NM_CONTAINER_LOCALIZER_LOG_LEVEL,
|
||||
YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT);
|
||||
addLog4jSystemProperties(logLevel, command);
|
||||
command.add(ContainerLocalizer.class.getName());
|
||||
command.add(user);
|
||||
command.add(appId);
|
||||
|
@ -421,6 +427,16 @@ public class ContainerLocalizer {
|
|||
}
|
||||
}
|
||||
|
||||
private static void addLog4jSystemProperties(
|
||||
String logLevel, List<String> command) {
|
||||
command.add("-Dlog4j.configuration=container-log4j.properties");
|
||||
command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
|
||||
ApplicationConstants.LOG_DIR_EXPANSION_VAR);
|
||||
command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0");
|
||||
command.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
|
||||
command.add("-Dhadoop.root.logfile=container-localizer-syslog");
|
||||
}
|
||||
|
||||
public static void main(String[] argv) throws Throwable {
|
||||
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
|
||||
int nRet = 0;
|
||||
|
@ -431,6 +447,7 @@ public class ContainerLocalizer {
|
|||
// MKDIR $x/$user/appcache/$appid/filecache
|
||||
// LOAD $x/$user/appcache/$appid/appTokens
|
||||
try {
|
||||
createLogDir();
|
||||
String user = argv[0];
|
||||
String appId = argv[1];
|
||||
String locId = argv[2];
|
||||
|
@ -466,6 +483,31 @@ public class ContainerLocalizer {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the log directory, if the directory exists, make sure its permission
|
||||
* is 750.
|
||||
*/
|
||||
private static void createLogDir() {
|
||||
FileContext localFs;
|
||||
try {
|
||||
localFs = FileContext.getLocalFSFileContext(new Configuration());
|
||||
|
||||
String logDir = System.getProperty(
|
||||
YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
|
||||
|
||||
if (logDir != null && !logDir.trim().isEmpty()) {
|
||||
Path containerLogPath = new Path(logDir);
|
||||
FsPermission containerLogDirPerm= new FsPermission((short)0750);
|
||||
localFs.mkdir(containerLogPath, containerLogDirPerm, true);
|
||||
// set permission again to make sure the permission is correct
|
||||
// in case the directory is already there.
|
||||
localFs.setPermission(containerLogPath, containerLogDirPerm);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new YarnRuntimeException("Unable to create the log dir", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void initDirs(Configuration conf, String user, String appId,
|
||||
FileContext lfs, List<Path> localDirs) throws IOException {
|
||||
if (null == localDirs || 0 == localDirs.size()) {
|
||||
|
|
|
@ -269,7 +269,7 @@ public class TestLinuxContainerExecutorWithMocks {
|
|||
.build());
|
||||
|
||||
List<String> result=readMockParams();
|
||||
Assert.assertEquals(result.size(), 19);
|
||||
Assert.assertEquals(result.size(), 24);
|
||||
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
|
||||
Assert.assertEquals(result.get(1), "test");
|
||||
Assert.assertEquals(result.get(2), "0" );
|
||||
|
@ -277,12 +277,17 @@ public class TestLinuxContainerExecutorWithMocks {
|
|||
Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
|
||||
Assert.assertEquals(result.get(8), "-classpath" );
|
||||
Assert.assertEquals(result.get(11), "-Xmx256m" );
|
||||
Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
|
||||
Assert.assertEquals(result.get(13), "test");
|
||||
Assert.assertEquals(result.get(14), "application_0");
|
||||
Assert.assertEquals(result.get(15),"12345" );
|
||||
Assert.assertEquals(result.get(16),"localhost" );
|
||||
Assert.assertEquals(result.get(17),"8040" );
|
||||
Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" );
|
||||
Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345");
|
||||
Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0");
|
||||
Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA");
|
||||
Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog");
|
||||
Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
|
||||
Assert.assertEquals(result.get(18), "test");
|
||||
Assert.assertEquals(result.get(19), "application_0");
|
||||
Assert.assertEquals(result.get(20),"12345" );
|
||||
Assert.assertEquals(result.get(21),"localhost" );
|
||||
Assert.assertEquals(result.get(22),"8040" );
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
LOG.error("Error:"+e.getMessage(),e);
|
||||
|
|
Loading…
Reference in New Issue