MAPREDUCE-5232. Add a configuration to be able to log classpath and other system properties on mapreduce JVMs startup. Contributed by Sangjin Lee.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1482643 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 4d8e350750)
Conflicts:
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
This commit is contained in:
Vinod Kumar Vavilapalli
Gera Shegalov
parent
2e99210e4e
commit
6044b5e5f6
|
|
@ -189,6 +189,9 @@ Release 2.8.0 - UNRELEASED
|
|||
MAPREDUCE-6382. Don't escape HTML links in Diagnostics in JHS job overview.
|
||||
(Siqi Li via gera)
|
||||
|
||||
MAPREDUCE-5232. Add a configuration to be able to log classpath and other
|
||||
system properties on mapreduce JVMs startup. (Sangjin Lee via vinodkv)
|
||||
|
||||
Release 2.7.1 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -141,6 +141,12 @@ class YarnChild {
|
|||
// Create the job-conf and set credentials
|
||||
configureTask(job, task, credentials, jt);
|
||||
|
||||
// log the system properties
|
||||
String systemPropsToLog = MRApps.getSystemPropertiesToLog(job);
|
||||
if (systemPropsToLog != null) {
|
||||
LOG.info(systemPropsToLog);
|
||||
}
|
||||
|
||||
// Initiate Java VM metrics
|
||||
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
|
||||
childUGI = UserGroupInformation.createRemoteUser(System
|
||||
|
|
|
|||
|
|
@ -1495,6 +1495,12 @@ public class MRAppMaster extends CompositeService {
|
|||
conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
|
||||
|
||||
MRWebAppUtil.initialize(conf);
|
||||
// log the system properties
|
||||
String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf);
|
||||
if (systemPropsToLog != null) {
|
||||
LOG.info(systemPropsToLog);
|
||||
}
|
||||
|
||||
String jobUserName = System
|
||||
.getenv(ApplicationConstants.Environment.USER.name());
|
||||
conf.set(MRJobConfig.USER_NAME, jobUserName);
|
||||
|
|
|
|||
|
|
@ -733,4 +733,35 @@ public class MRApps extends Apps {
|
|||
MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
|
||||
return crossPlatform ? env.$$() : env.$();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return lines for system property keys and values per configuration.
|
||||
*
|
||||
* @return the formatted string for the system property lines or null if no
|
||||
* properties are specified.
|
||||
*/
|
||||
public static String getSystemPropertiesToLog(Configuration conf) {
|
||||
String key = conf.get(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG,
|
||||
MRJobConfig.DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG);
|
||||
if (key != null) {
|
||||
key = key.trim(); // trim leading and trailing whitespace from the config
|
||||
if (!key.isEmpty()) {
|
||||
String[] props = key.split(",");
|
||||
if (props.length > 0) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("\n/************************************************************\n");
|
||||
sb.append("[system properties]\n");
|
||||
for (String prop: props) {
|
||||
prop = prop.trim(); // trim leading and trailing whitespace
|
||||
if (!prop.isEmpty()) {
|
||||
sb.append(prop).append(": ").append(System.getProperty(prop)).append('\n');
|
||||
}
|
||||
}
|
||||
sb.append("************************************************************/");
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -480,7 +480,26 @@ public class TestMRApps {
|
|||
}
|
||||
public void initialize(URI name, Configuration conf) throws IOException {}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testLogSystemProperties() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
// test no logging
|
||||
conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
|
||||
String value = MRApps.getSystemPropertiesToLog(conf);
|
||||
assertNull(value);
|
||||
|
||||
// test logging of selected keys
|
||||
String classpath = "java.class.path";
|
||||
String os = "os.name";
|
||||
String version = "java.version";
|
||||
conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os);
|
||||
value = MRApps.getSystemPropertiesToLog(conf);
|
||||
assertNotNull(value);
|
||||
assertTrue(value.contains(classpath));
|
||||
assertTrue(value.contains(os));
|
||||
assertFalse(value.contains(version));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTaskStateUI() {
|
||||
|
|
|
|||
|
|
@ -205,6 +205,11 @@ public interface MRJobConfig {
|
|||
|
||||
public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
|
||||
|
||||
public static final String MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG = "mapreduce.jvm.system-properties-to-log";
|
||||
public static final String DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG =
|
||||
"os.name,os.version,java.home,java.runtime.version,java.vendor," +
|
||||
"java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name";
|
||||
|
||||
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
|
||||
|
||||
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
|
||||
|
|
|
|||
|
|
@ -1981,6 +1981,12 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jvm.system-properties-to-log</name>
|
||||
<value>os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name</value>
|
||||
<description>Comma-delimited list of system properties to log on mapreduce JVM start</description>
|
||||
</property>
|
||||
|
||||
<!-- jobhistory properties -->
|
||||
|
||||
<property>
|
||||
|
|
|
|||
Loading…
Reference in New Issue