Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena)

This commit is contained in:
Varun Saxena 2016-10-14 23:32:29 +05:30
parent 156b92e36b
commit e9a58691ab
7 changed files with 37 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -508,18 +508,6 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
10;
/**
* The {@code AMLauncher.createAMContainerLaunchContext()} method will log the
* command being executed to the RM log if this property is true. Commands
* may contain sensitive information, such as application or service
* passwords, making logging the commands a security risk. In cases where
* the cluster may be running applications with such commands, this property
* should be set to false. Commands are only logged at the debug level.
*/
public static final String RM_AMLAUNCHER_LOG_COMMAND =
RM_PREFIX + "amlauncher.log.command";
public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false;
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -298,19 +298,6 @@
<value>50</value>
</property>
<property>
<description>
The resource manager will log all commands being executed to the RM log
if this property is true. Commands may contain sensitive information,
such as application or service passwords, making logging the commands a
security risk. In cases where the cluster may be running applications with
such commands this property should be set to false. Commands are only
logged at the debug level.
</description>
<name>yarn.resourcemanager.amlauncher.log.command</name>
<value>false</value>
</property>
<property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
View File

@ -98,4 +98,7 @@ public class ApplicationMetricsConstants {
public static final String AM_NODE_LABEL_EXPRESSION =
"YARN_AM_NODE_LABEL_EXPRESSION";
public static final String AM_CONTAINER_LAUNCH_COMMAND =
"YARN_AM_CONTAINER_LAUNCH_COMMAND";
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI
import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
/**
* The launch of the AM itself.
@ -80,7 +79,6 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
private final boolean logCommandLine;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@ -93,9 +91,6 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
this.logCommandLine =
conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND,
YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND);
}
private void connect() throws IOException {
@ -192,22 +187,6 @@ public class AMLauncher implements Runnable {
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
if (LOG.isDebugEnabled()) {
StringBuilder message = new StringBuilder("Command to launch container ");
message.append(containerID).append(" : ");
if (logCommandLine) {
message.append(Joiner.on(",").join(container.getCommands()));
} else {
message.append("<REDACTED> -- Set ");
message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND);
message.append(" to true to reenable command logging");
}
LOG.debug(message.toString());
}
// Populate the current queue name in the environment variable.
setupQueueNameEnv(container, applicationMasterContext);

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
View File

@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Priority;
public class ApplicationCreatedEvent extends
@ -39,6 +40,7 @@ public class ApplicationCreatedEvent extends
private String appNodeLabelsExpression;
private String amNodeLabelsExpression;
private final CallerContext callerContext;
private ContainerLaunchContext amContainerSpec;
public ApplicationCreatedEvent(ApplicationId appId,
@ -53,7 +55,8 @@ public class ApplicationCreatedEvent extends
Priority applicationPriority,
String appNodeLabelsExpression,
String amNodeLabelsExpression,
CallerContext callerContext) {
CallerContext callerContext,
ContainerLaunchContext amContainerSpec) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@ -67,6 +70,7 @@ public class ApplicationCreatedEvent extends
this.appNodeLabelsExpression = appNodeLabelsExpression;
this.amNodeLabelsExpression = amNodeLabelsExpression;
this.callerContext = callerContext;
this.amContainerSpec = amContainerSpec;
}
@Override
@ -121,4 +125,8 @@ public class ApplicationCreatedEvent extends
public CallerContext getCallerContext() {
return callerContext;
}
public ContainerLaunchContext getAmContainerSpec() {
return amContainerSpec;
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@ -118,7 +119,8 @@ public class SystemMetricsPublisher extends CompositeService {
appSubmissionContext.getPriority(),
app.getAppNodeLabelExpression(),
app.getAmNodeLabelExpression(),
app.getCallerContext()));
app.getCallerContext(),
appSubmissionContext.getAMContainerSpec()));
}
}
@ -318,6 +320,11 @@ public class SystemMetricsPublisher extends CompositeService {
event.getCallerContext().getSignature());
}
}
ContainerLaunchContext amContainerSpec = event.getAmContainerSpec();
entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
amContainerSpec.getCommands());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map;
@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@ -118,6 +120,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(1));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
ContainerLaunchContext containerLaunchContext =
mock(ContainerLaunchContext.class);
when(containerLaunchContext.getCommands())
.thenReturn(Collections.singletonList("java -Xmx1024m"));
when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
metricsPublisher.appUpdated(app, 4L);
} else {
@ -197,6 +204,11 @@ public class TestSystemMetricsPublisher {
Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
app.getApplicationSubmissionContext().getAMContainerSpec()
.getCommands(),
entity.getOtherInfo()
.get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND));
} else {
Assert.assertEquals(
"",
@ -492,6 +504,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(10));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
ContainerLaunchContext containerLaunchContext =
mock(ContainerLaunchContext.class);
when(containerLaunchContext.getCommands())
.thenReturn(Collections.singletonList("java -Xmx1024m"));
when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);