Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena)

This commit is contained in:
Varun Saxena 2016-09-28 16:10:10 +05:30
parent bc2656f09f
commit 9b0fd01d2e
8 changed files with 41 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -534,18 +534,6 @@ public class YarnConfiguration extends Configuration {
public static final int
DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
/**
* The {@code AMLauncher.createAMContainerLaunchContext()} method will log the
* command being executed to the RM log if this property is true. Commands
* may contain sensitive information, such as application or service
* passwords, making logging the commands a security risk. In cases where
* the cluster may be running applications with such commands, this property
* should be set to false. Commands are only logged at the debug level.
*/
public static final String RM_AMLAUNCHER_LOG_COMMAND =
RM_PREFIX + "amlauncher.log.command";
public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false;
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -298,19 +298,6 @@
<value>50</value>
</property>
<property>
<description>
The resource manager will log all commands being executed to the RM log
if this property is true. Commands may contain sensitive information,
such as application or service passwords, making logging the commands a
security risk. In cases where the cluster may be running applications with
such commands this property should be set to false. Commands are only
logged at the debug level.
</description>
<name>yarn.resourcemanager.amlauncher.log.command</name>
<value>false</value>
</property>
<property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
View File

@ -98,4 +98,7 @@ public class ApplicationMetricsConstants {
public static final String AM_NODE_LABEL_EXPRESSION =
"YARN_AM_NODE_LABEL_EXPRESSION";
public static final String AM_CONTAINER_LAUNCH_COMMAND =
"YARN_AM_CONTAINER_LAUNCH_COMMAND";
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -66,7 +66,6 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
/**
* The launch of the AM itself.
@ -82,7 +81,6 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
private final boolean logCommandLine;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@ -95,9 +93,6 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
this.logCommandLine =
conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND,
YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND);
}
private void connect() throws IOException {
@ -194,22 +189,6 @@ public class AMLauncher implements Runnable {
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
if (LOG.isDebugEnabled()) {
StringBuilder message = new StringBuilder("Command to launch container ");
message.append(containerID).append(" : ");
if (logCommandLine) {
message.append(Joiner.on(",").join(container.getCommands()));
} else {
message.append("<REDACTED> -- Set ");
message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND);
message.append(" to true to reenable command logging");
}
LOG.debug(message.toString());
}
// Populate the current queue name in the environment variable.
setupQueueNameEnv(container, applicationMasterContext);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.TimelineClient;
@ -103,6 +104,11 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
}
}
ContainerLaunchContext amContainerSpec =
app.getApplicationSubmissionContext().getAMContainerSpec();
entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
amContainerSpec.getCommands());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
@ -128,6 +129,10 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
app.getCallerContext().getSignature());
}
}
ContainerLaunchContext amContainerSpec =
app.getApplicationSubmissionContext().getAMContainerSpec();
entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
amContainerSpec.getCommands());
entity.setInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map;
@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@ -118,6 +120,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(1));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
ContainerLaunchContext containerLaunchContext =
mock(ContainerLaunchContext.class);
when(containerLaunchContext.getCommands())
.thenReturn(Collections.singletonList("java -Xmx1024m"));
when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
metricsPublisher.appUpdated(app, 4L);
} else {
@ -197,6 +204,12 @@ public class TestSystemMetricsPublisher {
Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
app.getApplicationSubmissionContext().getAMContainerSpec()
.getCommands(),
entity.getOtherInfo()
.get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND));
} else {
Assert.assertEquals(
"",
@ -492,6 +505,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(10));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
ContainerLaunchContext containerLaunchContext =
mock(ContainerLaunchContext.class);
when(containerLaunchContext.getCommands())
.thenReturn(Collections.singletonList("java -Xmx1024m"));
when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
View File

@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@ -354,6 +355,14 @@ public class TestSystemMetricsPublisherForV2 {
mock(ApplicationSubmissionContext.class);
when(appSubmissionContext.getPriority())
.thenReturn(Priority.newInstance(0));
ContainerLaunchContext containerLaunchContext =
mock(ContainerLaunchContext.class);
when(containerLaunchContext.getCommands())
.thenReturn(Collections.singletonList("java -Xmx1024m"));
when(appSubmissionContext.getAMContainerSpec())
.thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext())
.thenReturn(appSubmissionContext);
return app;