From e9a58691ab2d48b68af0dcfb64c62b6be033ec89 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Fri, 14 Oct 2016 23:32:29 +0530 Subject: [PATCH] YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena) --- .../hadoop/yarn/conf/YarnConfiguration.java | 12 ----------- .../src/main/resources/yarn-default.xml | 13 ------------ .../metrics/ApplicationMetricsConstants.java | 3 +++ .../amlauncher/AMLauncher.java | 21 ------------------- .../metrics/ApplicationCreatedEvent.java | 10 ++++++++- .../metrics/SystemMetricsPublisher.java | 9 +++++++- .../metrics/TestSystemMetricsPublisher.java | 17 +++++++++++++++ 7 files changed, 37 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ba7d04674b5..5716e3a8f80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -508,18 +508,6 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; - /** - * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the - * command being executed to the RM log if this property is true. Commands - * may contain sensitive information, such as application or service - * passwords, making logging the commands a security risk. In cases where - * the cluster may be running applications with such commands, this property - * should be set to false. Commands are only logged at the debug level. - */ - public static final String RM_AMLAUNCHER_LOG_COMMAND = - RM_PREFIX + "amlauncher.log.command"; - public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false; - //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1d3327aa58b..a1eff52c189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -298,19 +298,6 @@ 50 - - - The resource manager will log all commands being executed to the RM log - if this property is true. Commands may contain sensitive information, - such as application or service passwords, making logging the commands a - security risk. In cases where the cluster may be running applications with - such commands this property should be set to false. Commands are only - logged at the debug level. - - yarn.resourcemanager.amlauncher.log.command - false - - The class to use as the resource scheduler. yarn.resourcemanager.scheduler.class diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index d06b7cb68c7..17742089223 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -98,4 +98,7 @@ public class ApplicationMetricsConstants { public static final String AM_NODE_LABEL_EXPRESSION = "YARN_AM_NODE_LABEL_EXPRESSION"; + + public static final String AM_CONTAINER_LAUNCH_COMMAND = + "YARN_AM_CONTAINER_LAUNCH_COMMAND"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index e7105f996c2..e5131985863 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; /** * The launch of the AM itself. @@ -80,7 +79,6 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; - private final boolean logCommandLine; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,9 +91,6 @@ public class AMLauncher implements Runnable { this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); - this.logCommandLine = - conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND, - YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND); } private void connect() throws IOException { @@ -192,22 +187,6 @@ public class AMLauncher implements Runnable { ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); - if (LOG.isDebugEnabled()) { - StringBuilder message = new StringBuilder("Command to launch container "); - - message.append(containerID).append(" : "); - - if (logCommandLine) { - message.append(Joiner.on(",").join(container.getCommands())); - } else { - message.append(" -- Set "); - message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND); - message.append(" to true to reenable command logging"); - } - - LOG.debug(message.toString()); - } - // Populate the current queue name in the environment variable. setupQueueNameEnv(container, applicationMasterContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 968a8fd9c88..1adf89f5af1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -22,6 +22,7 @@ import java.util.Set; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; public class ApplicationCreatedEvent extends @@ -39,6 +40,7 @@ public class ApplicationCreatedEvent extends private String appNodeLabelsExpression; private String amNodeLabelsExpression; private final CallerContext callerContext; + private ContainerLaunchContext amContainerSpec; public ApplicationCreatedEvent(ApplicationId appId, @@ -53,7 +55,8 @@ public class ApplicationCreatedEvent extends Priority applicationPriority, String appNodeLabelsExpression, String amNodeLabelsExpression, - CallerContext callerContext) { + CallerContext callerContext, + ContainerLaunchContext amContainerSpec) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -67,6 +70,7 @@ public class ApplicationCreatedEvent extends this.appNodeLabelsExpression = appNodeLabelsExpression; this.amNodeLabelsExpression = amNodeLabelsExpression; this.callerContext = callerContext; + this.amContainerSpec = amContainerSpec; } @Override @@ -121,4 +125,8 @@ public class ApplicationCreatedEvent extends public CallerContext getCallerContext() { return callerContext; } + + public ContainerLaunchContext getAmContainerSpec() { + return amContainerSpec; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b0b976d91e5..250523a72de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; @@ -118,7 +119,8 @@ public class SystemMetricsPublisher extends CompositeService { appSubmissionContext.getPriority(), app.getAppNodeLabelExpression(), app.getAmNodeLabelExpression(), - app.getCallerContext())); + app.getCallerContext(), + appSubmissionContext.getAMContainerSpec())); } } @@ -318,6 +320,11 @@ public class SystemMetricsPublisher extends CompositeService { event.getCallerContext().getSignature()); } } + + ContainerLaunchContext amContainerSpec = event.getAmContainerSpec(); + entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND, + amContainerSpec.getCommands()); + entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 9ea2baa3dc0..914e5b96007 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.Map; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -118,6 +120,11 @@ public class TestSystemMetricsPublisher { when(asc.getUnmanagedAM()).thenReturn(false); when(asc.getPriority()).thenReturn(Priority.newInstance(1)); when(asc.getNodeLabelExpression()).thenReturn("high-cpu"); + ContainerLaunchContext containerLaunchContext = + mock(ContainerLaunchContext.class); + when(containerLaunchContext.getCommands()) + .thenReturn(Collections.singletonList("java -Xmx1024m")); + when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); metricsPublisher.appUpdated(app, 4L); } else { @@ -197,6 +204,11 @@ public class TestSystemMetricsPublisher { Assert.assertEquals("uers1,user2", entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + Assert.assertEquals( + app.getApplicationSubmissionContext().getAMContainerSpec() + .getCommands(), + entity.getOtherInfo() + .get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND)); } else { Assert.assertEquals( "", @@ -492,6 +504,11 @@ public class TestSystemMetricsPublisher { when(asc.getUnmanagedAM()).thenReturn(false); when(asc.getPriority()).thenReturn(Priority.newInstance(10)); when(asc.getNodeLabelExpression()).thenReturn("high-cpu"); + ContainerLaunchContext containerLaunchContext = + mock(ContainerLaunchContext.class); + when(containerLaunchContext.getCommands()) + .thenReturn(Collections.singletonList("java -Xmx1024m")); + when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); when(app.getAppNodeLabelExpression()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class);