From 02272a6a77ce958cab54242933cdc67ff22d2ca7 Mon Sep 17 00:00:00 2001 From: Ray Chiang Date: Fri, 2 Sep 2016 14:57:05 -0700 Subject: [PATCH] YARN-5549. AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately. (Daniel Templeton via rchiang) (cherry picked from commit 4ee1729cb04e72f3015666d750ad1e54257345d2) --- .../hadoop/yarn/conf/YarnConfiguration.java | 12 ++++++++ .../src/main/resources/yarn-default.xml | 13 +++++++++ .../amlauncher/AMLauncher.java | 28 +++++++++++++++---- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 5ce171c4eda..96c4f6faff8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -403,6 +403,18 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; + /** + * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the + * command being executed to the RM log if this property is true. Commands + * may contain sensitive information, such as application or service + * passwords, making logging the commands a security risk. In cases where + * the cluster may be running applications with such commands, this property + * should be set to false. Commands are only logged at the debug level. + */ + public static final String RM_AMLAUNCHER_LOG_COMMAND = + RM_PREFIX + "amlauncher.log.command"; + public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false; + //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 788b0fd75c7..512db818e06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -298,6 +298,19 @@ 50 + + + The resource manager will log all commands being executed to the RM log + if this property is true. Commands may contain sensitive information, + such as application or service passwords, making logging the commands a + security risk. In cases where the cluster may be running applications with + such commands this property should be set to false. Commands are only + logged at the debug level. + + yarn.resourcemanager.amlauncher.log.command + false + + The class to use as the resource scheduler. yarn.resourcemanager.scheduler.class diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 4c840e74a47..e7105f996c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; /** * The launch of the AM itself. @@ -79,7 +80,8 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; - + private final boolean logCommandLine; + @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -91,6 +93,9 @@ public class AMLauncher implements Runnable { this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.logCommandLine = + conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND, + YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND); } private void connect() throws IOException { @@ -186,11 +191,22 @@ public class AMLauncher implements Runnable { // Construct the actual Container ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); - LOG.info("Command to launch container " - + containerID - + " : " - + StringUtils.arrayToString(container.getCommands().toArray( - new String[0]))); + + if (LOG.isDebugEnabled()) { + StringBuilder message = new StringBuilder("Command to launch container "); + + message.append(containerID).append(" : "); + + if (logCommandLine) { + message.append(Joiner.on(",").join(container.getCommands())); + } else { + message.append(" -- Set "); + message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND); + message.append(" to true to reenable command logging"); + } + + LOG.debug(message.toString()); + } // Populate the current queue name in the environment variable. setupQueueNameEnv(container, applicationMasterContext);