YARN-1320. Fixed Distributed Shell application to respect custom log4j properties file. Contributed by Xuan Gong.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1544364 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -170,6 +170,9 @@ Release 2.3.0 - UNRELEASED
     YARN-1053. Diagnostic message from ContainerExitEvent is ignored in
     ContainerImpl (Omkar Vinit Joshi via bikas)
 
+    YARN-1320. Fixed Distributed Shell application to respect custom log4j
+    properties file. (Xuan Gong via vinodkv)
+
 Release 2.2.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java

@@ -220,6 +220,9 @@ public class ApplicationMaster {
   // Hardcoded path to shell script in launch container's local env
   private final String ExecShellStringPath = "ExecShellScript.sh";
 
+  // Hardcoded path to custom log_properties
+  private final String log4jPath = "log4j.properties";
+
   private final String shellCommandPath = "shellCommands";
 
   private volatile boolean done;
@@ -327,6 +330,16 @@ public class ApplicationMaster {
           "No args specified for application master to initialize");
     }
 
+    //Check whether customer log4j.properties file exists
+    File customerLog4jFile = new File(log4jPath);
+    if (customerLog4jFile.exists()) {
+      try {
+        Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
+      } catch (Exception e) {
+        LOG.warn("Can not set up custom log4j properties. " + e);
+      }
+    }
+
     if (cliParser.hasOption("help")) {
       printUsage(opts);
       return false;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java

@@ -166,6 +166,9 @@ public class Client {
   private Options opts;
 
   private final String shellCommandPath = "shellCommands";
+  // Hardcoded path to custom log_properties
+  private final String log4jPath = "log4j.properties";
+
   /**
    * @param args Command line arguments 
    */
@@ -259,6 +262,15 @@ public class Client {
       throw new IllegalArgumentException("No args specified for client to initialize");
     }
 
+    if (cliParser.hasOption("log_properties")) {
+      String log4jPath = cliParser.getOptionValue("log_properties");
+      try {
+        Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath);
+      } catch (Exception e) {
+        LOG.warn("Can not set up custom log4j properties. " + e);
+      }
+    }
+
     if (cliParser.hasOption("help")) {
       printUsage();
       return false;
@@ -455,16 +467,16 @@ public class Client {
     // Set the log4j properties if needed 
     if (!log4jPropFile.isEmpty()) {
       Path log4jSrc = new Path(log4jPropFile);
-      Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
+      String log4jPathSuffix = appName + "/" + appId.getId() + "/" + log4jPath;
+      Path log4jDst = new Path(fs.getHomeDirectory(), log4jPathSuffix);
       fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
       FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
-      LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
+      LocalResource log4jRsrc =
-      log4jRsrc.setType(LocalResourceType.FILE);
+          LocalResource.newInstance(
-      log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);	   
+              ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()),
-      log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
+              LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
-      log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
+              log4jFileStatus.getLen(), log4jFileStatus.getModificationTime());
-      log4jRsrc.setSize(log4jFileStatus.getLen());
+      localResources.put(log4jPath, log4jRsrc);
-      localResources.put("log4j.properties", log4jRsrc);
     }			
 
     // The shell script has to be made available on the final container(s)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.applications.distributedshell;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.PropertyConfigurator;
+
+
+public class Log4jPropertyHelper {
+
+  public static void updateLog4jConfiguration(Class<?> targetClass,
+      String log4jPath) throws Exception {
+    Properties customProperties = new Properties();
+    FileInputStream fs = null;
+    InputStream is = null;
+    try {
+      fs = new FileInputStream(log4jPath);
+      is = targetClass.getResourceAsStream("/log4j.properties");
+      customProperties.load(fs);
+      Properties originalProperties = new Properties();
+      originalProperties.load(is);
+      for (Entry<Object, Object> entry : customProperties.entrySet()) {
+        originalProperties.setProperty(entry.getKey().toString(), entry
+            .getValue().toString());
+      }
+      LogManager.resetConfiguration();
+      PropertyConfigurator.configure(originalProperties);
+    }finally {
+      IOUtils.closeQuietly(is);
+      IOUtils.closeQuietly(fs);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java

@@ -25,6 +25,7 @@ import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintWriter;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -174,6 +175,68 @@ public class TestDistributedShell {
   }
 
   @Test(timeout=90000)
+  public void testDSShellWithCustomLogPropertyFile() throws Exception {
+    final File basedir =
+        new File("target", TestDistributedShell.class.getName());
+    final File tmpDir = new File(basedir, "tmpDir");
+    tmpDir.mkdirs();
+    final File customLogProperty = new File(tmpDir, "custom_log4j.properties");
+    if (customLogProperty.exists()) {
+      customLogProperty.delete();
+    }
+    if(!customLogProperty.createNewFile()) {
+      Assert.fail("Can not create custom log4j property file.");
+    }
+    PrintWriter fileWriter = new PrintWriter(customLogProperty);
+    // set the output to DEBUG level
+    fileWriter.write("log4j.rootLogger=debug,stdout");
+    fileWriter.close();
+    String[] args = {
+        "--jar",
+        APPMASTER_JAR,
+        "--num_containers",
+        "3",
+        "--shell_command",
+        "echo",
+        "--shell_args",
+        "HADOOP",
+        "--log_properties",
+        customLogProperty.getAbsolutePath(),
+        "--master_memory",
+        "512",
+        "--master_vcores",
+        "2",
+        "--container_memory",
+        "128",
+        "--container_vcores",
+        "1"
+    };
+
+    //Before run the DS, the default the log level is INFO
+    final Log LOG_Client =
+        LogFactory.getLog(Client.class);
+    Assert.assertTrue(LOG_Client.isInfoEnabled());
+    Assert.assertFalse(LOG_Client.isDebugEnabled());
+    final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class);
+    Assert.assertTrue(LOG_AM.isInfoEnabled());
+    Assert.assertFalse(LOG_AM.isDebugEnabled());
+
+    LOG.info("Initializing DS Client");
+    final Client client =
+        new Client(new Configuration(yarnCluster.getConfig()));
+    boolean initSuccess = client.init(args);
+    Assert.assertTrue(initSuccess);
+    LOG.info("Running DS Client");
+    boolean result = client.run();
+    LOG.info("Client run completed. Result=" + result);
+    Assert.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10);
+    //After DS is finished, the log level should be DEBUG
+    Assert.assertTrue(LOG_Client.isInfoEnabled());
+    Assert.assertTrue(LOG_Client.isDebugEnabled());
+    Assert.assertTrue(LOG_AM.isInfoEnabled());
+    Assert.assertTrue(LOG_AM.isDebugEnabled());
+  }
+
   public void testDSShellWithCommands() throws Exception {
 
     String[] args = {