diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8a7d2f04bf8..037ea87b5ab 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -152,6 +152,9 @@ Release 2.3.0 - UNRELEASED YARN-1053. Diagnostic message from ContainerExitEvent is ignored in ContainerImpl (Omkar Vinit Joshi via bikas) + YARN-1320. Fixed Distributed Shell application to respect custom log4j + properties file. (Xuan Gong via vinodkv) + Release 2.2.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index ac0c6cd343b..1928003119c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -220,6 +220,9 @@ public class ApplicationMaster { // Hardcoded path to shell script in launch container's local env private final String ExecShellStringPath = "ExecShellScript.sh"; + // Hardcoded path to custom log_properties + private final String log4jPath = "log4j.properties"; + private final String shellCommandPath = "shellCommands"; private volatile boolean done; @@ -327,6 +330,16 @@ public class ApplicationMaster { "No args specified for application master to initialize"); } + //Check whether customer log4j.properties file exists + File customerLog4jFile = new File(log4jPath); + if (customerLog4jFile.exists()) { + try { + Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } + if (cliParser.hasOption("help")) { printUsage(opts); return false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index d1ac2fea3ae..067db0c9da1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -166,6 +166,9 @@ public class Client { private Options opts; private final String shellCommandPath = "shellCommands"; + // Hardcoded path to custom log_properties + private final String log4jPath = "log4j.properties"; + /** * @param args Command line arguments */ @@ -257,7 +260,16 @@ public class Client { if (args.length == 0) { throw new IllegalArgumentException("No args specified for client to initialize"); - } + } + + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } if (cliParser.hasOption("help")) { printUsage(); @@ -455,16 +467,16 @@ public class Client { // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); - Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); + String log4jPathSuffix = appName + "/" + appId.getId() + "/" + log4jPath; + Path log4jDst = new Path(fs.getHomeDirectory(), log4jPathSuffix); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); - LocalResource log4jRsrc = Records.newRecord(LocalResource.class); - log4jRsrc.setType(LocalResourceType.FILE); - log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION); - log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); - log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); - log4jRsrc.setSize(log4jFileStatus.getLen()); - localResources.put("log4j.properties", log4jRsrc); + LocalResource log4jRsrc = + LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()), + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, + log4jFileStatus.getLen(), log4jFileStatus.getModificationTime()); + localResources.put(log4jPath, log4jRsrc); } // The shell script has to be made available on the final container(s) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java new file mode 100644 index 00000000000..cbfe16c6d43 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.applications.distributedshell; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.LogManager; +import org.apache.log4j.PropertyConfigurator; + + +public class Log4jPropertyHelper { + + public static void updateLog4jConfiguration(Class targetClass, + String log4jPath) throws Exception { + Properties customProperties = new Properties(); + FileInputStream fs = null; + InputStream is = null; + try { + fs = new FileInputStream(log4jPath); + is = targetClass.getResourceAsStream("/log4j.properties"); + customProperties.load(fs); + Properties originalProperties = new Properties(); + originalProperties.load(is); + for (Entry entry : customProperties.entrySet()) { + originalProperties.setProperty(entry.getKey().toString(), entry + .getValue().toString()); + } + LogManager.resetConfiguration(); + PropertyConfigurator.configure(originalProperties); + }finally { + IOUtils.closeQuietly(is); + IOUtils.closeQuietly(fs); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index a5bbbb4ec2e..9df580d3f90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -25,6 +25,7 @@ import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintWriter; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -174,6 +175,68 @@ public class TestDistributedShell { } @Test(timeout=90000) + public void testDSShellWithCustomLogPropertyFile() throws Exception { + final File basedir = + new File("target", TestDistributedShell.class.getName()); + final File tmpDir = new File(basedir, "tmpDir"); + tmpDir.mkdirs(); + final File customLogProperty = new File(tmpDir, "custom_log4j.properties"); + if (customLogProperty.exists()) { + customLogProperty.delete(); + } + if(!customLogProperty.createNewFile()) { + Assert.fail("Can not create custom log4j property file."); + } + PrintWriter fileWriter = new PrintWriter(customLogProperty); + // set the output to DEBUG level + fileWriter.write("log4j.rootLogger=debug,stdout"); + fileWriter.close(); + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "3", + "--shell_command", + "echo", + "--shell_args", + "HADOOP", + "--log_properties", + customLogProperty.getAbsolutePath(), + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + + //Before run the DS, the default the log level is INFO + final Log LOG_Client = + LogFactory.getLog(Client.class); + Assert.assertTrue(LOG_Client.isInfoEnabled()); + Assert.assertFalse(LOG_Client.isDebugEnabled()); + final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class); + Assert.assertTrue(LOG_AM.isInfoEnabled()); + Assert.assertFalse(LOG_AM.isDebugEnabled()); + + LOG.info("Initializing DS Client"); + final Client client = + new Client(new Configuration(yarnCluster.getConfig())); + boolean initSuccess = client.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running DS Client"); + boolean result = client.run(); + LOG.info("Client run completed. Result=" + result); + Assert.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10); + //After DS is finished, the log level should be DEBUG + Assert.assertTrue(LOG_Client.isInfoEnabled()); + Assert.assertTrue(LOG_Client.isDebugEnabled()); + Assert.assertTrue(LOG_AM.isInfoEnabled()); + Assert.assertTrue(LOG_AM.isDebugEnabled()); + } + public void testDSShellWithCommands() throws Exception { String[] args = {