From 6bbfa158d75af56d979d7f9685e5ea8cce3a336e Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 31 Oct 2017 16:36:02 +0900 Subject: [PATCH] YARN-7407. Moving logging APIs over to slf4j in hadoop-yarn-applications. Contributed by Yeliang Cang. (cherry picked from commit 785f1b0d11a3bf0af9851c080ff0acc34539f17b) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java --- .../distributedshell/ApplicationMaster.java | 13 +++++++------ .../yarn/applications/distributedshell/Client.java | 9 ++++----- .../ContainerLaunchFailAppMaster.java | 10 +++++----- .../distributedshell/TestDSFailedAppMaster.java | 7 ++++--- .../distributedshell/TestDSSleepingAppMaster.java | 8 +++++--- .../distributedshell/TestDistributedShell.java | 14 +++++++------- .../TestDistributedShellWithNodeLabels.java | 8 ++++---- .../unmanagedamlauncher/UnmanagedAMLauncher.java | 9 +++++---- .../TestUnmanagedAMLauncher.java | 12 ++++++------ 9 files changed, 47 insertions(+), 43 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 5ec9409096e..94fade91bd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -47,8 +47,6 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; @@ -111,6 +109,8 @@ import org.apache.log4j.LogManager; import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.api.client.ClientHandlerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * An ApplicationMaster for executing shell commands on a set of launched @@ -177,7 +177,8 @@ import com.sun.jersey.api.client.ClientHandlerException; @InterfaceStability.Unstable public class ApplicationMaster { - private static final Log LOG = LogFactory.getLog(ApplicationMaster.class); + private static final Logger LOG = LoggerFactory + .getLogger(ApplicationMaster.class); @VisibleForTesting @Private @@ -341,7 +342,7 @@ public class ApplicationMaster { appMaster.run(); result = appMaster.finish(); } catch (Throwable t) { - LOG.fatal("Error running ApplicationMaster", t); + LOG.error("Error running ApplicationMaster", t); LogManager.shutdown(); ExitUtil.terminate(1, t); } @@ -380,7 +381,7 @@ public class ApplicationMaster { } catch (IOException e) { e.printStackTrace(); } finally { - IOUtils.cleanup(LOG, buf); + IOUtils.cleanupWithLogger(LOG, buf); } } @@ -618,7 +619,7 @@ public class ApplicationMaster { LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token token = iter.next(); - LOG.info(token); + LOG.info(token.toString()); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index eedb5016e4f..d96f19feeb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -36,8 +36,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -80,8 +78,9 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Client for Distributed Shell application submission to YARN. @@ -118,7 +117,7 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @InterfaceStability.Unstable public class Client { - private static final Log LOG = LogFactory.getLog(Client.class); + private static final Logger LOG = LoggerFactory.getLogger(Client.class); // Configuration private Configuration conf; @@ -224,7 +223,7 @@ public class Client { } result = client.run(); } catch (Throwable t) { - LOG.fatal("Error running Client", t); + LOG.error("Error running Client", t); System.exit(1); } if (result) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java index 8e561c67362..1d41c0d4e56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java @@ -21,14 +21,14 @@ package org.apache.hadoop.yarn.applications.distributedshell; import java.nio.ByteBuffer; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ContainerLaunchFailAppMaster extends ApplicationMaster { - private static final Log LOG = - LogFactory.getLog(ContainerLaunchFailAppMaster.class); + private static final Logger LOG = + LoggerFactory.getLogger(ContainerLaunchFailAppMaster.class); public ContainerLaunchFailAppMaster() { super(); @@ -69,7 +69,7 @@ public class ContainerLaunchFailAppMaster extends ApplicationMaster { appMaster.run(); result = appMaster.finish(); } catch (Throwable t) { - LOG.fatal("Error running ApplicationMaster", t); + LOG.error("Error running ApplicationMaster", t); System.exit(1); } if (result) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java index 26022d427cc..4f424f930b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java @@ -20,13 +20,14 @@ package org.apache.hadoop.yarn.applications.distributedshell; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestDSFailedAppMaster extends ApplicationMaster { - private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestDSFailedAppMaster.class); @Override public void run() throws YarnException, IOException, InterruptedException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSSleepingAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSSleepingAppMaster.java index 3004b6954e3..25975bf8daf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSSleepingAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSSleepingAppMaster.java @@ -18,12 +18,14 @@ package org.apache.hadoop.yarn.applications.distributedshell; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestDSSleepingAppMaster extends ApplicationMaster{ - private static final Log LOG = LogFactory.getLog(TestDSSleepingAppMaster.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestDSSleepingAppMaster.class); private static final long SLEEP_TIME = 5000; public static void main(String[] args) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 8347f832560..b1d645d0763 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -39,8 +39,6 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; @@ -96,11 +94,13 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestDistributedShell { - private static final Log LOG = - LogFactory.getLog(TestDistributedShell.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestDistributedShell.class); protected MiniYARNCluster yarnCluster = null; protected MiniDFSCluster hdfsCluster = null; @@ -892,11 +892,11 @@ public class TestDistributedShell { }; //Before run the DS, the default the log level is INFO - final Log LOG_Client = - LogFactory.getLog(Client.class); + final Logger LOG_Client = + LoggerFactory.getLogger(Client.class); Assert.assertTrue(LOG_Client.isInfoEnabled()); Assert.assertFalse(LOG_Client.isDebugEnabled()); - final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class); + final Logger LOG_AM = LoggerFactory.getLogger(ApplicationMaster.class); Assert.assertTrue(LOG_AM.isInfoEnabled()); Assert.assertFalse(LOG_AM.isDebugEnabled()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java index 143e5c4e472..e9ba6e8731e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -34,10 +32,12 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestDistributedShellWithNodeLabels { - private static final Log LOG = - LogFactory.getLog(TestDistributedShellWithNodeLabels.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestDistributedShellWithNodeLabels.class); static final int NUM_NMS = 2; TestDistributedShell distShellTest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index 030a730884c..f3bb5fde3a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -36,8 +36,6 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.security.Credentials; @@ -60,6 +58,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.Records; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The UnmanagedLauncher is a simple client that launches and unmanaged AM. An @@ -75,7 +75,8 @@ import org.apache.hadoop.yarn.util.Records; * report app completion. */ public class UnmanagedAMLauncher { - private static final Log LOG = LogFactory.getLog(UnmanagedAMLauncher.class); + private static final Logger LOG = LoggerFactory + .getLogger(UnmanagedAMLauncher.class); private Configuration conf; @@ -110,7 +111,7 @@ public class UnmanagedAMLauncher { } client.run(); } catch (Throwable t) { - LOG.fatal("Error running Client", t); + LOG.error("Error running Client", t); System.exit(1); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java index 3aba01a0efe..134cba80ddc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java @@ -28,8 +28,6 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URL; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -47,10 +45,12 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestUnmanagedAMLauncher { - private static final Log LOG = LogFactory - .getLog(TestUnmanagedAMLauncher.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestUnmanagedAMLauncher.class); protected static MiniYARNCluster yarnCluster = null; protected static Configuration conf = new YarnConfiguration(); @@ -128,7 +128,7 @@ public class TestUnmanagedAMLauncher { String classpath = getTestRuntimeClasspath(); String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { - LOG.fatal("JAVA_HOME not defined. Test not running."); + LOG.error("JAVA_HOME not defined. Test not running."); return; } String[] args = { @@ -170,7 +170,7 @@ public class TestUnmanagedAMLauncher { String classpath = getTestRuntimeClasspath(); String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { - LOG.fatal("JAVA_HOME not defined. Test not running."); + LOG.error("JAVA_HOME not defined. Test not running."); return; } String[] args = {