From c5a241f1ddc3f7bca516c6fb1d7be29f2a7a9975 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Mon, 20 Jan 2014 19:29:41 +0000 Subject: [PATCH 01/20] MAPREDUCE-5729. mapred job -list throws NPE (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1559811 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../apache/hadoop/mapreduce/TypeConverter.java | 18 +++++++++++++----- .../hadoop/mapreduce/TestTypeConverter.java | 11 +++++++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d9a02feca67..88b0ff8ec1a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -280,6 +280,8 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5724. JobHistoryServer does not start if HDFS is not running. (tucu) + MAPREDUCE-5729. mapred job -list throws NPE (kasha) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index d62045ba2ca..6b4aa4ed1e4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -43,6 +43,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -445,11 +446,18 @@ public class TypeConverter { jobStatus.setStartTime(application.getStartTime()); jobStatus.setFinishTime(application.getFinishTime()); jobStatus.setFailureInfo(application.getDiagnostics()); - jobStatus.setNeededMem(application.getApplicationResourceUsageReport().getNeededResources().getMemory()); - jobStatus.setNumReservedSlots(application.getApplicationResourceUsageReport().getNumReservedContainers()); - jobStatus.setNumUsedSlots(application.getApplicationResourceUsageReport().getNumUsedContainers()); - jobStatus.setReservedMem(application.getApplicationResourceUsageReport().getReservedResources().getMemory()); - jobStatus.setUsedMem(application.getApplicationResourceUsageReport().getUsedResources().getMemory()); + ApplicationResourceUsageReport resourceUsageReport = + application.getApplicationResourceUsageReport(); + if (resourceUsageReport != null) { + jobStatus.setNeededMem( + resourceUsageReport.getNeededResources().getMemory()); + jobStatus.setNumReservedSlots( + resourceUsageReport.getNumReservedContainers()); + jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers()); + jobStatus.setReservedMem( + resourceUsageReport.getReservedResources().getMemory()); + jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory()); + } return jobStatus; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index ea3324e5023..cc42b9c220f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -23,8 +23,6 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; -import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -40,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Records; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -112,6 +111,14 @@ public class TestTypeConverter { when(mockReport.getUser()).thenReturn("dummy-user"); when(mockReport.getQueue()).thenReturn("dummy-queue"); String jobFile = "dummy-path/job.xml"; + + try { + JobStatus status = TypeConverter.fromYarn(mockReport, jobFile); + } catch (NullPointerException npe) { + Assert.fail("Type converstion from YARN fails for jobs without " + + "ApplicationUsageReport"); + } + ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); From 362e4fc891775ef41e2f253f86aa125495267e4b Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Tue, 21 Jan 2014 01:23:44 +0000 Subject: [PATCH 02/20] MAPREDUCE-5725. Make explicit that TestNetworkedJob relies on the Capacity Scheduler (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1559882 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/TestNetworkedJob.java | 20 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 88b0ff8ec1a..104bcff9fc2 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -203,6 +203,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5672. Provide optional RollingFileAppender for container log4j (syslog) (Gera Shegalov via jlowe) + MAPREDUCE-5725. Make explicit that TestNetworkedJob relies on the Capacity + Scheduler (Sandy Ryza) + OPTIMIZATIONS MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java index e160dba435f..cfe4705ad53 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java @@ -45,7 +45,9 @@ import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.junit.Test; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -76,8 +78,7 @@ public class TestNetworkedJob { FileSystem fileSys = null; try { - mr = MiniMRClientClusterFactory.create(this.getClass(), 2, - new Configuration()); + mr = createMiniClusterWithCapacityScheduler(); JobConf job = new JobConf(mr.getConfig()); @@ -129,8 +130,7 @@ public class TestNetworkedJob { FileSystem fileSys = null; try { - Configuration conf = new Configuration(); - mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf); + mr = createMiniClusterWithCapacityScheduler(); JobConf job = new JobConf(mr.getConfig()); @@ -315,8 +315,7 @@ public class TestNetworkedJob { FileSystem fileSys = null; PrintStream oldOut = System.out; try { - Configuration conf = new Configuration(); - mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf); + mr = createMiniClusterWithCapacityScheduler(); JobConf job = new JobConf(mr.getConfig()); @@ -392,4 +391,13 @@ public class TestNetworkedJob { } } } + + private MiniMRClientCluster createMiniClusterWithCapacityScheduler() + throws IOException { + Configuration conf = new Configuration(); + // Expected queue names depending on Capacity Scheduler queue naming + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + CapacityScheduler.class); + return MiniMRClientClusterFactory.create(this.getClass(), 2, conf); + } } From e0ce554b86fd1ecb9bfd184d15c8e1decea02d5e Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Tue, 21 Jan 2014 02:01:51 +0000 Subject: [PATCH 03/20] YARN-1616. RMFatalEventDispatcher should log the cause of the event (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1559902 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 ++ .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 305a8217cf7..8b9ee189660 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -229,6 +229,8 @@ Release 2.4.0 - UNRELEASED YARN-1567. In Fair Scheduler, allow empty queues to change between leaf and parent on allocation file reload (Sandy Ryza) + YARN-1616. RMFatalEventDispatcher should log the cause of the event (kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 83d7f9df833..a98be13e3bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -607,7 +607,7 @@ public class ResourceManager extends CompositeService implements Recoverable { @Override public void handle(RMFatalEvent event) { LOG.fatal("Received a " + RMFatalEvent.class.getName() + " of type " + - event.getType().name()); + event.getType().name() + ". Cause:\n" + event.getCause()); if (event.getType() == RMFatalEventType.STATE_STORE_FENCED) { LOG.info("RMStateStore has been fenced"); From b2dd9d74c20844a589263a8ac40a736dbb92be7e Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Tue, 21 Jan 2014 05:14:10 +0000 Subject: [PATCH 04/20] HBASE-10143 replace WritableFactories's hashmap with ConcurrentHashMap (Liang Xie via Stack) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1559923 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/io/WritableFactories.java | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 12d9a19b8f6..896723d4a46 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -113,6 +113,9 @@ Trunk (Unreleased) HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley) + HADOOP-10143 replace WritableFactories's hashmap with ConcurrentHashMap + (Liang Xie via stack) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java index 49bd0b914ef..a8fdbfe98df 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java @@ -22,25 +22,26 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.*; import org.apache.hadoop.util.ReflectionUtils; -import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** Factories for non-public writables. Defining a factory permits {@link * ObjectWritable} to be able to construct instances of non-public classes. */ @InterfaceAudience.Public @InterfaceStability.Stable public class WritableFactories { - private static final HashMap CLASS_TO_FACTORY = - new HashMap(); + private static final Map CLASS_TO_FACTORY = + new ConcurrentHashMap(); private WritableFactories() {} // singleton /** Define a factory for a class. */ - public static synchronized void setFactory(Class c, WritableFactory factory) { + public static void setFactory(Class c, WritableFactory factory) { CLASS_TO_FACTORY.put(c, factory); } /** Define a factory for a class. */ - public static synchronized WritableFactory getFactory(Class c) { + public static WritableFactory getFactory(Class c) { return CLASS_TO_FACTORY.get(c); } From d1963ad4bd1fb735bfa39e2b70773bdf1f623a87 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Tue, 21 Jan 2014 19:23:34 +0000 Subject: [PATCH 05/20] MAPREDUCE-5693. Restore MRv1 behavior for log flush. Contributed by Gera Shegalov git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1560148 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../org/apache/hadoop/mapred/YarnChild.java | 10 +-- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 10 +++ .../org/apache/hadoop/mapred/TaskLog.java | 88 ++++++++++++++++++- .../apache/hadoop/mapred/TaskLogAppender.java | 4 +- .../hadoop/yarn/ContainerLogAppender.java | 6 +- 6 files changed, 113 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 104bcff9fc2..f10ba51d523 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -285,6 +285,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5729. mapred job -list throws NPE (kasha) + MAPREDUCE-5693. Restore MRv1 behavior for log flush (Gera Shegalov via + jlowe) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index d8f4f76d845..29e82d71ccc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -27,6 +27,7 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ScheduledExecutorService; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -61,7 +62,6 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.log4j.LogManager; /** * The main() for MapReduce task processes. @@ -123,6 +123,7 @@ class YarnChild { LOG.debug("PID: " + System.getenv().get("JVM_PID")); Task task = null; UserGroupInformation childUGI = null; + ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; @@ -161,6 +162,8 @@ class YarnChild { // set job classloader if configured before invoking the task MRApps.setJobClassLoader(job); + logSyncer = TaskLog.createLogSyncer(); + // Create a final reference to the task for the doAs block final Task taskFinal = task; childUGI.doAs(new PrivilegedExceptionAction() { @@ -214,10 +217,7 @@ class YarnChild { } finally { RPC.stopProxy(umbilical); DefaultMetricsSystem.shutdown(); - // Shutting down log4j of the child-vm... - // This assumes that on return from Task.run() - // there is no more logging done. - LogManager.shutdown(); + TaskLog.syncLogsShutdown(logSyncer); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 6fafdb5aa23..2bae130a4e3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.io.IOUtils; @@ -45,6 +46,7 @@ import org.apache.hadoop.mapred.FileOutputCommitter; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.TaskAttemptListenerImpl; +import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapred.TaskUmbilicalProtocol; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.OutputCommitter; @@ -212,6 +214,7 @@ public class MRAppMaster extends CompositeService { boolean errorHappenedShutDown = false; private String shutDownMessage = null; JobStateInternal forcedState = null; + private final ScheduledExecutorService logSyncer; private long recoveredJobStartTime = 0; @@ -240,6 +243,7 @@ public class MRAppMaster extends CompositeService { this.nmHttpPort = nmHttpPort; this.metrics = MRAppMetrics.create(); this.maxAppAttempts = maxAppAttempts; + logSyncer = TaskLog.createLogSyncer(); LOG.info("Created MRAppMaster for application " + applicationAttemptId); } @@ -1078,6 +1082,12 @@ public class MRAppMaster extends CompositeService { // All components have started, start the job. startJobs(); } + + @Override + public void stop() { + super.stop(); + TaskLog.syncLogsShutdown(logSyncer); + } private void processRecovery() { if (appAttemptID.getAttemptId() == 1) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index d929813b370..48640ab79c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -23,12 +23,17 @@ import java.io.BufferedReader; import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.Flushable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,6 +49,8 @@ import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.util.ProcessTree; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Appender; import org.apache.log4j.LogManager; @@ -262,7 +269,86 @@ public class TaskLog { } writeToIndexFile(logLocation, isCleanup); } - + + public static synchronized void syncLogsShutdown( + ScheduledExecutorService scheduler) + { + // flush standard streams + // + System.out.flush(); + System.err.flush(); + + if (scheduler != null) { + scheduler.shutdownNow(); + } + + // flush & close all appenders + LogManager.shutdown(); + } + + @SuppressWarnings("unchecked") + public static synchronized void syncLogs() { + // flush standard streams + // + System.out.flush(); + System.err.flush(); + + // flush flushable appenders + // + final Logger rootLogger = Logger.getRootLogger(); + flushAppenders(rootLogger); + final Enumeration allLoggers = rootLogger.getLoggerRepository(). + getCurrentLoggers(); + while (allLoggers.hasMoreElements()) { + final Logger l = allLoggers.nextElement(); + flushAppenders(l); + } + } + + @SuppressWarnings("unchecked") + private static void flushAppenders(Logger l) { + final Enumeration allAppenders = l.getAllAppenders(); + while (allAppenders.hasMoreElements()) { + final Appender a = allAppenders.nextElement(); + if (a instanceof Flushable) { + try { + ((Flushable) a).flush(); + } catch (IOException ioe) { + System.err.println(a + ": Failed to flush!" + + StringUtils.stringifyException(ioe)); + } + } + } + } + + public static ScheduledExecutorService createLogSyncer() { + final ScheduledExecutorService scheduler = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + final Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + t.setName("Thread for syncLogs"); + return t; + } + }); + ShutdownHookManager.get().addShutdownHook(new Runnable() { + @Override + public void run() { + TaskLog.syncLogsShutdown(scheduler); + } + }, 50); + scheduler.scheduleWithFixedDelay( + new Runnable() { + @Override + public void run() { + TaskLog.syncLogs(); + } + }, 0L, 5L, TimeUnit.SECONDS); + return scheduler; + } + /** * The filter for userlogs. */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java index 0b79837f62d..2162a2602ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred; +import java.io.Flushable; import java.util.LinkedList; import java.util.Queue; @@ -31,7 +32,7 @@ import org.apache.log4j.spi.LoggingEvent; * */ @InterfaceStability.Unstable -public class TaskLogAppender extends FileAppender { +public class TaskLogAppender extends FileAppender implements Flushable { private String taskId; //taskId should be managed as String rather than TaskID object //so that log4j can configure it from the configuration(log4j.properties). private Integer maxEvents; @@ -92,6 +93,7 @@ public class TaskLogAppender extends FileAppender { } } + @Override public void flush() { if (qw != null) { qw.flush(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java index 23988b0dcd6..95876efa453 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn; import java.io.File; +import java.io.Flushable; import java.util.LinkedList; import java.util.Queue; @@ -33,7 +34,9 @@ import org.apache.log4j.spi.LoggingEvent; */ @Public @Unstable -public class ContainerLogAppender extends FileAppender { +public class ContainerLogAppender extends FileAppender + implements Flushable +{ private String containerLogDir; //so that log4j can configure it from the configuration(log4j.properties). private int maxEvents; @@ -65,6 +68,7 @@ public class ContainerLogAppender extends FileAppender { } } + @Override public void flush() { if (qw != null) { qw.flush(); From cc7cc5a4d9e789d4736ea52c57d156464ce35b29 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 21 Jan 2014 22:55:35 +0000 Subject: [PATCH 06/20] HDFS-5748. Too much information shown in the dfs health page. Contributed by Haohui Mai git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1560215 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../src/main/webapps/hdfs/dfshealth.html | 417 +++++++++--------- .../src/main/webapps/hdfs/dfshealth.js | 172 +++++--- .../src/main/webapps/static/hadoop.css | 20 + 4 files changed, 338 insertions(+), 274 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 65bae76dd7e..18d6ceb8366 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -778,6 +778,9 @@ Release 2.4.0 - UNRELEASED HDFS-5800. Fix a typo in DFSClient.renewLease(). (Kousuke Saruta via szetszwo) + HDFS-5748. Too much information shown in the dfs health page. + (Haohui Mai via brandonli) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 2a6b90601b9..461031a7937 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -23,25 +23,53 @@ Namenode information + + + +
+ +
+
+ +
+
+ +
+
+
+
+
+
+
-
-

Hadoop, 2013.

-
Legacy UI
+
+

Hadoop, 2014.

+
Legacy UI
+
-
-
Snapshot Summary
-
- {#fs.SnapshotStats} - - - - - - - - - - -
Snapshottable directoriesSnapshotted directories
{SnapshottableDirectories}{Snapshots}
- {/fs.SnapshotStats} -
-
-
+ -{#startup} -
-
Startup Progress
-
-

Elapsed Time: {elapsedTime|fmt_time}, Percent Complete: {percentComplete|fmt_percentage}

- - - + + +