From 444acf8ea795e4bc782f1ce3b5ef7a1a47d1d27d Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 19 Sep 2014 11:26:29 -0700 Subject: [PATCH] YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed by Zhijie Shen --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ApplicationHistoryServer.java | 4 +- .../ahs/RMApplicationHistoryWriter.java | 49 ++++++++++--------- .../ahs/TestRMApplicationHistoryWriter.java | 18 +++++++ 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8f717aed7fa..7175b14a26d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -400,6 +400,9 @@ Release 2.6.0 - UNRELEASED YARN-2568. Fixed the potential test failures due to race conditions when RM work-preserving recovery is enabled. (Jian He via zjshen) + YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless + explicitly set. (Zhijie Shen via jianhe) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 204e118c7ab..f52ab07cf45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -182,7 +182,9 @@ public class ApplicationHistoryServer extends CompositeService { // APPLICATION_HISTORY_STORE is neither null nor empty, it means that the // user has enabled it explicitly. if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null || - conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) { + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals( + NullApplicationHistoryStore.class.getName())) { return new ApplicationHistoryManagerOnTimelineStore( timelineDataManager, aclsManager); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index 58d2e3d1ea4..bd328abe7cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -52,6 +52,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import com.google.common.annotations.VisibleForTesting; + /** *

* {@link ResourceManager} uses this class to write the information of @@ -71,8 +73,10 @@ public class RMApplicationHistoryWriter extends CompositeService { .getLog(RMApplicationHistoryWriter.class); private Dispatcher dispatcher; - private ApplicationHistoryWriter writer; - private boolean historyServiceEnabled; + @VisibleForTesting + ApplicationHistoryWriter writer; + @VisibleForTesting + boolean historyServiceEnabled; public RMApplicationHistoryWriter() { super(RMApplicationHistoryWriter.class.getName()); @@ -80,13 +84,18 @@ public class RMApplicationHistoryWriter extends CompositeService { @Override protected synchronized void serviceInit(Configuration conf) throws Exception { - historyServiceEnabled = conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED); + if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals( + NullApplicationHistoryStore.class.getName())) { + historyServiceEnabled = false; + } - // Only create the services when the history service is enabled, preventing - // wasting the system resources. + // Only create the services when the history service is enabled and not + // using the null store, preventing wasting the system resources. if (historyServiceEnabled) { writer = createApplicationHistoryStore(conf); addIfService(writer); @@ -112,25 +121,19 @@ public class RMApplicationHistoryWriter extends CompositeService { protected ApplicationHistoryStore createApplicationHistoryStore( Configuration conf) { - // If the history writer is not enabled, a dummy store will be used to - // write nothing - if (historyServiceEnabled) { - try { - Class storeClass = - conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE, - FileSystemApplicationHistoryStore.class, + try { + Class storeClass = + conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE, + NullApplicationHistoryStore.class, ApplicationHistoryStore.class); - return storeClass.newInstance(); - } catch (Exception e) { - String msg = - "Could not instantiate ApplicationHistoryWriter: " - + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE, - FileSystemApplicationHistoryStore.class.getName()); - LOG.error(msg, e); - throw new YarnRuntimeException(msg, e); - } - } else { - return new NullApplicationHistoryStore(); + return storeClass.newInstance(); + } catch (Exception e) { + String msg = + "Could not instantiate ApplicationHistoryWriter: " + + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE, + NullApplicationHistoryStore.class.getName()); + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index e83a6b9cc48..1c90ad213dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -78,6 +78,8 @@ public class TestRMApplicationHistoryWriter { store = new MemoryApplicationHistoryStore(); Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, + MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); writer = new RMApplicationHistoryWriter() { @Override @@ -174,6 +176,22 @@ public class TestRMApplicationHistoryWriter { return container; } + @Test + public void testDefaultStoreSetup() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + RMApplicationHistoryWriter writer = new RMApplicationHistoryWriter(); + writer.init(conf); + writer.start(); + try { + Assert.assertFalse(writer.historyServiceEnabled); + Assert.assertNull(writer.writer); + } finally { + writer.stop(); + writer.close(); + } + } + @Test public void testWriteApplication() throws Exception { RMApp app = createRMApp(ApplicationId.newInstance(0, 1));