From e167e585e9dd5c86ae763c257d62fdcc83260200 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sun, 23 Feb 2014 02:04:58 +0000 Subject: [PATCH] YARN-1749. Updated application-history related configs to reflect the latest reality and to be consistently named. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1570948 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 134 ++++++++++-------- .../yarn/client/api/impl/AHSClientImpl.java | 6 +- .../client/api/impl/TimelineClientImpl.java | 8 +- .../yarn/client/api/impl/YarnClientImpl.java | 4 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 8 +- .../src/main/resources/yarn-default.xml | 114 +++++++-------- .../ApplicationHistoryClientService.java | 12 +- .../ApplicationHistoryManagerImpl.java | 3 +- .../ApplicationHistoryServer.java | 4 +- .../FileSystemApplicationHistoryStore.java | 6 +- .../TestApplicationHistoryClientService.java | 2 +- .../TestApplicationHistoryManagerImpl.java | 2 +- ...TestFileSystemApplicationHistoryStore.java | 2 +- .../ahs/RMApplicationHistoryWriter.java | 40 +++--- .../ahs/TestRMApplicationHistoryWriter.java | 2 +- 16 files changed, 180 insertions(+), 170 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8d07df9534e..b40ceeca6fd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -224,6 +224,9 @@ Release 2.4.0 - UNRELEASED YARN-1687. Renamed user-facing records for the timeline-service to be simply named after 'timeline' instead of 'apptimeline'. (Zhijie Shen via vinodkv) + YARN-1749. Updated application-history related configs to reflect the latest + reality and to be consistently named. (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0e89ce83c42..f2722a58499 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -300,11 +300,6 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 10; - /** The implementation class of ApplicationHistoryStore, which is to be used - * by RMApplicationHistoryWriter. */ - public static final String RM_HISTORY_WRITER_CLASS = RM_PREFIX - + "history-writer.class"; - //Delegation token related keys public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; @@ -993,63 +988,6 @@ public class YarnConfiguration extends Configuration { public static final String YARN_APP_CONTAINER_LOG_BACKUPS = YARN_PREFIX + "app.container.log.backups"; - //////////////////////////////// - // AHS Configs - //////////////////////////////// - - public static final String AHS_PREFIX = YARN_PREFIX + "ahs."; - - /** The setting that controls whether history-service is enabled or not.. */ - public static final String YARN_HISTORY_SERVICE_ENABLED = AHS_PREFIX - + "enabled"; - public static final boolean DEFAULT_YARN_HISTORY_SERVICE_ENABLED = false; - - /** URI for FileSystemApplicationHistoryStore */ - public static final String FS_HISTORY_STORE_URI = AHS_PREFIX + "fs-history-store.uri"; - - /** T-file compression types used to compress history data.*/ - public static final String FS_HISTORY_STORE_COMPRESSION_TYPE = AHS_PREFIX + "fs-history-store.compression-type"; - public static final String DEFAULT_FS_HISTORY_STORE_COMPRESSION_TYPE = "none"; - - /** AHS store class */ - public static final String AHS_STORE = AHS_PREFIX + "store.class"; - - /** host:port address for Application History Server API. */ - public static final String AHS_ADDRESS = AHS_PREFIX + "address"; - public static final int DEFAULT_AHS_PORT = 10200; - public static final String DEFAULT_AHS_ADDRESS = "0.0.0.0:" - + DEFAULT_AHS_PORT; - - /** The number of threads to handle client API requests. */ - public static final String AHS_CLIENT_THREAD_COUNT = AHS_PREFIX - + "client.thread-count"; - public static final int DEFAULT_AHS_CLIENT_THREAD_COUNT = 10; - - - /** The address of the AHS web application.*/ - public static final String AHS_WEBAPP_ADDRESS = AHS_PREFIX - + "webapp.address"; - - public static final int DEFAULT_AHS_WEBAPP_PORT = 8188; - public static final String DEFAULT_AHS_WEBAPP_ADDRESS = "0.0.0.0:" - + DEFAULT_AHS_WEBAPP_PORT; - - /** The https address of the AHS web application.*/ - public static final String AHS_WEBAPP_HTTPS_ADDRESS = AHS_PREFIX - + "webapp.https.address"; - - public static final int DEFAULT_AHS_WEBAPP_HTTPS_PORT = 8190; - public static final String DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" - + DEFAULT_AHS_WEBAPP_HTTPS_PORT; - - /**The kerberos principal to be used for spnego filter for AHS.*/ - public static final String AHS_WEBAPP_SPNEGO_USER_NAME_KEY = - AHS_PREFIX + "webapp.spnego-principal"; - - /**The kerberos keytab to be used for spnego filter for AHS.*/ - public static final String AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = - AHS_PREFIX + "webapp.spnego-keytab-file"; - //////////////////////////////// // Timeline Service Configs //////////////////////////////// @@ -1057,6 +995,78 @@ public class YarnConfiguration extends Configuration { public static final String TIMELINE_SERVICE_PREFIX = YARN_PREFIX + "timeline-service."; + + // mark app-history related configs @Private as application history is going + // to be integrated into the timeline service + @Private + public static final String APPLICATION_HISTORY_PREFIX = + TIMELINE_SERVICE_PREFIX + "generic-application-history."; + + /** + * The setting that controls whether application history service is + * enabled or not. + */ + @Private + public static final String APPLICATION_HISTORY_ENABLED = + APPLICATION_HISTORY_PREFIX + "enabled"; + @Private + public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false; + + /** Application history store class */ + @Private + public static final String APPLICATION_HISTORY_STORE = + APPLICATION_HISTORY_PREFIX + "store-class"; + + /** URI for FileSystemApplicationHistoryStore */ + @Private + public static final String FS_APPLICATION_HISTORY_STORE_URI = + APPLICATION_HISTORY_PREFIX + "fs-history-store.uri"; + + /** T-file compression types used to compress history data.*/ + @Private + public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = + APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type"; + @Private + public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = + "none"; + + /** host:port address for timeline service RPC APIs. */ + public static final String TIMELINE_SERVICE_ADDRESS = + TIMELINE_SERVICE_PREFIX + "address"; + public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200; + public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:" + + DEFAULT_TIMELINE_SERVICE_PORT; + + /** The number of threads to handle client RPC API requests. */ + public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT = + TIMELINE_SERVICE_PREFIX + "handler-thread-count"; + public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10; + + + /** The address of the timeline service web application.*/ + public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS = + TIMELINE_SERVICE_PREFIX + "webapp.address"; + + public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188; + public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS = + "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT; + + /** The https address of the timeline service web application.*/ + public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = + TIMELINE_SERVICE_PREFIX + "webapp.https.address"; + + public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190; + public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + + /**The kerberos principal to be used for spnego filter for timeline service.*/ + public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY = + TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal"; + + /**The kerberos keytab to be used for spnego filter for timeline service.*/ + public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = + TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file"; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java index eea2e18559e..b4c76f3ce7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java @@ -63,9 +63,9 @@ public class AHSClientImpl extends AHSClient { } private static InetSocketAddress getAHSAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_PORT); + return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index cb5bdcc27c9..7323e15f00d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -67,13 +67,13 @@ public class TimelineClientImpl extends TimelineClient { if (YarnConfiguration.useHttps(conf)) { resURI = URI .create(JOINER.join("https://", conf.get( - YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS), + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS), RESOURCE_URI_STR)); } else { resURI = URI.create(JOINER.join("http://", conf.get( - YarnConfiguration.AHS_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR)); } super.serviceInit(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index feb3bb7ffa6..51a73538ed9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -114,8 +114,8 @@ public class YarnClientImpl extends YarnClient { YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS); } - if (conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED)) { + if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { historyServiceEnabled = true; historyClient = AHSClientImpl.createAHSClient(); historyClient.init(getConfig()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index a7dce0732c0..60bc66bb97c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -142,11 +142,11 @@ public class WebAppUtils { public static String getAHSWebAppURLWithoutScheme(Configuration conf) { if (YarnConfiguration.useHttps(conf)) { - return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS); + return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS); } else { - return conf.get(YarnConfiguration.AHS_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS); + return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 7e2100876a2..5b3ede7d148 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -572,30 +572,12 @@ org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy - - Indicate to ResourceManager as well as clients whether - history-service is enabled or not. If enabled, ResourceManager starts - recording historical data that ApplicationHistory service can consume. - Similarly, clients can redirect to the history service when applications - finish if this is enabled. - yarn.ahs.enabled - false - - Number of worker threads that write the history data. yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size 10 - - The implementation class of ApplicationHistoryStore, which is - to be used by RMApplicationHistoryWriter. - - yarn.resourcemanager.history-writer.class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - The class to use as the configuration provider. If org.apache.hadoop.yarn.LocalConfigurationProvider is used, @@ -1085,62 +1067,32 @@ - + - The hostname of the AHS. - yarn.ahs.hostname + The hostname of the timeline service web application. + yarn.timeline-service.hostname 0.0.0.0 - The http address of the AHS web application. - yarn.ahs.webapp.address - ${yarn.ahs.hostname}:8188 - - - - The https adddress of the AHS web application. - yarn.ahs.webapp.https.address - ${yarn.ahs.hostname}:8190 - - - - URI pointing to the location of the FileSystem path where - the history will be persisted. This must be supplied when using - org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore - as the value for yarn.resourcemanager.history-writer.store.class - yarn.ahs.fs-history-store.uri - ${hadoop.log.dir}/yarn/system/ahstore - - - - This is default address for the Application History server - to start the RPC server. - yarn.ahs.address + This is default address for the timeline server to start the + RPC server. + yarn.timeline-service.address 0.0.0.0:10200 - CLient thread count to serve the client requests. - yarn.ahs.client.thread-count - 10 - - - - T-file compression types used to compress history data. - yarn.ahs.fs-history-store.compression-type - none - - - - Store class name for history store, defaulting to file - system store - yarn.ahs.store.class - org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + The http address of the timeline service web application. + yarn.timeline-service.webapp.address + ${yarn.timeline-service.hostname}:8188 - + + The https adddress of the timeline service web application. + yarn.timeline-service.webapp.https.address + ${yarn.timeline-service.hostname}:8190 + Store class name for timeline store @@ -1154,6 +1106,44 @@ ${yarn.log.dir}/timeline + + Handler thread count to serve the client RPC requests. + yarn.timeline-service.handler-thread-count + 10 + + + + Indicate to ResourceManager as well as clients whether + history-service is enabled or not. If enabled, ResourceManager starts + recording historical data that ApplicationHistory service can consume. + Similarly, clients can redirect to the history service when applications + finish if this is enabled. + yarn.timeline-service.generic-application-history.enabled + false + + + + URI pointing to the location of the FileSystem path where + the history will be persisted. This must be supplied when using + org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + as the value for yarn.timeline-service.generic-application-history.store-class + yarn.timeline-service.generic-application-history.fs-history-store.uri + ${hadoop.log.dir}/yarn/system/history + + + + T-file compression types used to compress history data. + yarn.timeline-service.generic-application-history.fs-history-store.compression-type + none + + + + Store class name for history store, defaulting to file + system store + yarn.timeline-service.generic-application-history.store-class + org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + + The interval that the yarn client library uses to poll the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 56558dabc0a..e15198b13f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -76,19 +76,19 @@ public class ApplicationHistoryClientService extends AbstractService { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = - conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_PORT); + conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); server = rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler, address, conf, null, conf.getInt( - YarnConfiguration.AHS_CLIENT_THREAD_COUNT, - YarnConfiguration.DEFAULT_AHS_CLIENT_THREAD_COUNT)); + YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT)); server.start(); this.bindAddress = - conf.updateConnectAddr(YarnConfiguration.AHS_ADDRESS, + conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, server.getListenerAddress()); LOG.info("Instantiated ApplicationHistoryClientService at " + this.bindAddress); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 1e13a233629..17a287d21c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -79,7 +79,8 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements protected ApplicationHistoryStore createApplicationHistoryStore( Configuration conf) { return ReflectionUtils.newInstance(conf.getClass( - YarnConfiguration.AHS_STORE, FileSystemApplicationHistoryStore.class, + YarnConfiguration.APPLICATION_HISTORY_STORE, + FileSystemApplicationHistoryStore.class, ApplicationHistoryStore.class), conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 4d7cdf8149f..89e13f487df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -158,9 +158,9 @@ public class ApplicationHistoryServer extends CompositeService { ahsClientService, "ws") .with(getConfig()) .withHttpSpnegoPrincipalKey( - YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY) + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( - YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .at(bindAddress) .start(new AHSWebApp(historyManager, timelineStore)); } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index a321976c5b1..b78475af06e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -113,7 +113,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService @Override public void serviceInit(Configuration conf) throws Exception { Path fsWorkingPath = - new Path(conf.get(YarnConfiguration.FS_HISTORY_STORE_URI)); + new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI)); rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME); try { fs = fsWorkingPath.getFileSystem(conf); @@ -727,8 +727,8 @@ public class FileSystemApplicationHistoryStore extends AbstractService fs.setPermission(historyFile, HISTORY_FILE_UMASK); writer = new TFile.Writer(fsdos, MIN_BLOCK_SIZE, getConfig().get( - YarnConfiguration.FS_HISTORY_STORE_COMPRESSION_TYPE, - YarnConfiguration.DEFAULT_FS_HISTORY_STORE_COMPRESSION_TYPE), null, + YarnConfiguration.FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE, + YarnConfiguration.DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE), null, getConfig()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index ceeb909c9aa..213b55cc7d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -57,7 +57,7 @@ public class TestApplicationHistoryClientService extends public void setup() { historyServer = new ApplicationHistoryServer(); Configuration config = new YarnConfiguration(); - config.setClass(YarnConfiguration.AHS_STORE, + config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); historyServer.init(config); historyServer.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java index bab85c62d92..fec2bf3457e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java @@ -38,7 +38,7 @@ public class TestApplicationHistoryManagerImpl extends @Before public void setup() throws Exception { Configuration config = new Configuration(); - config.setClass(YarnConfiguration.AHS_STORE, + config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl(); applicationHistoryManagerImpl.init(config); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index 960b68f3d43..bc16d366962 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -52,7 +52,7 @@ public class TestFileSystemApplicationHistoryStore extends fs.initialize(new URI("/"), conf); fsWorkingPath = new Path("Test"); fs.delete(fsWorkingPath, true); - conf.set(YarnConfiguration.FS_HISTORY_STORE_URI, fsWorkingPath.toString()); + conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString()); store = new FileSystemApplicationHistoryStore(); store.init(conf); store.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index 7aa8e81096b..13c53ef1d60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData; @@ -81,8 +82,8 @@ public class RMApplicationHistoryWriter extends CompositeService { protected synchronized void serviceInit(Configuration conf) throws Exception { historyServiceEnabled = - conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED); + conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED); writer = createApplicationHistoryStore(conf); addIfService(writer); @@ -112,14 +113,15 @@ public class RMApplicationHistoryWriter extends CompositeService { if (historyServiceEnabled) { try { Class storeClass = - conf.getClass(YarnConfiguration.RM_HISTORY_WRITER_CLASS, - NullApplicationHistoryStore.class, ApplicationHistoryStore.class); + conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE, + FileSystemApplicationHistoryStore.class, + ApplicationHistoryStore.class); return storeClass.newInstance(); } catch (Exception e) { String msg = "Could not instantiate ApplicationHistoryWriter: " - + conf.get(YarnConfiguration.RM_HISTORY_WRITER_CLASS, - NullApplicationHistoryStore.class.getName()); + + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE, + FileSystemApplicationHistoryStore.class.getName()); LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } @@ -214,21 +216,25 @@ public class RMApplicationHistoryWriter extends CompositeService { @SuppressWarnings("unchecked") public void applicationStarted(RMApp app) { - dispatcher.getEventHandler().handle( - new WritingApplicationStartEvent(app.getApplicationId(), - ApplicationStartData.newInstance(app.getApplicationId(), app.getName(), - app.getApplicationType(), app.getQueue(), app.getUser(), - app.getSubmitTime(), app.getStartTime()))); + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( + new WritingApplicationStartEvent(app.getApplicationId(), + ApplicationStartData.newInstance(app.getApplicationId(), app.getName(), + app.getApplicationType(), app.getQueue(), app.getUser(), + app.getSubmitTime(), app.getStartTime()))); + } } @SuppressWarnings("unchecked") public void applicationFinished(RMApp app, RMAppState finalState) { - dispatcher.getEventHandler().handle( - new WritingApplicationFinishEvent(app.getApplicationId(), - ApplicationFinishData.newInstance(app.getApplicationId(), - app.getFinishTime(), app.getDiagnostics().toString(), - app.getFinalApplicationStatus(), - RMServerUtils.createApplicationState(finalState)))); + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( + new WritingApplicationFinishEvent(app.getApplicationId(), + ApplicationFinishData.newInstance(app.getApplicationId(), + app.getFinishTime(), app.getDiagnostics().toString(), + app.getFinalApplicationStatus(), + RMServerUtils.createApplicationState(finalState)))); + } } @SuppressWarnings("unchecked") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index feaab3fe3ed..373cd36d2fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -77,7 +77,7 @@ public class TestRMApplicationHistoryWriter { public void setup() { store = new MemoryApplicationHistoryStore(); Configuration conf = new Configuration(); - conf.setBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, true); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); writer = new RMApplicationHistoryWriter() { @Override