From 42ce052585dcb3ff5e954c41409c2a75d507bbb2 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 27 May 2015 16:51:48 -0700 Subject: [PATCH] YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong. (cherry picked from commit 54504133f41e36eaea6bb06c7b9ddb249468ecd7) (cherry picked from commit 839f81a6326b2f8b3d5183178382c1551b0bc259) (cherry picked from commit 058380d9ef35f35e8c624fb8783eac0904c4d1f5) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 9 ++++++ .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../src/main/resources/yarn-default.xml | 9 ++++++ .../ApplicationHistoryClientService.java | 2 +- .../ApplicationHistoryManager.java | 12 ++++--- .../ApplicationHistoryManagerImpl.java | 2 +- ...licationHistoryManagerOnTimelineStore.java | 13 +++++--- .../TestApplicationHistoryClientService.java | 26 ++++++++++++++-- ...licationHistoryManagerOnTimelineStore.java | 12 ++++--- .../hadoop/yarn/server/webapp/AppsBlock.java | 6 ++++ .../yarn/server/webapp/WebServices.java | 31 +++++++------------ .../src/site/apt/TimelineServer.apt.vm | 8 +++++ 13 files changed, 97 insertions(+), 37 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e2419d6a440..328a0edd152 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -144,6 +144,9 @@ Release 2.6.1 - UNRELEASED YARN-2766. Made ApplicationHistoryManager return a sorted list of apps, attempts and containers. (Robert Kanter via zjshen) + YARN-3700. Made generic history service load a number of latest applications + according to the parameter or the configuration. (Xuan Gong via zjshen) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6d9c0dad3ad..bc89ed095b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1285,6 +1285,15 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + /** + * Defines the max number of applications could be fetched using + * REST API or application history protocol and shown in timeline + * server web ui. + */ + public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS = + APPLICATION_HISTORY_PREFIX + "max-applications"; + public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 055f895a904..74b2415489f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -33,6 +33,7 @@ public interface YarnWebParams { String ENTITY_STRING = "entity.string"; String APP_OWNER = "app.owner"; String APP_STATE = "app.state"; + String APPS_NUM = "apps.num"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; String WEB_UI_TYPE = "web.ui.type"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9d37196bde6..e1686002a89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1268,6 +1268,15 @@ + + + Defines the max number of applications could be fetched using REST API or + application history protocol and shown in timeline server web ui. + + yarn.timeline-service.generic-application-history.max-applications + 10000 + + Store class name for timeline store. yarn.timeline-service.store-class diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 848224d39df..632853e9cf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -178,7 +178,7 @@ public class ApplicationHistoryClientService extends AbstractService implements IOException { GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList( - history.getAllApplications().values())); + history.getApplications(request.getLimit()).values())); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java index 041c31bc376..c218eec0000 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java @@ -51,16 +51,20 @@ public interface ApplicationHistoryManager { IOException; /** - * This method returns all Application {@link ApplicationReport}s - * + * This method returns the given number of Application + * {@link ApplicationReport}s. + * + * @param appsNum + * * @return map of {@link ApplicationId} to {@link ApplicationReport}s. * @throws YarnException * @throws IOException */ @Public @Unstable - Map getAllApplications() - throws YarnException, IOException; + Map + getApplications(long appsNum) throws YarnException, + IOException; /** * Application can have multiple application attempts diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 803dc01d1d2..779b182bdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements } @Override - public Map getAllApplications() + public Map getApplications(long appsNum) throws IOException { Map histData = historyStore.getAllApplications(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index f9ea14f0b64..56ad4696ffd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -73,6 +74,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService private TimelineDataManager timelineDataManager; private ApplicationACLsManager aclsManager; private String serverHttpAddress; + private long maxLoadedApplications; public ApplicationHistoryManagerOnTimelineStore( TimelineDataManager timelineDataManager, @@ -86,6 +88,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService protected void serviceInit(Configuration conf) throws Exception { serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + maxLoadedApplications = + conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS); super.serviceInit(conf); } @@ -96,12 +101,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService } @Override - public Map getAllApplications() + public Map getApplications(long appsNum) throws YarnException, IOException { TimelineEntities entities = timelineDataManager.getEntities( - ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, - null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), - UserGroupInformation.getLoginUser()); + ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null, + null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, + EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); Map apps = new LinkedHashMap(); if (entities != null && entities.getEntities() != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index b3fb3de4332..3a6214f5f5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -53,6 +53,7 @@ import org.junit.Test; public class TestApplicationHistoryClientService { private static ApplicationHistoryClientService clientService; + private static TimelineDataManager dataManager; @BeforeClass public static void setup() throws Exception { @@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService { TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.createStore(2); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); - TimelineDataManager dataManager = + dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = @@ -97,8 +98,27 @@ public class TestApplicationHistoryClientService { clientService.getApplications(request); List appReport = response.getApplicationList(); Assert.assertNotNull(appReport); - Assert.assertEquals(appId, appReport.get(0).getApplicationId()); - Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); + Assert.assertEquals(appId, appReport.get(1).getApplicationId()); + Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); + + // Create a historyManager, and set the max_apps can be loaded + // as 1. + Configuration conf = new YarnConfiguration(); + conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1); + ApplicationHistoryManagerOnTimelineStore historyManager2 = + new ApplicationHistoryManagerOnTimelineStore(dataManager, + new ApplicationACLsManager(conf)); + historyManager2.init(conf); + historyManager2.start(); + @SuppressWarnings("resource") + ApplicationHistoryClientService clientService2 = + new ApplicationHistoryClientService(historyManager2); + response = clientService2.getApplications(request); + appReport = response.getApplicationList(); + Assert.assertNotNull(appReport); + Assert.assertTrue(appReport.size() == 1); + // Expected to get the appReport for application with appId1 + Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index ec96e0c85a0..c44f22ed491 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -175,8 +175,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { Assert.assertEquals("test app type", app.getApplicationType()); Assert.assertEquals("user1", app.getUser()); Assert.assertEquals("test queue", app.getQueue()); - Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime()); - Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime()); + Assert.assertEquals(Integer.MAX_VALUE + 2L + + app.getApplicationId().getId(), app.getStartTime()); + Assert.assertEquals(Integer.MAX_VALUE + 3L + + +app.getApplicationId().getId(), app.getFinishTime()); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. @@ -324,7 +326,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { @Test public void testGetApplications() throws Exception { Collection apps = - historyManager.getAllApplications().values(); + historyManager.getApplications(Long.MAX_VALUE).values(); Assert.assertNotNull(apps); Assert.assertEquals(SCALE + 1, apps.size()); } @@ -450,12 +452,12 @@ public class TestApplicationHistoryManagerOnTimelineStore { entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); - tEvent.setTimestamp(Integer.MAX_VALUE + 2L); + tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId()); entity.addEvent(tEvent); tEvent = new TimelineEvent(); tEvent.setEventType( ApplicationMetricsConstants.FINISHED_EVENT_TYPE); - tEvent.setTimestamp(Integer.MAX_VALUE + 3L); + tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId()); Map eventInfo = new HashMap(); eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 161486df1b2..583a0ae5441 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; @@ -80,6 +81,11 @@ public class AppsBlock extends HtmlBlock { try { final GetApplicationsRequest request = GetApplicationsRequest.newInstance(reqAppStates); + String appsNumStr = $(APPS_NUM); + if (appsNumStr != null && !appsNumStr.isEmpty()) { + long appsNum = Long.parseLong(appsNumStr); + request.setLimit(appsNum); + } if (callerUGI == null) { appReports = appBaseProt.getApplications(request).getApplicationList(); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index a02b80fb670..0d6f1662dca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -31,6 +31,13 @@ import javax.ws.rs.WebApplicationException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -39,13 +46,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; @@ -71,13 +71,11 @@ public class WebServices { String startedEnd, String finishBegin, String finishEnd, Set applicationTypes) { UserGroupInformation callerUGI = getUser(req); - long num = 0; - boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; - long countNum = 0; + long countNum = Long.MAX_VALUE; // set values suitable in case both of begin/end not specified long sBegin = 0; @@ -86,7 +84,6 @@ public class WebServices { long fEnd = Long.MAX_VALUE; if (count != null && !count.isEmpty()) { - checkCount = true; countNum = Long.parseLong(count); if (countNum <= 0) { throw new BadRequestException("limit value must be greater then 0"); @@ -147,19 +144,20 @@ public class WebServices { AppsInfo allApps = new AppsInfo(); Collection appReports = null; + final GetApplicationsRequest request = + GetApplicationsRequest.newInstance(); + request.setLimit(countNum); try { if (callerUGI == null) { // TODO: the request should take the params like what RMWebServices does // in YARN-1819. - GetApplicationsRequest request = GetApplicationsRequest.newInstance(); appReports = appBaseProt.getApplications(request).getApplicationList(); } else { appReports = callerUGI.doAs( new PrivilegedExceptionAction> () { @Override public Collection run() throws Exception { - return appBaseProt.getApplications( - GetApplicationsRequest.newInstance()).getApplicationList(); + return appBaseProt.getApplications(request).getApplicationList(); } }); } @@ -168,10 +166,6 @@ public class WebServices { } for (ApplicationReport appReport : appReports) { - if (checkCount && num == countNum) { - break; - } - if (checkAppStates && !appStates.contains(appReport.getYarnApplicationState().toString() .toLowerCase())) { @@ -211,7 +205,6 @@ public class WebServices { AppInfo app = new AppInfo(appReport); allApps.add(app); - num++; } return allApps; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index 92c73770996..7332b86baf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -103,6 +103,14 @@ YARN Timeline Server 10 + + The max number of applications could be fetched by using REST API + or application history protocol and shown in timeline server web ui. Defaults + to `10000`. + yarn.timeline-service.generic-application-history.max-applications + 10000 + + Enables cross-origin support (CORS) for web services where cross-origin web response headers are needed. For example, javascript making