YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.

(cherry picked from commit 54504133f4)
(cherry picked from commit 839f81a632)
(cherry picked from commit 058380d9ef35f35e8c624fb8783eac0904c4d1f5)
This commit is contained in:
Zhijie Shen 2015-05-27 16:51:48 -07:00 committed by Vinod Kumar Vavilapalli
parent 0f33fcd507
commit 42ce052585
13 changed files with 97 additions and 37 deletions

View File

@ -144,6 +144,9 @@ Release 2.6.1 - UNRELEASED
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps, YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
attempts and containers. (Robert Kanter via zjshen) attempts and containers. (Robert Kanter via zjshen)
YARN-3700. Made generic history service load a number of latest applications
according to the parameter or the configuration. (Xuan Gong via zjshen)
Release 2.6.0 - 2014-11-18 Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -1285,6 +1285,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
/**
* Defines the max number of applications could be fetched using
* REST API or application history protocol and shown in timeline
* server web ui.
*/
public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
APPLICATION_HISTORY_PREFIX + "max-applications";
public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
/** Timeline service store class */ /** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE = public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class"; TIMELINE_SERVICE_PREFIX + "store-class";

View File

@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string"; String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner"; String APP_OWNER = "app.owner";
String APP_STATE = "app.state"; String APP_STATE = "app.state";
String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name"; String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state"; String NODE_STATE = "node.state";
String WEB_UI_TYPE = "web.ui.type"; String WEB_UI_TYPE = "web.ui.type";

View File

@ -1268,6 +1268,15 @@
<value></value> <value></value>
</property> </property>
<property>
<description>
Defines the max number of applications could be fetched using REST API or
application history protocol and shown in timeline server web ui.
</description>
<name>yarn.timeline-service.generic-application-history.max-applications</name>
<value>10000</value>
</property>
<property> <property>
<description>Store class name for timeline store.</description> <description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name> <name>yarn.timeline-service.store-class</name>

View File

@ -178,7 +178,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException { IOException {
GetApplicationsResponse response = GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>( GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
history.getAllApplications().values())); history.getApplications(request.getLimit()).values()));
return response; return response;
} }

View File

@ -51,7 +51,10 @@ public interface ApplicationHistoryManager {
IOException; IOException;
/** /**
* This method returns all Application {@link ApplicationReport}s * This method returns the given number of Application
* {@link ApplicationReport}s.
*
* @param appsNum
* *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s. * @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException * @throws YarnException
@ -59,8 +62,9 @@ public interface ApplicationHistoryManager {
*/ */
@Public @Public
@Unstable @Unstable
Map<ApplicationId, ApplicationReport> getAllApplications() Map<ApplicationId, ApplicationReport>
throws YarnException, IOException; getApplications(long appsNum) throws YarnException,
IOException;
/** /**
* Application can have multiple application attempts * Application can have multiple application attempts

View File

@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
} }
@Override @Override
public Map<ApplicationId, ApplicationReport> getAllApplications() public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException { throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData = Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications(); historyStore.getAllApplications();

View File

@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@ -73,6 +74,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager; private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager; private ApplicationACLsManager aclsManager;
private String serverHttpAddress; private String serverHttpAddress;
private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore( public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager, TimelineDataManager timelineDataManager,
@ -86,6 +88,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception { protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf); WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
maxLoadedApplications =
conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf); super.serviceInit(conf);
} }
@ -96,12 +101,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
} }
@Override @Override
public Map<ApplicationId, ApplicationReport> getAllApplications() public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException { throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities( TimelineEntities entities = timelineDataManager.getEntities(
ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
UserGroupInformation.getLoginUser()); EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps = Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>(); new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) { if (entities != null && entities.getEntities() != null) {

View File

@ -53,6 +53,7 @@ import org.junit.Test;
public class TestApplicationHistoryClientService { public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService; private static ApplicationHistoryClientService clientService;
private static TimelineDataManager dataManager;
@BeforeClass @BeforeClass
public static void setup() throws Exception { public static void setup() throws Exception {
@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store = TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(2); TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf); TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
TimelineDataManager dataManager = dataManager =
new TimelineDataManager(store, aclsManager); new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager = ApplicationHistoryManagerOnTimelineStore historyManager =
@ -97,8 +98,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request); clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList(); List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport); Assert.assertNotNull(appReport);
Assert.assertEquals(appId, appReport.get(0).getApplicationId()); Assert.assertEquals(appId, appReport.get(1).getApplicationId());
Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
// Create a historyManager, and set the max_apps can be loaded
// as 1.
Configuration conf = new YarnConfiguration();
conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
ApplicationHistoryManagerOnTimelineStore historyManager2 =
new ApplicationHistoryManagerOnTimelineStore(dataManager,
new ApplicationACLsManager(conf));
historyManager2.init(conf);
historyManager2.start();
@SuppressWarnings("resource")
ApplicationHistoryClientService clientService2 =
new ApplicationHistoryClientService(historyManager2);
response = clientService2.getApplications(request);
appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
Assert.assertTrue(appReport.size() == 1);
// Expected to get the appReport for application with appId1
Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
} }
@Test @Test

View File

@ -175,8 +175,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType()); Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser()); Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue()); Assert.assertEquals("test queue", app.getQueue());
Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime()); Assert.assertEquals(Integer.MAX_VALUE + 2L
Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime()); + app.getApplicationId().getId(), app.getStartTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app. // Nobody except admin and owner has access to the details of the app.
@ -324,7 +326,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test @Test
public void testGetApplications() throws Exception { public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps = Collection<ApplicationReport> apps =
historyManager.getAllApplications().values(); historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps); Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size()); Assert.assertEquals(SCALE + 1, apps.size());
} }
@ -450,12 +452,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo); entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L); tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent); entity.addEvent(tEvent);
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
tEvent.setEventType( tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE); ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 3L); tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>(); Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info"); "test diagnostics info");

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@ -80,6 +81,11 @@ public class AppsBlock extends HtmlBlock {
try { try {
final GetApplicationsRequest request = final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates); GetApplicationsRequest.newInstance(reqAppStates);
String appsNumStr = $(APPS_NUM);
if (appsNumStr != null && !appsNumStr.isEmpty()) {
long appsNum = Long.parseLong(appsNumStr);
request.setLimit(appsNum);
}
if (callerUGI == null) { if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList(); appReports = appBaseProt.getApplications(request).getApplicationList();
} else { } else {

View File

@ -31,6 +31,13 @@ import javax.ws.rs.WebApplicationException;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -39,13 +46,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@ -71,13 +71,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd, String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) { Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req); UserGroupInformation callerUGI = getUser(req);
long num = 0;
boolean checkCount = false;
boolean checkStart = false; boolean checkStart = false;
boolean checkEnd = false; boolean checkEnd = false;
boolean checkAppTypes = false; boolean checkAppTypes = false;
boolean checkAppStates = false; boolean checkAppStates = false;
long countNum = 0; long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified // set values suitable in case both of begin/end not specified
long sBegin = 0; long sBegin = 0;
@ -86,7 +84,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE; long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) { if (count != null && !count.isEmpty()) {
checkCount = true;
countNum = Long.parseLong(count); countNum = Long.parseLong(count);
if (countNum <= 0) { if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0"); throw new BadRequestException("limit value must be greater then 0");
@ -147,19 +144,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo(); AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null; Collection<ApplicationReport> appReports = null;
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance();
request.setLimit(countNum);
try { try {
if (callerUGI == null) { if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does // TODO: the request should take the params like what RMWebServices does
// in YARN-1819. // in YARN-1819.
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList(); appReports = appBaseProt.getApplications(request).getApplicationList();
} else { } else {
appReports = callerUGI.doAs( appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () { new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override @Override
public Collection<ApplicationReport> run() throws Exception { public Collection<ApplicationReport> run() throws Exception {
return appBaseProt.getApplications( return appBaseProt.getApplications(request).getApplicationList();
GetApplicationsRequest.newInstance()).getApplicationList();
} }
}); });
} }
@ -168,10 +166,6 @@ public class WebServices {
} }
for (ApplicationReport appReport : appReports) { for (ApplicationReport appReport : appReports) {
if (checkCount && num == countNum) {
break;
}
if (checkAppStates if (checkAppStates
&& !appStates.contains(appReport.getYarnApplicationState().toString() && !appStates.contains(appReport.getYarnApplicationState().toString()
.toLowerCase())) { .toLowerCase())) {
@ -211,7 +205,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport); AppInfo app = new AppInfo(appReport);
allApps.add(app); allApps.add(app);
num++;
} }
return allApps; return allApps;
} }

View File

@ -103,6 +103,14 @@ YARN Timeline Server
<value>10</value> <value>10</value>
</property> </property>
<property>
<description>The max number of applications could be fetched by using REST API
or application history protocol and shown in timeline server web ui. Defaults
to `10000`.</description>
<name>yarn.timeline-service.generic-application-history.max-applications</name>
<value>10000</value>
</property>
<property> <property>
<description>Enables cross-origin support (CORS) for web services where <description>Enables cross-origin support (CORS) for web services where
cross-origin web response headers are needed. For example, javascript making cross-origin web response headers are needed. For example, javascript making