YARN-3787. Allowed generic history service to load a number of applications whose started time is within the given range. Contributed by Xuan Gong.
This commit is contained in:
parent
c45784bc90
commit
8d0ef31632
|
@ -300,6 +300,9 @@ Release 2.8.0 - UNRELEASED
|
|||
YARN-3786. Document yarn class path options.
|
||||
(Brahma Reddy Battula via cnauroth)
|
||||
|
||||
YARN-3787. Allowed generic history service to load a number of applications whose
|
||||
started time is within the given range. (Xuan Gong via zjshen)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
YARN-3339. TestDockerContainerExecutor should pull a single image and not
|
||||
|
|
|
@ -33,6 +33,8 @@ public interface YarnWebParams {
|
|||
String ENTITY_STRING = "entity.string";
|
||||
String APP_OWNER = "app.owner";
|
||||
String APP_STATE = "app.state";
|
||||
String APP_START_TIME_BEGIN = "app.started-time.begin";
|
||||
String APP_START_TIME_END = "app.started-time.end";
|
||||
String APPS_NUM = "apps.num";
|
||||
String QUEUE_NAME = "queue.name";
|
||||
String NODE_STATE = "node.state";
|
||||
|
|
|
@ -184,9 +184,16 @@ public class ApplicationHistoryClientService extends AbstractService implements
|
|||
public GetApplicationsResponse
|
||||
getApplications(GetApplicationsRequest request) throws YarnException,
|
||||
IOException {
|
||||
long startedBegin =
|
||||
request.getStartRange() == null ? 0L : request.getStartRange()
|
||||
.getMinimumLong();
|
||||
long startedEnd =
|
||||
request.getStartRange() == null ? Long.MAX_VALUE : request
|
||||
.getStartRange().getMaximumLong();
|
||||
GetApplicationsResponse response =
|
||||
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
|
||||
history.getApplications(request.getLimit()).values()));
|
||||
history.getApplications(request.getLimit(), startedBegin, startedEnd)
|
||||
.values()));
|
||||
return response;
|
||||
}
|
||||
|
||||
|
|
|
@ -51,10 +51,14 @@ public interface ApplicationHistoryManager {
|
|||
IOException;
|
||||
|
||||
/**
|
||||
* This method returns the given number of Application
|
||||
* This method returns the given number of Application in the
|
||||
* given appStartedTime period.
|
||||
*
|
||||
* {@link ApplicationReport}s.
|
||||
*
|
||||
* @param appsNum
|
||||
* @param appStartedTimeBegin
|
||||
* @param appStartedTimeEnd
|
||||
*
|
||||
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
|
||||
* @throws YarnException
|
||||
|
@ -62,9 +66,9 @@ public interface ApplicationHistoryManager {
|
|||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
Map<ApplicationId, ApplicationReport>
|
||||
getApplications(long appsNum) throws YarnException,
|
||||
IOException;
|
||||
Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
|
||||
long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
|
||||
IOException;
|
||||
|
||||
/**
|
||||
* Application can have multiple application attempts
|
||||
|
|
|
@ -98,8 +98,8 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
|
||||
throws IOException {
|
||||
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
|
||||
long appStartedTimeBegin, long appStartedTimeEnd) throws IOException {
|
||||
Map<ApplicationId, ApplicationHistoryData> histData =
|
||||
historyStore.getAllApplications();
|
||||
HashMap<ApplicationId, ApplicationReport> applicationsReport =
|
||||
|
|
|
@ -109,12 +109,15 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
|
||||
throws YarnException, IOException {
|
||||
TimelineEntities entities = timelineDataManager.getEntities(
|
||||
ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
|
||||
null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
|
||||
EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
|
||||
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
|
||||
long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
|
||||
IOException {
|
||||
TimelineEntities entities =
|
||||
timelineDataManager.getEntities(
|
||||
ApplicationMetricsConstants.ENTITY_TYPE, null, null,
|
||||
appStartedTimeBegin, appStartedTimeEnd, null, null,
|
||||
appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
|
||||
EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
|
||||
Map<ApplicationId, ApplicationReport> apps =
|
||||
new LinkedHashMap<ApplicationId, ApplicationReport>();
|
||||
if (entities != null && entities.getEntities() != null) {
|
||||
|
|
|
@ -342,13 +342,21 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
@Test
|
||||
public void testGetApplications() throws Exception {
|
||||
Collection<ApplicationReport> apps =
|
||||
historyManager.getApplications(Long.MAX_VALUE).values();
|
||||
historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE)
|
||||
.values();
|
||||
Assert.assertNotNull(apps);
|
||||
Assert.assertEquals(SCALE + 1, apps.size());
|
||||
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
|
||||
for (ApplicationReport app : apps) {
|
||||
Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
|
||||
}
|
||||
|
||||
// Get apps by given appStartedTime period
|
||||
apps =
|
||||
historyManager.getApplications(Long.MAX_VALUE, 2147483653L,
|
||||
Long.MAX_VALUE).values();
|
||||
Assert.assertNotNull(apps);
|
||||
Assert.assertEquals(2, apps.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.webapp;
|
|||
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.join;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_BEGIN;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_END;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
|
||||
|
@ -30,6 +32,7 @@ import java.util.Collection;
|
|||
import java.util.EnumSet;
|
||||
|
||||
import org.apache.commons.lang.StringEscapeUtils;
|
||||
import org.apache.commons.lang.math.LongRange;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
|
@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
|||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
|
||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
|
||||
|
@ -79,6 +83,32 @@ public class AppsBlock extends HtmlBlock {
|
|||
long appsNum = Long.parseLong(appsNumStr);
|
||||
request.setLimit(appsNum);
|
||||
}
|
||||
|
||||
String appStartedTimeBegainStr = $(APP_START_TIME_BEGIN);
|
||||
long appStartedTimeBegain = 0;
|
||||
if (appStartedTimeBegainStr != null && !appStartedTimeBegainStr.isEmpty()) {
|
||||
appStartedTimeBegain = Long.parseLong(appStartedTimeBegainStr);
|
||||
if (appStartedTimeBegain < 0) {
|
||||
throw new BadRequestException(
|
||||
"app.started-time.begin must be greater than 0");
|
||||
}
|
||||
}
|
||||
String appStartedTimeEndStr = $(APP_START_TIME_END);
|
||||
long appStartedTimeEnd = Long.MAX_VALUE;
|
||||
if (appStartedTimeEndStr != null && !appStartedTimeEndStr.isEmpty()) {
|
||||
appStartedTimeEnd = Long.parseLong(appStartedTimeEndStr);
|
||||
if (appStartedTimeEnd < 0) {
|
||||
throw new BadRequestException(
|
||||
"app.started-time.end must be greater than 0");
|
||||
}
|
||||
}
|
||||
if (appStartedTimeBegain > appStartedTimeEnd) {
|
||||
throw new BadRequestException(
|
||||
"app.started-time.end must be greater than app.started-time.begin");
|
||||
}
|
||||
request.setStartRange(
|
||||
new LongRange(appStartedTimeBegain, appStartedTimeEnd));
|
||||
|
||||
if (callerUGI == null) {
|
||||
appReports = appBaseProt.getApplications(request).getApplicationList();
|
||||
} else {
|
||||
|
|
|
@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletRequest;
|
|||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.ws.rs.WebApplicationException;
|
||||
|
||||
import org.apache.commons.lang.math.LongRange;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AuthorizationException;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
@ -75,7 +76,6 @@ public class WebServices {
|
|||
String startedEnd, String finishBegin, String finishEnd,
|
||||
Set<String> applicationTypes) {
|
||||
UserGroupInformation callerUGI = getUser(req);
|
||||
boolean checkStart = false;
|
||||
boolean checkEnd = false;
|
||||
boolean checkAppTypes = false;
|
||||
boolean checkAppStates = false;
|
||||
|
@ -95,14 +95,12 @@ public class WebServices {
|
|||
}
|
||||
|
||||
if (startedBegin != null && !startedBegin.isEmpty()) {
|
||||
checkStart = true;
|
||||
sBegin = Long.parseLong(startedBegin);
|
||||
if (sBegin < 0) {
|
||||
throw new BadRequestException("startedTimeBegin must be greater than 0");
|
||||
}
|
||||
}
|
||||
if (startedEnd != null && !startedEnd.isEmpty()) {
|
||||
checkStart = true;
|
||||
sEnd = Long.parseLong(startedEnd);
|
||||
if (sEnd < 0) {
|
||||
throw new BadRequestException("startedTimeEnd must be greater than 0");
|
||||
|
@ -151,6 +149,7 @@ public class WebServices {
|
|||
final GetApplicationsRequest request =
|
||||
GetApplicationsRequest.newInstance();
|
||||
request.setLimit(countNum);
|
||||
request.setStartRange(new LongRange(sBegin, sEnd));
|
||||
try {
|
||||
if (callerUGI == null) {
|
||||
// TODO: the request should take the params like what RMWebServices does
|
||||
|
@ -198,10 +197,6 @@ public class WebServices {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (checkStart
|
||||
&& (appReport.getStartTime() < sBegin || appReport.getStartTime() > sEnd)) {
|
||||
continue;
|
||||
}
|
||||
if (checkEnd
|
||||
&& (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) {
|
||||
continue;
|
||||
|
|
Loading…
Reference in New Issue