YARN-3787. Allowed generic history service to load a number of applications whose started time is within the given range. Contributed by Xuan Gong.

(cherry picked from commit 8d0ef31632)

hadoop-yarn-project/CHANGES.txt

@@ -252,6 +252,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3786. Document yarn class path options.
     (Brahma Reddy Battula via cnauroth)
 
+    YARN-3787. Allowed generic history service to load a number of applications whose
+    started time is within the given range. (Xuan Gong via zjshen)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java

@@ -33,6 +33,8 @@ public interface YarnWebParams {
   String ENTITY_STRING = "entity.string";
   String APP_OWNER = "app.owner";
   String APP_STATE = "app.state";
+  String APP_START_TIME_BEGIN = "app.started-time.begin";
+  String APP_START_TIME_END = "app.started-time.end";
   String APPS_NUM = "apps.num";
   String QUEUE_NAME = "queue.name";
   String NODE_STATE = "node.state";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java

@@ -184,9 +184,16 @@ public class ApplicationHistoryClientService extends AbstractService implements
   public GetApplicationsResponse
       getApplications(GetApplicationsRequest request) throws YarnException,
           IOException {
+    long startedBegin =
+        request.getStartRange() == null ? 0L : request.getStartRange()
+          .getMinimumLong();
+    long startedEnd =
+        request.getStartRange() == null ? Long.MAX_VALUE : request
+          .getStartRange().getMaximumLong();
     GetApplicationsResponse response =
         GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
-          history.getApplications(request.getLimit()).values()));
+          history.getApplications(request.getLimit(), startedBegin, startedEnd)
+            .values()));
     return response;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java

@@ -51,10 +51,14 @@ public interface ApplicationHistoryManager {
       IOException;
 
   /**
-   * This method returns the given number of Application
+   * This method returns the given number of Application in the
+   * given appStartedTime period.
+   *
    * {@link ApplicationReport}s.
    *
    * @param appsNum
+   * @param appStartedTimeBegin
+   * @param appStartedTimeEnd
    *
    * @return map of {@link ApplicationId} to {@link ApplicationReport}s.
    * @throws YarnException
@@ -62,8 +66,8 @@ public interface ApplicationHistoryManager {
    */
   @Public
   @Unstable
-  Map<ApplicationId, ApplicationReport>
+  Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
-      getApplications(long appsNum) throws YarnException,
+      long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
       IOException;
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java

@@ -98,8 +98,8 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   }
 
   @Override
-  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
+  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
-      throws IOException {
+      long appStartedTimeBegin, long appStartedTimeEnd) throws IOException {
     Map<ApplicationId, ApplicationHistoryData> histData =
         historyStore.getAllApplications();
     HashMap<ApplicationId, ApplicationReport> applicationsReport =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -109,11 +109,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   }
 
   @Override
-  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
+  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
-      throws YarnException, IOException {
+      long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
-    TimelineEntities entities = timelineDataManager.getEntities(
+      IOException {
-        ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+    TimelineEntities entities =
-        null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+        timelineDataManager.getEntities(
+          ApplicationMetricsConstants.ENTITY_TYPE, null, null,
+          appStartedTimeBegin, appStartedTimeEnd, null, null,
+          appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
           EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
     Map<ApplicationId, ApplicationReport> apps =
         new LinkedHashMap<ApplicationId, ApplicationReport>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -342,13 +342,21 @@ public class TestApplicationHistoryManagerOnTimelineStore {
   @Test
   public void testGetApplications() throws Exception {
     Collection<ApplicationReport> apps =
-        historyManager.getApplications(Long.MAX_VALUE).values();
+        historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE)
+          .values();
     Assert.assertNotNull(apps);
     Assert.assertEquals(SCALE + 1, apps.size());
     ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
     for (ApplicationReport app : apps) {
       Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
     }
+
+    // Get apps by given appStartedTime period
+    apps =
+        historyManager.getApplications(Long.MAX_VALUE, 2147483653L,
+          Long.MAX_VALUE).values();
+    Assert.assertNotNull(apps);
+    Assert.assertEquals(2, apps.size());
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_BEGIN;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_END;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -30,6 +32,7 @@ import java.util.Collection;
 import java.util.EnumSet;
 
 import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.math.LongRange;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@@ -79,6 +83,32 @@ public class AppsBlock extends HtmlBlock {
       long appsNum = Long.parseLong(appsNumStr);
       request.setLimit(appsNum);
     }
+
+    String appStartedTimeBegainStr = $(APP_START_TIME_BEGIN);
+    long appStartedTimeBegain = 0;
+    if (appStartedTimeBegainStr != null && !appStartedTimeBegainStr.isEmpty()) {
+      appStartedTimeBegain = Long.parseLong(appStartedTimeBegainStr);
+      if (appStartedTimeBegain < 0) {
+        throw new BadRequestException(
+          "app.started-time.begin must be greater than 0");
+      }
+    }
+    String appStartedTimeEndStr = $(APP_START_TIME_END);
+    long appStartedTimeEnd = Long.MAX_VALUE;
+    if (appStartedTimeEndStr != null && !appStartedTimeEndStr.isEmpty()) {
+      appStartedTimeEnd = Long.parseLong(appStartedTimeEndStr);
+      if (appStartedTimeEnd < 0) {
+        throw new BadRequestException(
+          "app.started-time.end must be greater than 0");
+      }
+    }
+    if (appStartedTimeBegain > appStartedTimeEnd) {
+      throw new BadRequestException(
+        "app.started-time.end must be greater than app.started-time.begin");
+    }
+    request.setStartRange(
+        new LongRange(appStartedTimeBegain, appStartedTimeEnd));
+
     if (callerUGI == null) {
       appReports = appBaseProt.getApplications(request).getApplicationList();
     } else {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java

@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.WebApplicationException;
 
+import org.apache.commons.lang.math.LongRange;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.apache.hadoop.util.StringUtils;
@@ -75,7 +76,6 @@ public class WebServices {
       String startedEnd, String finishBegin, String finishEnd,
       Set<String> applicationTypes) {
     UserGroupInformation callerUGI = getUser(req);
-    boolean checkStart = false;
     boolean checkEnd = false;
     boolean checkAppTypes = false;
     boolean checkAppStates = false;
@@ -95,14 +95,12 @@ public class WebServices {
     }
 
     if (startedBegin != null && !startedBegin.isEmpty()) {
-      checkStart = true;
       sBegin = Long.parseLong(startedBegin);
       if (sBegin < 0) {
         throw new BadRequestException("startedTimeBegin must be greater than 0");
       }
     }
     if (startedEnd != null && !startedEnd.isEmpty()) {
-      checkStart = true;
       sEnd = Long.parseLong(startedEnd);
       if (sEnd < 0) {
         throw new BadRequestException("startedTimeEnd must be greater than 0");
@@ -151,6 +149,7 @@ public class WebServices {
     final GetApplicationsRequest request =
         GetApplicationsRequest.newInstance();
     request.setLimit(countNum);
+    request.setStartRange(new LongRange(sBegin, sEnd));
     try {
       if (callerUGI == null) {
         // TODO: the request should take the params like what RMWebServices does
@@ -198,10 +197,6 @@ public class WebServices {
         continue;
       }
 
-      if (checkStart
-          && (appReport.getStartTime() < sBegin || appReport.getStartTime() > sEnd)) {
-        continue;
-      }
       if (checkEnd
           && (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) {
         continue;