Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10035. Add ability to filter the Cluster Applications API request by name. Contributed by Adam Antal

This commit is contained in:
Szilard Nemeth 2020-01-06 16:26:33 +01:00
parent 4a76ab777f
commit 768ee22e9e
21 changed files with 95 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
View File

@ -393,4 +393,22 @@ public abstract class GetApplicationsRequest {
@Private
@Unstable
public abstract void setScope(ApplicationsRequestScope scope);
/**
* Set the name to filter applications.
*
* @return the name
*/
@Private
@Unstable
public abstract String getName();
/**
* Get the name to filter applications.
*
* @param name of the application
*/
@Private
@Unstable
public abstract void setName(String name);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
View File

@ -206,6 +206,7 @@ message GetApplicationsRequestProto {
optional int64 finish_end = 9;
repeated string applicationTags = 10;
optional ApplicationsRequestScopeProto scope = 11 [default = ALL];
optional string name = 12;
}
message GetApplicationsResponseProto {

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
View File

@ -55,6 +55,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
Range<Long> finish = null;
private Set<String> applicationTags;
private ApplicationsRequestScope scope;
private String name;
public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder();
@ -121,6 +122,9 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
builder.clearQueues();
builder.addAllQueues(queues);
}
if (name != null) {
builder.setName(name);
}
}
private void maybeInitBuilder() {
@ -370,6 +374,27 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
this.finish = Range.between(begin, end);
}
@Override
public synchronized String getName() {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
if (this.name != null) {
return this.name;
}
if (p.hasName()) {
this.name = p.getName();
}
return this.name;
}
@Override
public synchronized void setName(String name) {
maybeInitBuilder();
if (name == null) {
builder.clearName();
}
this.name = name;
}
@Override
public int hashCode() {
return getProto().hashCode();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
View File

@ -107,7 +107,8 @@ public class AHSWebServices extends WebServices {
public AppsInfo get(@Context HttpServletRequest req,
@Context HttpServletResponse res) {
return getApps(req, res, null, Collections.<String> emptySet(), null, null,
null, null, null, null, null, null, Collections.<String> emptySet());
null, null, null, null, null, null, null,
Collections.<String> emptySet());
}
@GET
@ -126,12 +127,13 @@ public class AHSWebServices extends WebServices {
@QueryParam("startedTimeEnd") String startedEnd,
@QueryParam("finishedTimeBegin") String finishBegin,
@QueryParam("finishedTimeEnd") String finishEnd,
@QueryParam("name") String name,
@QueryParam("applicationTypes") Set<String> applicationTypes) {
initForReadableEndpoints(res);
validateStates(stateQuery, statesQuery);
return super.getApps(req, res, stateQuery, statesQuery, finalStatusQuery,
userQuery, queueQuery, count, startedBegin, startedEnd, finishBegin,
finishEnd, applicationTypes);
finishEnd, name, applicationTypes);
}
@GET

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
View File

@ -75,7 +75,7 @@ public class WebServices {
public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res,
String stateQuery, Set<String> statesQuery, String finalStatusQuery,
String userQuery, String queueQuery, String count, String startedBegin,
String startedEnd, String finishBegin, String finishEnd,
String startedEnd, String finishBegin, String finishEnd, String nameQuery,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
boolean checkEnd = false;
@ -207,6 +207,11 @@ public class WebServices {
&& (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) {
continue;
}
if (nameQuery != null && !nameQuery.equals(appReport.getName())) {
continue;
}
AppInfo app = new AppInfo(appReport);
allApps.add(app);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -865,6 +865,7 @@ public class ClientRMService extends AbstractService implements
Range<Long> start = request.getStartRange();
Range<Long> finish = request.getFinishRange();
ApplicationsRequestScope scope = request.getScope();
String name = request.getName();
final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
Iterator<RMApp> appsIter = apps.values().iterator();
@ -943,6 +944,10 @@ public class ClientRMService extends AbstractService implements
continue;
}
if (name != null && !name.equals(application.getName())) {
continue;
}
reports.add(application.createAndGetApplicationReport(
callerUGI.getUserName(), allowAccess));
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ApplicationsRequestBuilder.java
View File

@ -44,6 +44,7 @@ public class ApplicationsRequestBuilder {
private long finishTimeEnd = Long.MAX_VALUE;
private Set<String> appTypes = Sets.newHashSet();
private Set<String> appTags = Sets.newHashSet();
private String name = null;
private ResourceManager rm;
private ApplicationsRequestBuilder() {
@ -137,6 +138,11 @@ public class ApplicationsRequestBuilder {
return this;
}
public ApplicationsRequestBuilder withName(String applicationName) {
name = applicationName;
return this;
}
private void validate() {
queues.forEach(q -> validateQueueExists(rm, q));
validateLimit();
@ -225,6 +231,9 @@ public class ApplicationsRequestBuilder {
if (!appTags.isEmpty()) {
request.setApplicationTags(appTags);
}
if (name != null) {
request.setName(name);
}
return request;
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
View File

@ -245,6 +245,7 @@ public final class RMWSConsts {
public static final String COMMAND = "command";
public static final String ACTIONS = "actions";
public static final String SUMMARIZE = "summarize";
public static final String NAME = "name";
private RMWSConsts() {
// not called

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
View File

@ -186,6 +186,7 @@ public interface RMWebServiceProtocol {
* @param finishEnd filter the result by finish end time. It is a QueryParam.
* @param applicationTypes filter the result by types. It is a QueryParam.
* @param applicationTags filter the result by tags. It is a QueryParam.
* @param name filter the name of the application. It is a QueryParam.
* @param unselectedFields De-selected params to avoid from report. It is a
* QueryParam.
* @return all apps in the cluster
@ -195,7 +196,7 @@ public interface RMWebServiceProtocol {
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields);
Set<String> applicationTags, String name, Set<String> unselectedFields);
/**
* This method retrieve all the activities in a specific node, and it is

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -610,6 +610,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd,
@QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes,
@QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags,
@QueryParam(RMWSConsts.NAME) String name,
@QueryParam(RMWSConsts.DESELECTS) Set<String> unselectedFields) {
initForReadableEndpoints();
@ -627,6 +628,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
.withFinishTimeEnd(finishEnd)
.withApplicationTypes(applicationTypes)
.withApplicationTags(applicationTags)
.withName(name)
.build();
List<ApplicationReport> appReports;

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -703,12 +703,14 @@ public class TestRMWebServices extends JerseyTestBase {
// verify we don't get any apps when querying
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet, null);
null, null, null, null, null, null, null, emptySet, emptySet, null,
null);
assertTrue(appsInfo.getApps().isEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
null, null, null, null, null, null, null, emptySet, emptySet, null);
null, null, null, null, null, null, null, emptySet, emptySet, null,
null);
assertTrue(appsInfo.getApps().isEmpty());
}
@ -947,7 +949,8 @@ public class TestRMWebServices extends JerseyTestBase {
Set<String> emptySet = Collections.unmodifiableSet(Collections.emptySet());
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet, null);
null, null, null, null, null, null, null, emptySet, emptySet,
null, null);
assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size());
assertEquals("Invalid XML Characters Present",

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java
View File

@ -172,7 +172,7 @@ public class DefaultRequestInterceptorREST
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields) {
Set<String> applicationTags, String name, Set<String> unselectedFields) {
// all the params are specified inside hsr
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppsInfo.class, HTTPMethods.GET,

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
View File

@ -672,7 +672,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields) {
Set<String> applicationTags, String name, Set<String> unselectedFields) {
AppsInfo apps = new AppsInfo();
long startTime = clock.getTime();
@ -701,7 +701,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
AppsInfo rmApps = interceptor.getApps(hsrCopy, stateQuery,
statesQuery, finalStatusQuery, userQuery, queueQuery, count,
startedBegin, startedEnd, finishBegin, finishEnd,
applicationTypes, applicationTags, unselectedFields);
applicationTypes, applicationTags, name, unselectedFields);
if (rmApps == null) {
routerMetrics.incrMultipleAppsFailedRetrieved();

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java
View File

@ -433,13 +433,14 @@ public class RouterWebServices implements RMWebServiceProtocol {
@QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd,
@QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes,
@QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags,
@QueryParam(RMWSConsts.NAME) String name,
@QueryParam(RMWSConsts.DESELECTS) Set<String> unselectedFields) {
init();
RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
return pipeline.getRootInterceptor().getApps(hsr, stateQuery, statesQuery,
finalStatusQuery, userQuery, queueQuery, count, startedBegin,
startedEnd, finishBegin, finishEnd, applicationTypes, applicationTags,
unselectedFields);
name, unselectedFields);
}
@GET

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java
View File

@ -169,7 +169,8 @@ public abstract class BaseRouterWebServicesTest {
protected AppsInfo getApps(String user)
throws IOException, InterruptedException {
return routerWebService.getApps(createHttpServletRequest(user), null, null,
null, null, null, null, null, null, null, null, null, null, null);
null, null, null, null, null, null, null, null, null, null, null,
null);
}
protected ActivitiesInfo getActivities(String user)

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
View File

@ -117,7 +117,7 @@ public class MockDefaultRequestInterceptorREST
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields) {
Set<String> applicationTags, String name, Set<String> unselectedFields) {
if (!isRunning) {
throw new RuntimeException("RM is stopped");
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java
View File

@ -128,7 +128,7 @@ public class MockRESTRequestInterceptor extends AbstractRESTRequestInterceptor {
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields) {
Set<String> applicationTags, String name, Set<String> unselectedFields) {
return new AppsInfo();
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java
View File

@ -153,11 +153,11 @@ public class PassThroughRESTRequestInterceptor
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags, Set<String> unselectedFields) {
Set<String> applicationTags, String name, Set<String> unselectedFields) {
return getNextInterceptor().getApps(hsr, stateQuery, statesQuery,
finalStatusQuery, userQuery, queueQuery, count, startedBegin,
startedEnd, finishBegin, finishEnd, applicationTypes, applicationTags,
unselectedFields);
name, unselectedFields);
}
@Override

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
View File

@ -391,7 +391,7 @@ public class TestFederationInterceptorREST extends BaseRouterWebServicesTest {
throws YarnException, IOException, InterruptedException {
AppsInfo responseGet = interceptor.getApps(null, null, null, null, null,
null, null, null, null, null, null, null, null, null);
null, null, null, null, null, null, null, null, null, null);
Assert.assertNotNull(responseGet);
Assert.assertEquals(NUM_SUBCLUSTER, responseGet.getApps().size());

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java
View File

@ -287,7 +287,7 @@ public class TestFederationInterceptorRESTRetry
setupCluster(Arrays.asList(bad2));
AppsInfo response = interceptor.getApps(null, null, null, null, null, null,
null, null, null, null, null, null, null, null);
null, null, null, null, null, null, null, null, null);
Assert.assertNull(response);
}
@ -301,7 +301,7 @@ public class TestFederationInterceptorRESTRetry
setupCluster(Arrays.asList(bad1, bad2));
AppsInfo response = interceptor.getApps(null, null, null, null, null, null,
null, null, null, null, null, null, null, null);
null, null, null, null, null, null, null, null, null);
Assert.assertNull(response);
}
@ -315,7 +315,7 @@ public class TestFederationInterceptorRESTRetry
setupCluster(Arrays.asList(good, bad2));
AppsInfo response = interceptor.getApps(null, null, null, null, null, null,
null, null, null, null, null, null, null, null);
null, null, null, null, null, null, null, null, null);
Assert.assertNotNull(response);
Assert.assertEquals(1, response.getApps().size());
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -1564,6 +1564,7 @@ Multiple parameters can be specified for GET operations. The started and finishe
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
* applicationTypes - applications matching the given application types, specified as a comma-separated list.
* applicationTags - applications matching any of the given application tags, specified as a comma-separated list.
* name - name of the application
* deSelects - a generic fields which will be skipped in the result.
### Elements of the *apps* (Applications) object