YARN-1074. Cleaned up YARN CLI application list to only display running applications by default. Contributed by Xuan Gong.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1517196 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-08-24 21:16:40 +00:00
parent e86036662c
commit 962da4dcc7
13 changed files with 630 additions and 124 deletions

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.util.EnumSet;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.client.api.YarnClientApplication;
@ -118,8 +120,10 @@ public class ResourceMgrDelegate extends YarnClient {
try { try {
Set<String> appTypes = new HashSet<String>(1); Set<String> appTypes = new HashSet<String>(1);
appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE);
EnumSet<YarnApplicationState> appStates =
EnumSet.noneOf(YarnApplicationState.class);
return TypeConverter.fromYarnApps( return TypeConverter.fromYarnApps(
client.getApplications(appTypes), this.conf); client.getApplications(appTypes, appStates), this.conf);
} catch (YarnException e) { } catch (YarnException e) {
throw new IOException(e); throw new IOException(e);
} }
@ -299,11 +303,27 @@ public class ResourceMgrDelegate extends YarnClient {
} }
@Override @Override
public List<ApplicationReport> getApplications( public List<ApplicationReport> getApplications(Set<String> applicationTypes)
Set<String> applicationTypes) throws YarnException, IOException { throws YarnException,
IOException {
return client.getApplications(applicationTypes); return client.getApplications(applicationTypes);
} }
@Override
public List<ApplicationReport> getApplications(
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
IOException {
return client.getApplications(applicationStates);
}
@Override
public List<ApplicationReport> getApplications(
Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates)
throws YarnException, IOException {
return client.getApplications(applicationTypes, applicationStates);
}
@Override @Override
public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException { IOException {

View File

@ -45,6 +45,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza). YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
YARN-1074. Cleaned up YARN CLI application list to only display running
applications by default. (Xuan Gong via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.protocolrecords; package org.apache.hadoop.yarn.api.protocolrecords;
import java.util.EnumSet;
import java.util.Set; import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -25,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
/** /**
@ -45,16 +47,68 @@ public abstract class GetApplicationsRequest {
return request; return request;
} }
/**
* <p>
* The request from clients to get a report of Applications matching the
* giving application types in the cluster from the
* <code>ResourceManager</code>.
* </p>
*
*
* @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
*/
@Public @Public
@Stable @Stable
public static GetApplicationsRequest newInstance( public static GetApplicationsRequest
Set<String> applicationTypes) { newInstance(Set<String> applicationTypes) {
GetApplicationsRequest request = GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class); Records.newRecord(GetApplicationsRequest.class);
request.setApplicationTypes(applicationTypes); request.setApplicationTypes(applicationTypes);
return request; return request;
} }
/**
* <p>
* The request from clients to get a report of Applications matching the
* giving application states in the cluster from the
* <code>ResourceManager</code>.
* </p>
*
*
* @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
*/
@Public
@Stable
public static GetApplicationsRequest newInstance(
EnumSet<YarnApplicationState> applicationStates) {
GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class);
request.setApplicationStates(applicationStates);
return request;
}
/**
* <p>
* The request from clients to get a report of Applications matching the
* giving and application types and application types in the cluster from the
* <code>ResourceManager</code>.
* </p>
*
*
* @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
*/
@Public
@Stable
public static GetApplicationsRequest newInstance(
Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) {
GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class);
request.setApplicationTypes(applicationTypes);
request.setApplicationStates(applicationStates);
return request;
}
/** /**
* Get the application types to filter applications on * Get the application types to filter applications on
* *
@ -75,4 +129,25 @@ public abstract class GetApplicationsRequest {
@Unstable @Unstable
public abstract void public abstract void
setApplicationTypes(Set<String> applicationTypes); setApplicationTypes(Set<String> applicationTypes);
/**
* Get the application states to filter applications on
*
* @return Set of Application states to filter on
*/
@Public
@Stable
public abstract EnumSet<YarnApplicationState> getApplicationStates();
/**
* Set the application states to filter applications on
*
* @param applicationStates
* A Set of Application states to filter on.
* If not defined, match all running applications
*/
@Private
@Unstable
public abstract void
setApplicationStates(EnumSet<YarnApplicationState> applicationStates);
} }

View File

@ -122,6 +122,7 @@ message GetClusterMetricsResponseProto {
message GetApplicationsRequestProto { message GetApplicationsRequestProto {
repeated string application_types = 1; repeated string application_types = 1;
repeated YarnApplicationStateProto application_states = 2;
} }
message GetApplicationsResponseProto { message GetApplicationsResponseProto {

View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api; package org.apache.hadoop.yarn.client.api;
import java.io.IOException; import java.io.IOException;
import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
@ -205,6 +207,50 @@ public abstract class YarnClient extends AbstractService {
public abstract List<ApplicationReport> getApplications( public abstract List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException; Set<String> applicationTypes) throws YarnException, IOException;
/**
* <p>
* Get a report (ApplicationReport) of Applications matching the given
* application states in the cluster.
* </p>
*
* <p>
* If the user does not have <code>VIEW_APP</code> access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
* </p>
*
* @param applicationStates
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List<ApplicationReport>
getApplications(EnumSet<YarnApplicationState> applicationStates)
throws YarnException, IOException;
/**
* <p>
* Get a report (ApplicationReport) of Applications matching the given
* application types and application states in the cluster.
* </p>
*
* <p>
* If the user does not have <code>VIEW_APP</code> access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
* </p>
*
* @param applicationTypes
* @param applicationStates
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List<ApplicationReport> getApplications(
Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
IOException;
/** /**
* <p> * <p>
* Get metrics ({@link YarnClusterMetrics}) about the cluster. * Get metrics ({@link YarnClusterMetrics}) about the cluster.

View File

@ -211,15 +211,29 @@ public class YarnClientImpl extends YarnClient {
@Override @Override
public List<ApplicationReport> getApplications() throws YarnException, public List<ApplicationReport> getApplications() throws YarnException,
IOException { IOException {
return getApplications(null); return getApplications(null, null);
}
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes)
throws YarnException,
IOException {
return getApplications(applicationTypes, null);
} }
@Override @Override
public List<ApplicationReport> getApplications( public List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException { EnumSet<YarnApplicationState> applicationStates)
throws YarnException, IOException {
return getApplications(null, applicationStates);
}
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
IOException {
GetApplicationsRequest request = GetApplicationsRequest request =
applicationTypes == null ? GetApplicationsRequest.newInstance() GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
: GetApplicationsRequest.newInstance(applicationTypes);
GetApplicationsResponse response = rmClient.getApplications(request); GetApplicationsResponse response = rmClient.getApplications(request);
return response.getApplicationList(); return response.getApplicationList();
} }

View File

@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.text.DecimalFormat; import java.text.DecimalFormat;
import java.util.EnumSet;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -49,6 +50,10 @@ public class ApplicationCLI extends YarnCLI {
System.getProperty("line.separator"); System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes"; private static final String APP_TYPE_CMD = "appTypes";
private static final String APP_STATE_CMD ="appStates";
private static final String ALLSTATES_OPTION = "ALL";
private boolean allAppStates;
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI(); ApplicationCLI cli = new ApplicationCLI();
@ -66,7 +71,8 @@ public class ApplicationCLI extends YarnCLI {
opts.addOption(STATUS_CMD, true, "Prints the status of the application."); opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications from the RM. " + opts.addOption(LIST_CMD, false, "List applications from the RM. " +
"Supports optional use of --appTypes to filter applications " + "Supports optional use of --appTypes to filter applications " +
"based on application type."); "based on application type, " +
"and --appStates to filter applications based on application state");
opts.addOption(KILL_CMD, true, "Kills the application."); opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(HELP_CMD, false, "Displays help for all commands."); opts.addOption(HELP_CMD, false, "Displays help for all commands.");
Option appTypeOpt = new Option(APP_TYPE_CMD, true, Option appTypeOpt = new Option(APP_TYPE_CMD, true,
@ -75,6 +81,16 @@ public class ApplicationCLI extends YarnCLI {
appTypeOpt.setArgs(Option.UNLIMITED_VALUES); appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
appTypeOpt.setArgName("Comma-separated list of application types"); appTypeOpt.setArgName("Comma-separated list of application types");
opts.addOption(appTypeOpt); opts.addOption(appTypeOpt);
Option appStateOpt =
new Option(
APP_STATE_CMD,
true,
"Works with --list to filter applications based on their state. "
+ getAllValidApplicationStates());
appStateOpt.setValueSeparator(',');
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
appStateOpt.setArgName("Comma-separated list of application states");
opts.addOption(appStateOpt);
opts.getOption(KILL_CMD).setArgName("Application ID"); opts.getOption(KILL_CMD).setArgName("Application ID");
opts.getOption(STATUS_CMD).setArgName("Application ID"); opts.getOption(STATUS_CMD).setArgName("Application ID");
CommandLine cliParser = new GnuParser().parse(opts, args); CommandLine cliParser = new GnuParser().parse(opts, args);
@ -87,18 +103,44 @@ public class ApplicationCLI extends YarnCLI {
} }
printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
} else if (cliParser.hasOption(LIST_CMD)) { } else if (cliParser.hasOption(LIST_CMD)) {
allAppStates = false;
Set<String> appTypes = new HashSet<String>(); Set<String> appTypes = new HashSet<String>();
if(cliParser.hasOption(APP_TYPE_CMD)) { if(cliParser.hasOption(APP_TYPE_CMD)) {
String[] types = cliParser.getOptionValues(APP_TYPE_CMD); String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
if (types != null) { if (types != null) {
for (String type : types) { for (String type : types) {
if (!type.trim().isEmpty()) { if (!type.trim().isEmpty()) {
appTypes.add(type.trim()); appTypes.add(type.toUpperCase().trim());
} }
} }
} }
} }
listApplications(appTypes);
EnumSet<YarnApplicationState> appStates =
EnumSet.noneOf(YarnApplicationState.class);
if (cliParser.hasOption(APP_STATE_CMD)) {
String[] states = cliParser.getOptionValues(APP_STATE_CMD);
if (states != null) {
for (String state : states) {
if (!state.trim().isEmpty()) {
if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
allAppStates = true;
break;
}
try {
appStates.add(YarnApplicationState.valueOf(state.toUpperCase()
.trim()));
} catch (IllegalArgumentException ex) {
sysout.println("The application state " + state
+ " is invalid.");
sysout.println(getAllValidApplicationStates());
return exitCode;
}
}
}
}
}
listApplications(appTypes, appStates);
} else if (cliParser.hasOption(KILL_CMD)) { } else if (cliParser.hasOption(KILL_CMD)) {
if (args.length != 2) { if (args.length != 2) {
printUsage(opts); printUsage(opts);
@ -127,19 +169,33 @@ public class ApplicationCLI extends YarnCLI {
/** /**
* Lists the applications matching the given application Types * Lists the applications matching the given application Types
* present in the Resource Manager * And application States present in the Resource Manager
* *
* @param appTypes * @param appTypes
* @param appStates
* @throws YarnException * @throws YarnException
* @throws IOException * @throws IOException
*/ */
private void listApplications(Set<String> appTypes) private void listApplications(Set<String> appTypes,
throws YarnException, IOException { EnumSet<YarnApplicationState> appStates) throws YarnException,
IOException {
PrintWriter writer = new PrintWriter(sysout); PrintWriter writer = new PrintWriter(sysout);
List<ApplicationReport> appsReport = if (allAppStates) {
client.getApplications(appTypes); for(YarnApplicationState appState : YarnApplicationState.values()) {
appStates.add(appState);
}
} else {
if (appStates.isEmpty()) {
appStates.add(YarnApplicationState.RUNNING);
}
}
writer.println("Total Applications:" + appsReport.size()); List<ApplicationReport> appsReport =
client.getApplications(appTypes, appStates);
writer
.println("Total number of applications (application-types: " + appTypes
+ " and states: " + appStates + ")" + ":" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id", writer.printf(APPLICATIONS_PATTERN, "Application-Id",
"Application-Name","Application-Type", "User", "Queue", "Application-Name","Application-Type", "User", "Queue",
"State", "Final-State","Progress", "Tracking-URL"); "State", "Final-State","Progress", "Tracking-URL");
@ -229,4 +285,16 @@ public class ApplicationCLI extends YarnCLI {
sysout.println(baos.toString("UTF-8")); sysout.println(baos.toString("UTF-8"));
} }
private String getAllValidApplicationStates() {
StringBuilder sb = new StringBuilder();
sb.append("The valid application state can be"
+ " one of the following: ");
sb.append(ALLSTATES_OPTION + ",");
for (YarnApplicationState appState : YarnApplicationState
.values()) {
sb.append(appState+",");
}
String output = sb.toString();
return output.substring(0, output.length()-1);
}
} }

View File

@ -27,6 +27,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException; import java.io.IOException;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -55,10 +56,12 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
@ -163,11 +166,15 @@ public class TestYarnClient {
List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports(); List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports();
List<ApplicationReport> reports = client.getApplications();
Assert.assertEquals(reports, expectedReports);
Set<String> appTypes = new HashSet<String>(); Set<String> appTypes = new HashSet<String>();
appTypes.add("YARN"); appTypes.add("YARN");
appTypes.add("NON-YARN"); appTypes.add("NON-YARN");
List<ApplicationReport> reports = client.getApplications(appTypes); reports =
client.getApplications(appTypes, null);
Assert.assertEquals(reports.size(), 2); Assert.assertEquals(reports.size(), 2);
Assert Assert
.assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports
@ -178,8 +185,28 @@ public class TestYarnClient {
Assert.assertTrue(expectedReports.contains(report)); Assert.assertTrue(expectedReports.contains(report));
} }
reports = client.getApplications(); EnumSet<YarnApplicationState> appStates =
Assert.assertEquals(reports, expectedReports); EnumSet.noneOf(YarnApplicationState.class);
appStates.add(YarnApplicationState.FINISHED);
appStates.add(YarnApplicationState.FAILED);
reports = client.getApplications(null, appStates);
Assert.assertEquals(reports.size(), 2);
Assert
.assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports
.get(1).getApplicationType().equals("NON-MAPREDUCE"))
|| (reports.get(1).getApplicationType().equals("NON-YARN") && reports
.get(0).getApplicationType().equals("NON-MAPREDUCE")));
for (ApplicationReport report : reports) {
Assert.assertTrue(expectedReports.contains(report));
}
reports = client.getApplications(appTypes, appStates);
Assert.assertEquals(reports.size(), 1);
Assert
.assertTrue((reports.get(0).getApplicationType().equals("NON-YARN")));
for (ApplicationReport report : reports) {
Assert.assertTrue(expectedReports.contains(report));
}
client.stop(); client.stop();
} }
@ -187,6 +214,8 @@ public class TestYarnClient {
private static class MockYarnClient extends YarnClientImpl { private static class MockYarnClient extends YarnClientImpl {
private ApplicationReport mockReport; private ApplicationReport mockReport;
private List<ApplicationReport> reports; private List<ApplicationReport> reports;
GetApplicationsResponse mockAppResponse =
mock(GetApplicationsResponse.class);
public MockYarnClient() { public MockYarnClient() {
super(); super();
@ -202,6 +231,8 @@ public class TestYarnClient {
try{ try{
when(rmClient.getApplicationReport(any( when(rmClient.getApplicationReport(any(
GetApplicationReportRequest.class))).thenReturn(mockResponse); GetApplicationReportRequest.class))).thenReturn(mockResponse);
when(rmClient.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(mockAppResponse);
} catch (YarnException e) { } catch (YarnException e) {
Assert.fail("Exception is not expected."); Assert.fail("Exception is not expected.");
} catch (IOException e) { } catch (IOException e) {
@ -212,16 +243,11 @@ public class TestYarnClient {
@Override @Override
public List<ApplicationReport> getApplications( public List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException { Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
GetApplicationsRequest request = throws YarnException, IOException {
applicationTypes == null ? GetApplicationsRequest.newInstance() when(mockAppResponse.getApplicationList()).thenReturn(
: GetApplicationsRequest.newInstance(applicationTypes); getApplicationReports(reports, applicationTypes, applicationStates));
when(rmClient.getApplications(request)) return super.getApplications(applicationTypes, applicationStates);
.thenReturn(
getApplicationReports(reports,
request));
GetApplicationsResponse response = rmClient.getApplications(request);
return response.getApplicationList();
} }
@Override @Override
@ -243,7 +269,7 @@ public class TestYarnClient {
ApplicationReport newApplicationReport = ApplicationReport.newInstance( ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports = List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>(); new ArrayList<ApplicationReport>();
@ -262,31 +288,44 @@ public class TestYarnClient {
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null, "user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null); null);
applicationReports.add(newApplicationReport3); applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
ApplicationReport newApplicationReport4 =
ApplicationReport.newInstance(
applicationId4,
ApplicationAttemptId.newInstance(applicationId4, 4),
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
"NON-MAPREDUCE", null);
applicationReports.add(newApplicationReport4);
return applicationReports; return applicationReports;
} }
private GetApplicationsResponse getApplicationReports( private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports, List<ApplicationReport> applicationReports,
GetApplicationsRequest request) { Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>(); List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
Set<String> appTypes = request.getApplicationTypes();
boolean bypassFilter = appTypes.isEmpty();
for (ApplicationReport appReport : applicationReports) { for (ApplicationReport appReport : applicationReports) {
if (!(bypassFilter || appTypes.contains( if (applicationTypes != null && !applicationTypes.isEmpty()) {
appReport.getApplicationType()))) { if (!applicationTypes.contains(appReport.getApplicationType())) {
continue; continue;
}
}
if (applicationStates != null && !applicationStates.isEmpty()) {
if (!applicationStates.contains(appReport.getYarnApplicationState())) {
continue;
}
} }
appReports.add(appReport); appReports.add(appReport);
} }
GetApplicationsResponse response = return appReports;
GetApplicationsResponse.newInstance(appReports);
return response;
} }
} }

View File

@ -33,6 +33,7 @@ import java.io.PrintStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -134,7 +135,7 @@ public class TestYarnCLI {
ApplicationReport newApplicationReport = ApplicationReport.newInstance( ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>(); List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport); applicationReports.add(newApplicationReport);
@ -152,23 +153,39 @@ public class TestYarnCLI {
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null, "user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null); null);
applicationReports.add(newApplicationReport3); applicationReports.add(newApplicationReport3);
Set<String> appType1 = new HashSet<String>(); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
appType1.add("YARN"); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(
applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
null);
applicationReports.add(newApplicationReport4);
when(client.getApplications(appType1)).thenReturn( // Test command yarn application -list
getApplicationReports(applicationReports, appType1)); // if the set appStates is empty, RUNNING state will be automatically added
int result = cli.run(new String[] { "-list", "-appTypes", "YARN" }); // to the appStates list
// the output of yarn application -list should be the same as
// equals to yarn application -list --appStates RUNNING
Set<String> appType1 = new HashSet<String>();
EnumSet<YarnApplicationState> appState1 =
EnumSet.noneOf(YarnApplicationState.class);
appState1.add(YarnApplicationState.RUNNING);
when(client.getApplications(appType1, appState1)).thenReturn(
getApplicationReports(applicationReports, appType1, appState1, false));
int result = cli.run(new String[] { "-list" });
assertEquals(0, result); assertEquals(0, result);
verify(client).getApplications(appType1); verify(client).getApplications(appType1, appState1);
ByteArrayOutputStream baos = new ByteArrayOutputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos); PrintWriter pw = new PrintWriter(baos);
pw.println("Total Applications:1"); pw.println("Total number of applications (application-types: " + appType1
+ " and states: " + appState1 + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name"); pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type"); pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t "); pw.print("\t User\t Queue\t State\t ");
@ -176,27 +193,41 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL"); pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t "); pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t "); pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t "); pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%"); pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A"); pw.println("\t N/A");
pw.print(" application_1234_0007\t ");
pw.print("appname3\t MAPREDUCE\t user3\t ");
pw.print("queue3\t RUNNING\t ");
pw.print("SUCCEEDED\t 73.79%");
pw.println("\t N/A");
pw.close(); pw.close();
String appsReportStr = baos.toString("UTF-8"); String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString()); Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
//Test command yarn application -list --appTypes apptype1,apptype2
//the output should be the same as
//yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING
sysOutStream.reset(); sysOutStream.reset();
Set<String> appType2 = new HashSet<String>(); Set<String> appType2 = new HashSet<String>();
appType2.add("YARN"); appType2.add("YARN");
appType2.add("FOO-YARN"); appType2.add("NON-YARN");
when(client.getApplications(appType2)).thenReturn(
getApplicationReports(applicationReports, appType2)); EnumSet<YarnApplicationState> appState2 =
cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN", EnumSet.noneOf(YarnApplicationState.class);
",,,,, YARN,," }); appState2.add(YarnApplicationState.RUNNING);
when(client.getApplications(appType2, appState2)).thenReturn(
getApplicationReports(applicationReports, appType2, appState2, false));
result =
cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN",
" ,, ,," });
assertEquals(0, result); assertEquals(0, result);
verify(client).getApplications(appType2); verify(client).getApplications(appType2, appState2);
baos = new ByteArrayOutputStream(); baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos); pw = new PrintWriter(baos);
pw.println("Total Applications:1"); pw.println("Total number of applications (application-types: " + appType2
+ " and states: " + appState2 + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name"); pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type"); pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t "); pw.print("\t User\t Queue\t State\t ");
@ -204,7 +235,7 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL"); pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t "); pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t "); pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t "); pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%"); pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A"); pw.println("\t N/A");
pw.close(); pw.close();
@ -212,29 +243,74 @@ public class TestYarnCLI {
Assert.assertEquals(appsReportStr, sysOutStream.toString()); Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
//Test command yarn application -list --appStates appState1,appState2
sysOutStream.reset(); sysOutStream.reset();
Set<String> appType3 = new HashSet<String>(); Set<String> appType3 = new HashSet<String>();
appType3.add("YARN");
appType3.add("NON-YARN");
when(client.getApplications(appType3)).thenReturn(
getApplicationReports(applicationReports, appType3));
result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" }); EnumSet<YarnApplicationState> appState3 =
EnumSet.noneOf(YarnApplicationState.class);
appState3.add(YarnApplicationState.FINISHED);
appState3.add(YarnApplicationState.FAILED);
when(client.getApplications(appType3, appState3)).thenReturn(
getApplicationReports(applicationReports, appType3, appState3, false));
result =
cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , FAILED",
",,FINISHED" });
assertEquals(0, result); assertEquals(0, result);
verify(client).getApplications(appType3); verify(client).getApplications(appType3, appState3);
baos = new ByteArrayOutputStream(); baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos); pw = new PrintWriter(baos);
pw.println("Total Applications:2"); pw.println("Total number of applications (application-types: " + appType3
+ " and states: " + appState3 + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name"); pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type"); pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t "); pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress"); pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL"); pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t "); pw.print(" application_1234_0006\t ");
pw.print("appname\t YARN\t user\t "); pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue\t FINISHED\t "); pw.print("queue2\t FINISHED\t ");
pw.print("SUCCEEDED\t 53.79%"); pw.print("SUCCEEDED\t 63.79%");
pw.println("\t N/A"); pw.println("\t N/A");
pw.print(" application_1234_0008\t ");
pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
pw.print("queue4\t FAILED\t ");
pw.print("SUCCEEDED\t 83.79%");
pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
// Test command yarn application -list --appTypes apptype1,apptype2
// --appStates appstate1,appstate2
sysOutStream.reset();
Set<String> appType4 = new HashSet<String>();
appType4.add("YARN");
appType4.add("NON-YARN");
EnumSet<YarnApplicationState> appState4 =
EnumSet.noneOf(YarnApplicationState.class);
appState4.add(YarnApplicationState.FINISHED);
appState4.add(YarnApplicationState.FAILED);
when(client.getApplications(appType4, appState4)).thenReturn(
getApplicationReports(applicationReports, appType4, appState4, false));
result =
cli.run(new String[] { "-list", "--appTypes", "YARN,NON-YARN",
"--appStates", "FINISHED ,, , FAILED" });
assertEquals(0, result);
verify(client).getApplications(appType2, appState2);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType4
+ " and states: " + appState4 + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0006\t "); pw.print(" application_1234_0006\t ");
pw.print("appname2\t NON-YARN\t user2\t "); pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue2\t FINISHED\t "); pw.print("queue2\t FINISHED\t ");
@ -243,19 +319,46 @@ public class TestYarnCLI {
pw.close(); pw.close();
appsReportStr = baos.toString("UTF-8"); appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString()); Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
//Test command yarn application -list --appStates with invalid appStates
sysOutStream.reset(); sysOutStream.reset();
Set<String> appType4 = new HashSet<String>(); result =
when(client.getApplications(appType4)).thenReturn( cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , INVALID" });
getApplicationReports(applicationReports, appType4)); assertEquals(-1, result);
result = cli.run(new String[] { "-list" });
assertEquals(0, result);
verify(client).getApplications(appType4);
baos = new ByteArrayOutputStream(); baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos); pw = new PrintWriter(baos);
pw.println("Total Applications:3"); pw.println("The application state INVALID is invalid.");
pw.print("The valid application state can be one of the following: ");
StringBuilder sb = new StringBuilder();
sb.append("ALL,");
for(YarnApplicationState state : YarnApplicationState.values()) {
sb.append(state+",");
}
String output = sb.toString();
pw.println(output.substring(0, output.length()-1));
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
//Test command yarn application -list --appStates all
sysOutStream.reset();
Set<String> appType5 = new HashSet<String>();
EnumSet<YarnApplicationState> appState5 =
EnumSet.noneOf(YarnApplicationState.class);
appState5.add(YarnApplicationState.FINISHED);
when(client.getApplications(appType5, appState5)).thenReturn(
getApplicationReports(applicationReports, appType5, appState5, true));
result =
cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , ALL" });
assertEquals(0, result);
verify(client).getApplications(appType5, appState5);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType5
+ " and states: " + appState5 + ")" + ":" + 4);
pw.print(" Application-Id\t Application-Name"); pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type"); pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t "); pw.print("\t User\t Queue\t State\t ");
@ -263,7 +366,7 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL"); pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t "); pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t "); pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t "); pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%"); pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A"); pw.println("\t N/A");
pw.print(" application_1234_0006\t "); pw.print(" application_1234_0006\t ");
@ -273,27 +376,80 @@ public class TestYarnCLI {
pw.println("\t N/A"); pw.println("\t N/A");
pw.print(" application_1234_0007\t "); pw.print(" application_1234_0007\t ");
pw.print("appname3\t MAPREDUCE\t user3\t "); pw.print("appname3\t MAPREDUCE\t user3\t ");
pw.print("queue3\t FINISHED\t "); pw.print("queue3\t RUNNING\t ");
pw.print("SUCCEEDED\t 73.79%"); pw.print("SUCCEEDED\t 73.79%");
pw.println("\t N/A"); pw.println("\t N/A");
pw.print(" application_1234_0008\t ");
pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
pw.print("queue4\t FAILED\t ");
pw.print("SUCCEEDED\t 83.79%");
pw.println("\t N/A");
pw.close(); pw.close();
appsReportStr = baos.toString("UTF-8"); appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString()); Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
// Test command yarn application user case insensitive
sysOutStream.reset();
Set<String> appType6 = new HashSet<String>();
appType6.add("YARN");
appType6.add("NON-YARN");
EnumSet<YarnApplicationState> appState6 =
EnumSet.noneOf(YarnApplicationState.class);
appState6.add(YarnApplicationState.FINISHED);
when(client.getApplications(appType6, appState6)).thenReturn(
getApplicationReports(applicationReports, appType6, appState6, false));
result =
cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN",
"--appStates", "finished" });
assertEquals(0, result);
verify(client).getApplications(appType6, appState6);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType6
+ " and states: " + appState6 + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0006\t ");
pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue2\t FINISHED\t ");
pw.print("SUCCEEDED\t 63.79%");
pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
} }
private List<ApplicationReport> getApplicationReports( private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports, List<ApplicationReport> applicationReports,
Set<String> appTypes) { Set<String> appTypes, EnumSet<YarnApplicationState> appStates,
boolean allStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>(); List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
boolean bypassFilter = appTypes.isEmpty();
for (ApplicationReport appReport : applicationReports) { if (allStates) {
if (!(bypassFilter || appTypes.contains( for(YarnApplicationState state : YarnApplicationState.values()) {
appReport.getApplicationType()))) { appStates.add(state);
continue;
} }
}
for (ApplicationReport appReport : applicationReports) {
if (appTypes != null && !appTypes.isEmpty()) {
if (!appTypes.contains(appReport.getApplicationType())) {
continue;
}
}
if (appStates != null && !appStates.isEmpty()) {
if (!appStates.contains(appReport.getYarnApplicationState())) {
continue;
}
}
appReports.add(appReport); appReports.add(appReport);
} }
return appReports; return appReports;

View File

@ -18,13 +18,18 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import java.util.EnumSet;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
@ -38,6 +43,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
boolean viaProto = false; boolean viaProto = false;
Set<String> applicationTypes = null; Set<String> applicationTypes = null;
EnumSet<YarnApplicationState> applicationStates = null;
public GetApplicationsRequestPBImpl() { public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder(); builder = GetApplicationsRequestProto.newBuilder();
@ -67,6 +73,40 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
if (this.applicationTypes != null) { if (this.applicationTypes != null) {
addLocalApplicationTypesToProto(); addLocalApplicationTypesToProto();
} }
if (this.applicationStates != null) {
maybeInitBuilder();
builder.clearApplicationStates();
Iterable<YarnApplicationStateProto> iterable =
new Iterable<YarnApplicationStateProto>() {
@Override
public Iterator<YarnApplicationStateProto> iterator() {
return new Iterator<YarnApplicationStateProto>() {
Iterator<YarnApplicationState> iter = applicationStates
.iterator();
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public YarnApplicationStateProto next() {
return ProtoUtils.convertToProtoFormat(iter.next());
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
};
builder.addAllApplicationStates(iterable);
}
} }
private void addLocalApplicationTypesToProto() { private void addLocalApplicationTypesToProto() {
@ -94,6 +134,20 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
this.applicationTypes.addAll(appTypeList); this.applicationTypes.addAll(appTypeList);
} }
private void initApplicationStates() {
if (this.applicationStates != null) {
return;
}
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
List<YarnApplicationStateProto> appStatesList =
p.getApplicationStatesList();
this.applicationStates = EnumSet.noneOf(YarnApplicationState.class);
for (YarnApplicationStateProto c : appStatesList) {
this.applicationStates.add(ProtoUtils.convertFromProtoFormat(c));
}
}
@Override @Override
public Set<String> getApplicationTypes() { public Set<String> getApplicationTypes() {
initApplicationTypes(); initApplicationTypes();
@ -108,6 +162,21 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
this.applicationTypes = applicationTypes; this.applicationTypes = applicationTypes;
} }
@Override
public EnumSet<YarnApplicationState> getApplicationStates() {
initApplicationStates();
return this.applicationStates;
}
@Override
public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
maybeInitBuilder();
if (applicationStates == null) {
builder.clearApplicationStates();
}
this.applicationStates = applicationStates;
}
@Override @Override
public int hashCode() { public int hashCode() {
return getProto().hashCode(); return getProto().hashCode();

View File

@ -73,10 +73,12 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.RPCUtil;
@ -86,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@ -394,7 +397,6 @@ public class ClientRMService extends AbstractService implements
@Override @Override
public GetApplicationsResponse getApplications( public GetApplicationsResponse getApplications(
GetApplicationsRequest request) throws YarnException { GetApplicationsRequest request) throws YarnException {
UserGroupInformation callerUGI; UserGroupInformation callerUGI;
try { try {
callerUGI = UserGroupInformation.getCurrentUser(); callerUGI = UserGroupInformation.getCurrentUser();
@ -404,12 +406,22 @@ public class ClientRMService extends AbstractService implements
} }
Set<String> applicationTypes = request.getApplicationTypes(); Set<String> applicationTypes = request.getApplicationTypes();
boolean bypassFilter = applicationTypes.isEmpty(); EnumSet<YarnApplicationState> applicationStates =
request.getApplicationStates();
List<ApplicationReport> reports = new ArrayList<ApplicationReport>(); List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
for (RMApp application : this.rmContext.getRMApps().values()) { for (RMApp application : this.rmContext.getRMApps().values()) {
if (!(bypassFilter || applicationTypes.contains(application if (applicationTypes != null && !applicationTypes.isEmpty()) {
.getApplicationType()))) { if (!applicationTypes.contains(application.getApplicationType())) {
continue; continue;
}
}
if (applicationStates != null && !applicationStates.isEmpty()) {
if (!applicationStates.contains(RMServerUtils
.createApplicationState(application.getState()))) {
continue;
}
} }
boolean allowAccess = checkAccess(callerUGI, application.getUser(), boolean allowAccess = checkAccess(callerUGI, application.getUser(),
ApplicationAccessType.VIEW_APP, application.getApplicationId()); ApplicationAccessType.VIEW_APP, application.getApplicationId());

View File

@ -28,9 +28,12 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@ -112,4 +115,27 @@ public class RMServerUtils {
} }
} }
} }
public static YarnApplicationState createApplicationState(RMAppState rmAppState) {
switch(rmAppState) {
case NEW:
return YarnApplicationState.NEW;
case NEW_SAVING:
return YarnApplicationState.NEW_SAVING;
case SUBMITTED:
return YarnApplicationState.SUBMITTED;
case ACCEPTED:
return YarnApplicationState.ACCEPTED;
case RUNNING:
return YarnApplicationState.RUNNING;
case FINISHING:
case FINISHED:
return YarnApplicationState.FINISHED;
case KILLED:
return YarnApplicationState.KILLED;
case FAILED:
return YarnApplicationState.FAILED;
}
throw new YarnRuntimeException("Unknown state passed!");
}
} }

View File

@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
@ -55,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@ -378,29 +378,6 @@ public class RMAppImpl implements RMApp, Recoverable {
} }
} }
private YarnApplicationState createApplicationState(RMAppState rmAppState) {
switch(rmAppState) {
case NEW:
return YarnApplicationState.NEW;
case NEW_SAVING:
return YarnApplicationState.NEW_SAVING;
case SUBMITTED:
return YarnApplicationState.SUBMITTED;
case ACCEPTED:
return YarnApplicationState.ACCEPTED;
case RUNNING:
return YarnApplicationState.RUNNING;
case FINISHING:
case FINISHED:
return YarnApplicationState.FINISHED;
case KILLED:
return YarnApplicationState.KILLED;
case FAILED:
return YarnApplicationState.FAILED;
}
throw new YarnRuntimeException("Unknown state passed!");
}
private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) { private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) {
switch(state) { switch(state) {
case NEW: case NEW:
@ -500,7 +477,7 @@ public class RMAppImpl implements RMApp, Recoverable {
return BuilderUtils.newApplicationReport(this.applicationId, return BuilderUtils.newApplicationReport(this.applicationId,
currentApplicationAttemptId, this.user, this.queue, currentApplicationAttemptId, this.user, this.queue,
this.name, host, rpcPort, clientToAMToken, this.name, host, rpcPort, clientToAMToken,
createApplicationState(this.stateMachine.getCurrentState()), diags, RMServerUtils.createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState, trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType, appUsageReport, origTrackingUrl, progress, this.applicationType,
amrmToken); amrmToken);