From 962da4dcc74d23c7ce78164dcde38ea5aaf3dd68 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 24 Aug 2013 21:16:40 +0000 Subject: [PATCH] YARN-1074. Cleaned up YARN CLI application list to only display running applications by default. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1517196 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapred/ResourceMgrDelegate.java | 26 +- hadoop-yarn-project/CHANGES.txt | 3 + .../GetApplicationsRequest.java | 79 +++++- .../src/main/proto/yarn_service_protos.proto | 1 + .../hadoop/yarn/client/api/YarnClient.java | 50 +++- .../yarn/client/api/impl/YarnClientImpl.java | 22 +- .../yarn/client/cli/ApplicationCLI.java | 86 +++++- .../yarn/client/api/impl/TestYarnClient.java | 93 +++++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 250 ++++++++++++++---- .../impl/pb/GetApplicationsRequestPBImpl.java | 69 +++++ .../resourcemanager/ClientRMService.java | 22 +- .../server/resourcemanager/RMServerUtils.java | 26 ++ .../resourcemanager/rmapp/RMAppImpl.java | 27 +- 13 files changed, 630 insertions(+), 124 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index ba02d86bdef..74b07c2f3db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; @@ -118,8 +120,10 @@ public class ResourceMgrDelegate extends YarnClient { try { Set appTypes = new HashSet(1); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); + EnumSet appStates = + EnumSet.noneOf(YarnApplicationState.class); return TypeConverter.fromYarnApps( - client.getApplications(appTypes), this.conf); + client.getApplications(appTypes, appStates), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -299,11 +303,27 @@ public class ResourceMgrDelegate extends YarnClient { } @Override - public List getApplications( - Set applicationTypes) throws YarnException, IOException { + public List getApplications(Set applicationTypes) + throws YarnException, + IOException { return client.getApplications(applicationTypes); } + @Override + public List getApplications( + EnumSet applicationStates) throws YarnException, + IOException { + return client.getApplications(applicationStates); + } + + @Override + public List getApplications( + Set applicationTypes, + EnumSet applicationStates) + throws YarnException, IOException { + return client.getApplications(applicationTypes, applicationStates); + } + @Override public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException { diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index db62f03d1cf..f208581e627 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -44,6 +44,9 @@ Release 2.1.1-beta - UNRELEASED IMPROVEMENTS YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza). + + YARN-1074. Cleaned up YARN CLI application list to only display running + applications by default. (Xuan Gong via vinodkv) OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 9a732210aa1..e53bf88a4fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.EnumSet; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -25,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Records; /** @@ -45,16 +47,68 @@ public abstract class GetApplicationsRequest { return request; } + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ @Public @Stable - public static GetApplicationsRequest newInstance( - Set applicationTypes) { + public static GetApplicationsRequest + newInstance(Set applicationTypes) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationTypes(applicationTypes); return request; } + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application states in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + EnumSet applicationStates) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationStates(applicationStates); + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving and application types and application types in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + Set applicationTypes, + EnumSet applicationStates) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationTypes(applicationTypes); + request.setApplicationStates(applicationStates); + return request; + } + /** * Get the application types to filter applications on * @@ -75,4 +129,25 @@ public abstract class GetApplicationsRequest { @Unstable public abstract void setApplicationTypes(Set applicationTypes); + + /** + * Get the application states to filter applications on + * + * @return Set of Application states to filter on + */ + @Public + @Stable + public abstract EnumSet getApplicationStates(); + + /** + * Set the application states to filter applications on + * + * @param applicationStates + * A Set of Application states to filter on. + * If not defined, match all running applications + */ + @Private + @Unstable + public abstract void + setApplicationStates(EnumSet applicationStates); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index bd009e0d4a6..7b3d0cf77cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -122,6 +122,7 @@ message GetClusterMetricsResponseProto { message GetApplicationsRequestProto { repeated string application_types = 1; + repeated YarnApplicationStateProto application_states = 2; } message GetApplicationsResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index c3587b29170..155ba5d51a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.api; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -171,13 +173,13 @@ public abstract class YarnClient extends AbstractService { *

* Get a report (ApplicationReport) of all Applications in the cluster. *

- * + * *

* If the user does not have VIEW_APP access for an application * then the corresponding report will be filtered as described in * {@link #getApplicationReport(ApplicationId)}. *

- * + * * @return a list of reports of all running applications * @throws YarnException * @throws IOException @@ -205,6 +207,50 @@ public abstract class YarnClient extends AbstractService { public abstract List getApplications( Set applicationTypes) throws YarnException, IOException; + /** + *

+ * Get a report (ApplicationReport) of Applications matching the given + * application states in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationStates + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List + getApplications(EnumSet applicationStates) + throws YarnException, IOException; + + /** + *

+ * Get a report (ApplicationReport) of Applications matching the given + * application types and application states in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationTypes + * @param applicationStates + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplications( + Set applicationTypes, + EnumSet applicationStates) throws YarnException, + IOException; + /** *

* Get metrics ({@link YarnClusterMetrics}) about the cluster. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 4a1b83ca9e9..d35e1a4300d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -211,15 +211,29 @@ public class YarnClientImpl extends YarnClient { @Override public List getApplications() throws YarnException, IOException { - return getApplications(null); + return getApplications(null, null); + } + + @Override + public List getApplications(Set applicationTypes) + throws YarnException, + IOException { + return getApplications(applicationTypes, null); } @Override public List getApplications( - Set applicationTypes) throws YarnException, IOException { + EnumSet applicationStates) + throws YarnException, IOException { + return getApplications(null, applicationStates); + } + + @Override + public List getApplications(Set applicationTypes, + EnumSet applicationStates) throws YarnException, + IOException { GetApplicationsRequest request = - applicationTypes == null ? GetApplicationsRequest.newInstance() - : GetApplicationsRequest.newInstance(applicationTypes); + GetApplicationsRequest.newInstance(applicationTypes, applicationStates); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 16e55a6a72d..69de37a76c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.text.DecimalFormat; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -49,6 +50,10 @@ public class ApplicationCLI extends YarnCLI { System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; + private static final String APP_STATE_CMD ="appStates"; + private static final String ALLSTATES_OPTION = "ALL"; + + private boolean allAppStates; public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); @@ -66,7 +71,8 @@ public class ApplicationCLI extends YarnCLI { opts.addOption(STATUS_CMD, true, "Prints the status of the application."); opts.addOption(LIST_CMD, false, "List applications from the RM. " + "Supports optional use of --appTypes to filter applications " + - "based on application type."); + "based on application type, " + + "and --appStates to filter applications based on application state"); opts.addOption(KILL_CMD, true, "Kills the application."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); Option appTypeOpt = new Option(APP_TYPE_CMD, true, @@ -75,6 +81,16 @@ public class ApplicationCLI extends YarnCLI { appTypeOpt.setArgs(Option.UNLIMITED_VALUES); appTypeOpt.setArgName("Comma-separated list of application types"); opts.addOption(appTypeOpt); + Option appStateOpt = + new Option( + APP_STATE_CMD, + true, + "Works with --list to filter applications based on their state. " + + getAllValidApplicationStates()); + appStateOpt.setValueSeparator(','); + appStateOpt.setArgs(Option.UNLIMITED_VALUES); + appStateOpt.setArgName("Comma-separated list of application states"); + opts.addOption(appStateOpt); opts.getOption(KILL_CMD).setArgName("Application ID"); opts.getOption(STATUS_CMD).setArgName("Application ID"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -87,18 +103,44 @@ public class ApplicationCLI extends YarnCLI { } printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); } else if (cliParser.hasOption(LIST_CMD)) { + allAppStates = false; Set appTypes = new HashSet(); if(cliParser.hasOption(APP_TYPE_CMD)) { String[] types = cliParser.getOptionValues(APP_TYPE_CMD); if (types != null) { for (String type : types) { if (!type.trim().isEmpty()) { - appTypes.add(type.trim()); + appTypes.add(type.toUpperCase().trim()); } } } } - listApplications(appTypes); + + EnumSet appStates = + EnumSet.noneOf(YarnApplicationState.class); + if (cliParser.hasOption(APP_STATE_CMD)) { + String[] states = cliParser.getOptionValues(APP_STATE_CMD); + if (states != null) { + for (String state : states) { + if (!state.trim().isEmpty()) { + if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) { + allAppStates = true; + break; + } + try { + appStates.add(YarnApplicationState.valueOf(state.toUpperCase() + .trim())); + } catch (IllegalArgumentException ex) { + sysout.println("The application state " + state + + " is invalid."); + sysout.println(getAllValidApplicationStates()); + return exitCode; + } + } + } + } + } + listApplications(appTypes, appStates); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); @@ -127,19 +169,33 @@ public class ApplicationCLI extends YarnCLI { /** * Lists the applications matching the given application Types - * present in the Resource Manager + * And application States present in the Resource Manager * * @param appTypes + * @param appStates * @throws YarnException * @throws IOException */ - private void listApplications(Set appTypes) - throws YarnException, IOException { + private void listApplications(Set appTypes, + EnumSet appStates) throws YarnException, + IOException { PrintWriter writer = new PrintWriter(sysout); - List appsReport = - client.getApplications(appTypes); + if (allAppStates) { + for(YarnApplicationState appState : YarnApplicationState.values()) { + appStates.add(appState); + } + } else { + if (appStates.isEmpty()) { + appStates.add(YarnApplicationState.RUNNING); + } + } - writer.println("Total Applications:" + appsReport.size()); + List appsReport = + client.getApplications(appTypes, appStates); + + writer + .println("Total number of applications (application-types: " + appTypes + + " and states: " + appStates + ")" + ":" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name","Application-Type", "User", "Queue", "State", "Final-State","Progress", "Tracking-URL"); @@ -229,4 +285,16 @@ public class ApplicationCLI extends YarnCLI { sysout.println(baos.toString("UTF-8")); } + private String getAllValidApplicationStates() { + StringBuilder sb = new StringBuilder(); + sb.append("The valid application state can be" + + " one of the following: "); + sb.append(ALLSTATES_OPTION + ","); + for (YarnApplicationState appState : YarnApplicationState + .values()) { + sb.append(appState+","); + } + String output = sb.toString(); + return output.substring(0, output.length()-1); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 8df430aa8d3..e7a66bd28ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -55,10 +56,12 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -163,11 +166,15 @@ public class TestYarnClient { List expectedReports = ((MockYarnClient)client).getReports(); + List reports = client.getApplications(); + Assert.assertEquals(reports, expectedReports); + Set appTypes = new HashSet(); appTypes.add("YARN"); appTypes.add("NON-YARN"); - List reports = client.getApplications(appTypes); + reports = + client.getApplications(appTypes, null); Assert.assertEquals(reports.size(), 2); Assert .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports @@ -178,8 +185,28 @@ public class TestYarnClient { Assert.assertTrue(expectedReports.contains(report)); } - reports = client.getApplications(); - Assert.assertEquals(reports, expectedReports); + EnumSet appStates = + EnumSet.noneOf(YarnApplicationState.class); + appStates.add(YarnApplicationState.FINISHED); + appStates.add(YarnApplicationState.FAILED); + reports = client.getApplications(null, appStates); + Assert.assertEquals(reports.size(), 2); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports + .get(1).getApplicationType().equals("NON-MAPREDUCE")) + || (reports.get(1).getApplicationType().equals("NON-YARN") && reports + .get(0).getApplicationType().equals("NON-MAPREDUCE"))); + for (ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } + + reports = client.getApplications(appTypes, appStates); + Assert.assertEquals(reports.size(), 1); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN"))); + for (ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } client.stop(); } @@ -187,6 +214,8 @@ public class TestYarnClient { private static class MockYarnClient extends YarnClientImpl { private ApplicationReport mockReport; private List reports; + GetApplicationsResponse mockAppResponse = + mock(GetApplicationsResponse.class); public MockYarnClient() { super(); @@ -202,6 +231,8 @@ public class TestYarnClient { try{ when(rmClient.getApplicationReport(any( GetApplicationReportRequest.class))).thenReturn(mockResponse); + when(rmClient.getApplications(any(GetApplicationsRequest.class))) + .thenReturn(mockAppResponse); } catch (YarnException e) { Assert.fail("Exception is not expected."); } catch (IOException e) { @@ -212,16 +243,11 @@ public class TestYarnClient { @Override public List getApplications( - Set applicationTypes) throws YarnException, IOException { - GetApplicationsRequest request = - applicationTypes == null ? GetApplicationsRequest.newInstance() - : GetApplicationsRequest.newInstance(applicationTypes); - when(rmClient.getApplications(request)) - .thenReturn( - getApplicationReports(reports, - request)); - GetApplicationsResponse response = rmClient.getApplications(request); - return response.getApplicationList(); + Set applicationTypes, EnumSet applicationStates) + throws YarnException, IOException { + when(mockAppResponse.getApplicationList()).thenReturn( + getApplicationReports(reports, applicationTypes, applicationStates)); + return super.getApplications(applicationTypes, applicationStates); } @Override @@ -243,7 +269,7 @@ public class TestYarnClient { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); @@ -262,31 +288,44 @@ public class TestYarnClient { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); + + ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); + ApplicationReport newApplicationReport4 = + ApplicationReport.newInstance( + applicationId4, + ApplicationAttemptId.newInstance(applicationId4, 4), + "user4", "queue4", "appname4", "host4", 127, null, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, + "NON-MAPREDUCE", null); + applicationReports.add(newApplicationReport4); return applicationReports; } - private GetApplicationsResponse getApplicationReports( + private List getApplicationReports( List applicationReports, - GetApplicationsRequest request) { + Set applicationTypes, EnumSet applicationStates) { List appReports = new ArrayList(); - Set appTypes = request.getApplicationTypes(); - boolean bypassFilter = appTypes.isEmpty(); - for (ApplicationReport appReport : applicationReports) { - if (!(bypassFilter || appTypes.contains( - appReport.getApplicationType()))) { - continue; + if (applicationTypes != null && !applicationTypes.isEmpty()) { + if (!applicationTypes.contains(appReport.getApplicationType())) { + continue; + } + } + + if (applicationStates != null && !applicationStates.isEmpty()) { + if (!applicationStates.contains(appReport.getYarnApplicationState())) { + continue; + } } appReports.add(appReport); } - GetApplicationsResponse response = - GetApplicationsResponse.newInstance(appReports); - return response; + return appReports; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index c33ddea38b7..6d53108f95a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -33,6 +33,7 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Date; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -134,7 +135,7 @@ public class TestYarnCLI { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -152,23 +153,39 @@ public class TestYarnCLI { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); - Set appType1 = new HashSet(); - appType1.add("YARN"); + ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); + ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( + applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), + "user4", "queue4", "appname4", "host4", 127, null, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", + null); + applicationReports.add(newApplicationReport4); - when(client.getApplications(appType1)).thenReturn( - getApplicationReports(applicationReports, appType1)); - int result = cli.run(new String[] { "-list", "-appTypes", "YARN" }); + // Test command yarn application -list + // if the set appStates is empty, RUNNING state will be automatically added + // to the appStates list + // the output of yarn application -list should be the same as + // equals to yarn application -list --appStates RUNNING + Set appType1 = new HashSet(); + EnumSet appState1 = + EnumSet.noneOf(YarnApplicationState.class); + appState1.add(YarnApplicationState.RUNNING); + when(client.getApplications(appType1, appState1)).thenReturn( + getApplicationReports(applicationReports, appType1, appState1, false)); + int result = cli.run(new String[] { "-list" }); assertEquals(0, result); - verify(client).getApplications(appType1); + verify(client).getApplications(appType1, appState1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.println("Total Applications:1"); + pw.println("Total number of applications (application-types: " + appType1 + + " and states: " + appState1 + ")" + ":" + 2); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -176,27 +193,41 @@ public class TestYarnCLI { pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); + pw.print("queue\t RUNNING\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t RUNNING\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); pw.close(); String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); + //Test command yarn application -list --appTypes apptype1,apptype2 + //the output should be the same as + //yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING sysOutStream.reset(); Set appType2 = new HashSet(); appType2.add("YARN"); - appType2.add("FOO-YARN"); - when(client.getApplications(appType2)).thenReturn( - getApplicationReports(applicationReports, appType2)); - cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN", - ",,,,, YARN,," }); + appType2.add("NON-YARN"); + + EnumSet appState2 = + EnumSet.noneOf(YarnApplicationState.class); + appState2.add(YarnApplicationState.RUNNING); + when(client.getApplications(appType2, appState2)).thenReturn( + getApplicationReports(applicationReports, appType2, appState2, false)); + result = + cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN", + " ,, ,," }); assertEquals(0, result); - verify(client).getApplications(appType2); + verify(client).getApplications(appType2, appState2); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); - pw.println("Total Applications:1"); + pw.println("Total number of applications (application-types: " + appType2 + + " and states: " + appState2 + ")" + ":" + 1); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -204,7 +235,7 @@ public class TestYarnCLI { pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); + pw.print("queue\t RUNNING\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); pw.close(); @@ -212,29 +243,74 @@ public class TestYarnCLI { Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); + //Test command yarn application -list --appStates appState1,appState2 sysOutStream.reset(); Set appType3 = new HashSet(); - appType3.add("YARN"); - appType3.add("NON-YARN"); - when(client.getApplications(appType3)).thenReturn( - getApplicationReports(applicationReports, appType3)); - result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" }); + EnumSet appState3 = + EnumSet.noneOf(YarnApplicationState.class); + appState3.add(YarnApplicationState.FINISHED); + appState3.add(YarnApplicationState.FAILED); + + when(client.getApplications(appType3, appState3)).thenReturn( + getApplicationReports(applicationReports, appType3, appState3, false)); + result = + cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , FAILED", + ",,FINISHED" }); assertEquals(0, result); - verify(client).getApplications(appType3); + verify(client).getApplications(appType3, appState3); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); - pw.println("Total Applications:2"); + pw.println("Total number of applications (application-types: " + appType3 + + " and states: " + appState3 + ")" + ":" + 2); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); pw.print("Final-State\t Progress"); pw.println("\t Tracking-URL"); - pw.print(" application_1234_0005\t "); - pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); - pw.print("SUCCEEDED\t 53.79%"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); pw.println("\t N/A"); + pw.print(" application_1234_0008\t "); + pw.print("appname4\t NON-MAPREDUCE\t user4\t "); + pw.print("queue4\t FAILED\t "); + pw.print("SUCCEEDED\t 83.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + + // Test command yarn application -list --appTypes apptype1,apptype2 + // --appStates appstate1,appstate2 + sysOutStream.reset(); + Set appType4 = new HashSet(); + appType4.add("YARN"); + appType4.add("NON-YARN"); + + EnumSet appState4 = + EnumSet.noneOf(YarnApplicationState.class); + appState4.add(YarnApplicationState.FINISHED); + appState4.add(YarnApplicationState.FAILED); + + when(client.getApplications(appType4, appState4)).thenReturn( + getApplicationReports(applicationReports, appType4, appState4, false)); + result = + cli.run(new String[] { "-list", "--appTypes", "YARN,NON-YARN", + "--appStates", "FINISHED ,, , FAILED" }); + assertEquals(0, result); + verify(client).getApplications(appType2, appState2); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType4 + + " and states: " + appState4 + ")" + ":" + 1); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); pw.print(" application_1234_0006\t "); pw.print("appname2\t NON-YARN\t user2\t "); pw.print("queue2\t FINISHED\t "); @@ -243,19 +319,46 @@ public class TestYarnCLI { pw.close(); appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); - verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + //Test command yarn application -list --appStates with invalid appStates sysOutStream.reset(); - Set appType4 = new HashSet(); - when(client.getApplications(appType4)).thenReturn( - getApplicationReports(applicationReports, appType4)); - result = cli.run(new String[] { "-list" }); - assertEquals(0, result); - verify(client).getApplications(appType4); - + result = + cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , INVALID" }); + assertEquals(-1, result); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); - pw.println("Total Applications:3"); + pw.println("The application state INVALID is invalid."); + pw.print("The valid application state can be one of the following: "); + StringBuilder sb = new StringBuilder(); + sb.append("ALL,"); + for(YarnApplicationState state : YarnApplicationState.values()) { + sb.append(state+","); + } + String output = sb.toString(); + pw.println(output.substring(0, output.length()-1)); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + + //Test command yarn application -list --appStates all + sysOutStream.reset(); + Set appType5 = new HashSet(); + + EnumSet appState5 = + EnumSet.noneOf(YarnApplicationState.class); + appState5.add(YarnApplicationState.FINISHED); + when(client.getApplications(appType5, appState5)).thenReturn( + getApplicationReports(applicationReports, appType5, appState5, true)); + result = + cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , ALL" }); + assertEquals(0, result); + verify(client).getApplications(appType5, appState5); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType5 + + " and states: " + appState5 + ")" + ":" + 4); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -263,7 +366,7 @@ public class TestYarnCLI { pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); + pw.print("queue\t RUNNING\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); pw.print(" application_1234_0006\t "); @@ -273,27 +376,80 @@ public class TestYarnCLI { pw.println("\t N/A"); pw.print(" application_1234_0007\t "); pw.print("appname3\t MAPREDUCE\t user3\t "); - pw.print("queue3\t FINISHED\t "); + pw.print("queue3\t RUNNING\t "); pw.print("SUCCEEDED\t 73.79%"); pw.println("\t N/A"); + pw.print(" application_1234_0008\t "); + pw.print("appname4\t NON-MAPREDUCE\t user4\t "); + pw.print("queue4\t FAILED\t "); + pw.print("SUCCEEDED\t 83.79%"); + pw.println("\t N/A"); pw.close(); appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); - verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt()); + + // Test command yarn application user case insensitive + sysOutStream.reset(); + Set appType6 = new HashSet(); + appType6.add("YARN"); + appType6.add("NON-YARN"); + + EnumSet appState6 = + EnumSet.noneOf(YarnApplicationState.class); + appState6.add(YarnApplicationState.FINISHED); + when(client.getApplications(appType6, appState6)).thenReturn( + getApplicationReports(applicationReports, appType6, appState6, false)); + result = + cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN", + "--appStates", "finished" }); + assertEquals(0, result); + verify(client).getApplications(appType6, appState6); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType6 + + " and states: " + appState6 + ")" + ":" + 1); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt()); } private List getApplicationReports( List applicationReports, - Set appTypes) { + Set appTypes, EnumSet appStates, + boolean allStates) { List appReports = new ArrayList(); - boolean bypassFilter = appTypes.isEmpty(); - for (ApplicationReport appReport : applicationReports) { - if (!(bypassFilter || appTypes.contains( - appReport.getApplicationType()))) { - continue; + if (allStates) { + for(YarnApplicationState state : YarnApplicationState.values()) { + appStates.add(state); } + } + for (ApplicationReport appReport : applicationReports) { + if (appTypes != null && !appTypes.isEmpty()) { + if (!appTypes.contains(appReport.getApplicationType())) { + continue; + } + } + + if (appStates != null && !appStates.isEmpty()) { + if (!appStates.contains(appReport.getYarnApplicationState())) { + continue; + } + } + appReports.add(appReport); } return appReports; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 48a8d85ab8e..33f74f0903e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -18,13 +18,18 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import java.util.EnumSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder; @@ -38,6 +43,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { boolean viaProto = false; Set applicationTypes = null; + EnumSet applicationStates = null; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -67,6 +73,40 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { if (this.applicationTypes != null) { addLocalApplicationTypesToProto(); } + if (this.applicationStates != null) { + maybeInitBuilder(); + builder.clearApplicationStates(); + Iterable iterable = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = applicationStates + .iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public YarnApplicationStateProto next() { + return ProtoUtils.convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllApplicationStates(iterable); + } } private void addLocalApplicationTypesToProto() { @@ -94,6 +134,20 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { this.applicationTypes.addAll(appTypeList); } + private void initApplicationStates() { + if (this.applicationStates != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + List appStatesList = + p.getApplicationStatesList(); + this.applicationStates = EnumSet.noneOf(YarnApplicationState.class); + + for (YarnApplicationStateProto c : appStatesList) { + this.applicationStates.add(ProtoUtils.convertFromProtoFormat(c)); + } + } + @Override public Set getApplicationTypes() { initApplicationTypes(); @@ -108,6 +162,21 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { this.applicationTypes = applicationTypes; } + @Override + public EnumSet getApplicationStates() { + initApplicationStates(); + return this.applicationStates; + } + + @Override + public void setApplicationStates(EnumSet applicationStates) { + maybeInitBuilder(); + if (applicationStates == null) { + builder.clearApplicationStates(); + } + this.applicationStates = applicationStates; + } + @Override public int hashCode() { return getProto().hashCode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 97f0ef8e0b1..81fdd56b833 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -73,10 +73,12 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -86,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -394,7 +397,6 @@ public class ClientRMService extends AbstractService implements @Override public GetApplicationsResponse getApplications( GetApplicationsRequest request) throws YarnException { - UserGroupInformation callerUGI; try { callerUGI = UserGroupInformation.getCurrentUser(); @@ -404,12 +406,22 @@ public class ClientRMService extends AbstractService implements } Set applicationTypes = request.getApplicationTypes(); - boolean bypassFilter = applicationTypes.isEmpty(); + EnumSet applicationStates = + request.getApplicationStates(); + List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { - if (!(bypassFilter || applicationTypes.contains(application - .getApplicationType()))) { - continue; + if (applicationTypes != null && !applicationTypes.isEmpty()) { + if (!applicationTypes.contains(application.getApplicationType())) { + continue; + } + } + + if (applicationStates != null && !applicationStates.isEmpty()) { + if (!applicationStates.contains(RMServerUtils + .createApplicationState(application.getState()))) { + continue; + } } boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 241d7bcbd7c..15d306293e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -28,9 +28,12 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; @@ -112,4 +115,27 @@ public class RMServerUtils { } } } + + public static YarnApplicationState createApplicationState(RMAppState rmAppState) { + switch(rmAppState) { + case NEW: + return YarnApplicationState.NEW; + case NEW_SAVING: + return YarnApplicationState.NEW_SAVING; + case SUBMITTED: + return YarnApplicationState.SUBMITTED; + case ACCEPTED: + return YarnApplicationState.ACCEPTED; + case RUNNING: + return YarnApplicationState.RUNNING; + case FINISHING: + case FINISHED: + return YarnApplicationState.FINISHED; + case KILLED: + return YarnApplicationState.KILLED; + case FAILED: + return YarnApplicationState.FAILED; + } + throw new YarnRuntimeException("Unknown state passed!"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index a11b05eedc9..79398840ce1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -55,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; @@ -378,29 +378,6 @@ public class RMAppImpl implements RMApp, Recoverable { } } - private YarnApplicationState createApplicationState(RMAppState rmAppState) { - switch(rmAppState) { - case NEW: - return YarnApplicationState.NEW; - case NEW_SAVING: - return YarnApplicationState.NEW_SAVING; - case SUBMITTED: - return YarnApplicationState.SUBMITTED; - case ACCEPTED: - return YarnApplicationState.ACCEPTED; - case RUNNING: - return YarnApplicationState.RUNNING; - case FINISHING: - case FINISHED: - return YarnApplicationState.FINISHED; - case KILLED: - return YarnApplicationState.KILLED; - case FAILED: - return YarnApplicationState.FAILED; - } - throw new YarnRuntimeException("Unknown state passed!"); - } - private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) { switch(state) { case NEW: @@ -500,7 +477,7 @@ public class RMAppImpl implements RMApp, Recoverable { return BuilderUtils.newApplicationReport(this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, - createApplicationState(this.stateMachine.getCurrentState()), diags, + RMServerUtils.createApplicationState(this.stateMachine.getCurrentState()), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken);