From 115706142984657469f7d447472cca8edff5d4cf Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Tue, 9 Jul 2013 23:07:15 +0000 Subject: [PATCH] merge -c 1501599 from trunk to branch-2 to fix YARN-727, MAPREDUCE-5325. ClientRMProtocol.getAllApplications should accept ApplicationType as a parameter. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1501600 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapred/ResourceMgrDelegate.java | 17 +- .../hadoop/mapred/TestClientRedirect.java | 8 +- .../mapred/TestResourceMgrDelegate.java | 12 +- .../apache/hadoop/mapred/TestYARNRunner.java | 10 +- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/api/ApplicationClientProtocol.java | 21 ++- .../GetAllApplicationsRequest.java | 44 ----- .../GetApplicationsRequest.java | 78 +++++++++ ...onse.java => GetApplicationsResponse.java} | 26 +-- .../proto/applicationclient_protocol.proto | 2 +- .../src/main/proto/yarn_service_protos.proto | 5 +- .../hadoop/yarn/client/api/YarnClient.java | 25 ++- .../yarn/client/api/impl/YarnClientImpl.java | 22 ++- .../yarn/client/cli/ApplicationCLI.java | 49 +++++- .../hadoop/yarn/client/cli/YarnCLI.java | 1 + .../yarn/client/api/impl/TestYarnClient.java | 109 +++++++++++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 165 +++++++++++++++++- ...ApplicationClientProtocolPBClientImpl.java | 20 +-- ...pplicationClientProtocolPBServiceImpl.java | 22 +-- .../pb/GetAllApplicationsRequestPBImpl.java | 67 ------- .../impl/pb/GetApplicationsRequestPBImpl.java | 128 ++++++++++++++ ...ava => GetApplicationsResponsePBImpl.java} | 38 ++-- .../resourcemanager/ClientRMService.java | 19 +- .../resourcemanager/TestApplicationACLs.java | 18 +- .../resourcemanager/TestClientRMService.java | 40 ++++- 26 files changed, 721 insertions(+), 231 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/{GetAllApplicationsResponse.java => GetApplicationsResponse.java} (79%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/{GetAllApplicationsResponsePBImpl.java => GetApplicationsResponsePBImpl.java} (80%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a429a1b17d5..8a50ae96845 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -442,6 +442,9 @@ Release 2.1.0-beta - 2013-07-02 MAPREDUCE-5334. Fix failing unit tests - TestContainerLauncher, TestContainerLauncherImpl. (Vinod Kumar Vavilapalli via sseth) + MAPREDUCE-5325. MR changes related to YARN-727. ClientRMProtocol.getAllApplications + should accept ApplicationType as a parameter. (Xuan Gong via hitesh) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index c687de665f1..45242629fd4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -20,7 +20,9 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -131,7 +133,10 @@ public class ResourceMgrDelegate extends YarnClient { public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnApps(client.getApplicationList(), this.conf); + Set appTypes = new HashSet(1); + appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); + return TypeConverter.fromYarnApps( + client.getApplications(appTypes), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -299,9 +304,15 @@ public class ResourceMgrDelegate extends YarnClient { } @Override - public List getApplicationList() throws YarnException, + public List getApplications() throws YarnException, IOException { - return client.getApplicationList(); + return client.getApplications(); + } + + @Override + public List getApplications( + Set applicationTypes) throws YarnException, IOException { + return client.getApplications(applicationTypes); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index a60cc4a8f1d..1f0b89f00ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -72,8 +72,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -314,8 +314,8 @@ public class TestClientRedirect { } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws IOException { + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws IOException { return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index 2339fb58d20..6c6df6673ee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -27,8 +27,8 @@ import junit.framework.Assert; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -93,8 +93,8 @@ public class TestResourceMgrDelegate { @Test public void tesAllJobs() throws Exception { final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class); - GetAllApplicationsResponse allApplicationsResponse = Records - .newRecord(GetAllApplicationsResponse.class); + GetApplicationsResponse allApplicationsResponse = Records + .newRecord(GetApplicationsResponse.class); List applications = new ArrayList(); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.FAILED)); @@ -106,8 +106,8 @@ public class TestResourceMgrDelegate { FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( - applicationsManager.getAllApplications(Mockito - .any(GetAllApplicationsRequest.class))).thenReturn( + applicationsManager.getApplications(Mockito + .any(GetApplicationsRequest.class))).thenReturn( allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 0046c9b3302..0c08b813aef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -60,8 +60,8 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -213,10 +213,10 @@ public class TestYARNRunner extends TestCase { verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class)); /* make sure getalljobs calls get all applications */ - when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))). - thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class)); + when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))). + thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class)); delegate.getAllJobs(); - verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class)); + verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class)); /* make sure getapplication report is called */ when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class))) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c54b270277d..3dbdac317fc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -216,6 +216,9 @@ Release 2.1.0-beta - 2013-07-02 YARN-791. Changed RM APIs and web-services related to nodes to ensure that both are consistent with each other. (Sandy Ryza via vinodkv) + YARN-727. ClientRMProtocol.getAllApplications should accept ApplicationType as + a parameter. (Xuan Gong via hitesh) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 5427d3bad54..74db3715771 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -205,27 +205,30 @@ public interface ApplicationClientProtocol { throws YarnException, IOException; /** - *

The interface used by clients to get a report of all Applications + *

The interface used by clients to get a report of Applications + * matching the filters defined by {@link GetApplicationsRequest} * in the cluster from the ResourceManager.

* *

The ResourceManager responds with a - * {@link GetAllApplicationsResponse} which includes the - * {@link ApplicationReport} for all the applications.

+ * {@link GetApplicationsResponse} which includes the + * {@link ApplicationReport} for the applications.

* *

If the user does not have VIEW_APP access for an * application then the corresponding report will be filtered as * described in {@link #getApplicationReport(GetApplicationReportRequest)}. *

* - * @param request request for report on all running applications - * @return report on all running applications + * @param request request for report on applications + * @return report on applications matching the given application types + * defined in the request * @throws YarnException * @throws IOException + * @see GetApplicationsRequest */ @Public @Stable - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java deleted file mode 100644 index 88902386dd5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The request from clients to get a report of all Applications - * in the cluster from the ResourceManager.

- * - *

Currently, this is empty.

- * - * @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest) - */ -@Public -@Stable -public abstract class GetAllApplicationsRequest { - @Public - @Stable - public static GetAllApplicationsRequest newInstance() { - GetAllApplicationsRequest request = - Records.newRecord(GetAllApplicationsRequest.class); - return request; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java new file mode 100644 index 00000000000..9a732210aa1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request from clients to get a report of Applications + * in the cluster from the ResourceManager.

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ +@Public +@Stable +public abstract class GetApplicationsRequest { + @Public + @Stable + public static GetApplicationsRequest newInstance() { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + return request; + } + + @Public + @Stable + public static GetApplicationsRequest newInstance( + Set applicationTypes) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationTypes(applicationTypes); + return request; + } + + /** + * Get the application types to filter applications on + * + * @return Set of Application Types to filter on + */ + @Public + @Stable + public abstract Set getApplicationTypes(); + + /** + * Set the application types to filter applications on + * + * @param applicationTypes + * A Set of Application Types to filter on. + * If not defined, match all applications + */ + @Private + @Unstable + public abstract void + setApplicationTypes(Set applicationTypes); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java similarity index 79% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java index ae04624ad07..45e8fdf0971 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java @@ -30,36 +30,36 @@ import org.apache.hadoop.yarn.util.Records; /** *

The response sent by the ResourceManager to a client - * requesting an {@link ApplicationReport} for all applications.

- * - *

The ApplicationReport for each application includes details - * such as user, queue, name, host on which the ApplicationMaster + * requesting an {@link ApplicationReport} for applications.

+ * + *

The ApplicationReport for each application includes details + * such as user, queue, name, host on which the ApplicationMaster * is running, RPC port, tracking URL, diagnostics, start time etc.

- * + * * @see ApplicationReport - * @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest) + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) */ @Public @Stable -public abstract class GetAllApplicationsResponse { +public abstract class GetApplicationsResponse { @Private @Unstable - public static GetAllApplicationsResponse newInstance( + public static GetApplicationsResponse newInstance( List applications) { - GetAllApplicationsResponse response = - Records.newRecord(GetAllApplicationsResponse.class); + GetApplicationsResponse response = + Records.newRecord(GetApplicationsResponse.class); response.setApplicationList(applications); return response; } /** - * Get ApplicationReport for all applications. - * @return ApplicationReport for all applications + * Get ApplicationReport for applications. + * @return ApplicationReport for applications */ @Public @Stable public abstract List getApplicationList(); - + @Private @Unstable public abstract void setApplicationList(List applications); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index a145f155fdb..3f76849de93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -36,7 +36,7 @@ service ApplicationClientProtocolService { rpc submitApplication (SubmitApplicationRequestProto) returns (SubmitApplicationResponseProto); rpc forceKillApplication (KillApplicationRequestProto) returns (KillApplicationResponseProto); rpc getClusterMetrics (GetClusterMetricsRequestProto) returns (GetClusterMetricsResponseProto); - rpc getAllApplications (GetAllApplicationsRequestProto) returns (GetAllApplicationsResponseProto); + rpc getApplications (GetApplicationsRequestProto) returns (GetApplicationsResponseProto); rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto); rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto); rpc getQueueUserAcls (GetQueueUserAclsInfoRequestProto) returns (GetQueueUserAclsInfoResponseProto); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index c82f5cc62e3..dd3b3214296 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -123,10 +123,11 @@ message GetClusterMetricsResponseProto { optional YarnClusterMetricsProto cluster_metrics = 1; } -message GetAllApplicationsRequestProto { +message GetApplicationsRequestProto { + repeated string application_types = 1; } -message GetAllApplicationsResponseProto { +message GetApplicationsResponseProto { repeated ApplicationReportProto applications = 1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index f55c5f36915..e8dca61d32a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api; import java.io.IOException; import java.net.InetSocketAddress; import java.util.List; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -174,8 +175,28 @@ public abstract class YarnClient extends AbstractService { * @throws YarnException * @throws IOException */ - public abstract List getApplicationList() throws YarnException, - IOException; + public abstract List getApplications() + throws YarnException, IOException; + + /** + *

+ * Get a report (ApplicationReport) of Applications + * matching the given application types in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationTypes + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplications( + Set applicationTypes) throws YarnException, IOException; /** *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 7a7affbeb34..b3b8bdf4316 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,8 +33,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -206,11 +207,18 @@ public class YarnClientImpl extends YarnClient { } @Override - public List getApplicationList() - throws YarnException, IOException { - GetAllApplicationsRequest request = - Records.newRecord(GetAllApplicationsRequest.class); - GetAllApplicationsResponse response = rmClient.getAllApplications(request); + public List getApplications() throws YarnException, + IOException { + return getApplications(null); + } + + @Override + public List getApplications( + Set applicationTypes) throws YarnException, IOException { + GetApplicationsRequest request = + applicationTypes == null ? GetApplicationsRequest.newInstance() + : GetApplicationsRequest.newInstance(applicationTypes); + GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 312aab25132..fa22b29ddb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -21,11 +21,14 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.text.DecimalFormat; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -35,6 +38,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.annotations.VisibleForTesting; + @Private @Unstable public class ApplicationCLI extends YarnCLI { @@ -42,6 +47,8 @@ public class ApplicationCLI extends YarnCLI { "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + System.getProperty("line.separator"); + private static final String APP_TYPE_CMD = "appTypes"; + public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); cli.setSysOutPrintStream(System.out); @@ -56,8 +63,19 @@ public class ApplicationCLI extends YarnCLI { Options opts = new Options(); opts.addOption(STATUS_CMD, true, "Prints the status of the application."); - opts.addOption(LIST_CMD, false, "Lists all the Applications from RM."); + opts.addOption(LIST_CMD, false, "List applications from the RM. " + + "Supports optional use of --appTypes to filter applications " + + "based on application type."); opts.addOption(KILL_CMD, true, "Kills the application."); + opts.addOption(HELP_CMD, false, "Displays help for all commands."); + Option appTypeOpt = new Option(APP_TYPE_CMD, true, + "Works with --list to filter applications based on their type."); + appTypeOpt.setValueSeparator(','); + appTypeOpt.setArgs(Option.UNLIMITED_VALUES); + appTypeOpt.setArgName("Comma-separated list of application types"); + opts.addOption(appTypeOpt); + opts.getOption(KILL_CMD).setArgName("Application ID"); + opts.getOption(STATUS_CMD).setArgName("Application ID"); CommandLine cliParser = new GnuParser().parse(opts, args); int exitCode = -1; @@ -68,13 +86,27 @@ public class ApplicationCLI extends YarnCLI { } printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); } else if (cliParser.hasOption(LIST_CMD)) { - listAllApplications(); + Set appTypes = new HashSet(); + if(cliParser.hasOption(APP_TYPE_CMD)) { + String[] types = cliParser.getOptionValues(APP_TYPE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + appTypes.add(type.trim()); + } + } + } + } + listApplications(appTypes); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); return exitCode; } killApplication(cliParser.getOptionValue(KILL_CMD)); + } else if (cliParser.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; } else { syserr.println("Invalid Command Usage : "); printUsage(opts); @@ -87,19 +119,24 @@ public class ApplicationCLI extends YarnCLI { * * @param opts */ - private void printUsage(Options opts) { + @VisibleForTesting + void printUsage(Options opts) { new HelpFormatter().printHelp("application", opts); } /** - * Lists all the applications present in the Resource Manager + * Lists the applications matching the given application Types + * present in the Resource Manager * + * @param appTypes * @throws YarnException * @throws IOException */ - private void listAllApplications() throws YarnException, IOException { + private void listApplications(Set appTypes) + throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); - List appsReport = client.getApplicationList(); + List appsReport = + client.getApplications(appTypes); writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java index 5f86033e657..921c1355b1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java @@ -33,6 +33,7 @@ public abstract class YarnCLI extends Configured implements Tool { public static final String STATUS_CMD = "status"; public static final String LIST_CMD = "list"; public static final String KILL_CMD = "kill"; + public static final String HELP_CMD = "help"; protected PrintStream sysout; protected PrintStream syserr; protected YarnClient client; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 1c9b7505581..b2cbcf0f6df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -25,7 +25,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import junit.framework.Assert; @@ -33,10 +37,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; @@ -138,12 +146,44 @@ public class TestYarnClient { Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType()); rm.stop(); } - + + @Test (timeout = 10000) + public void testGetApplications() throws YarnException, IOException { + Configuration conf = new Configuration(); + final YarnClient client = new MockYarnClient(); + client.init(conf); + client.start(); + + List expectedReports = ((MockYarnClient)client).getReports(); + + Set appTypes = new HashSet(); + appTypes.add("YARN"); + appTypes.add("NON-YARN"); + + List reports = client.getApplications(appTypes); + Assert.assertEquals(reports.size(), 2); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports + .get(1).getApplicationType().equals("NON-YARN")) + || (reports.get(1).getApplicationType().equals("YARN") && reports + .get(0).getApplicationType().equals("NON-YARN"))); + for(ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } + + reports = client.getApplications(); + Assert.assertEquals(reports, expectedReports); + + client.stop(); + } + private static class MockYarnClient extends YarnClientImpl { private ApplicationReport mockReport; + private List reports; public MockYarnClient() { super(); + reports = createAppReports(); } @Override @@ -163,6 +203,20 @@ public class TestYarnClient { when(mockResponse.getApplicationReport()).thenReturn(mockReport); } + @Override + public List getApplications( + Set applicationTypes) throws YarnException, IOException { + GetApplicationsRequest request = + applicationTypes == null ? GetApplicationsRequest.newInstance() + : GetApplicationsRequest.newInstance(applicationTypes); + when(rmClient.getApplications(request)) + .thenReturn( + getApplicationReports(reports, + request)); + GetApplicationsResponse response = rmClient.getApplications(request); + return response.getApplicationList(); + } + @Override public void stop() { } @@ -172,6 +226,59 @@ public class TestYarnClient { YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.NEW_SAVING, state); } + + public List getReports() { + return this.reports; + } + + private List createAppReports() { + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationReport newApplicationReport = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, 1), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); + List applicationReports = + new ArrayList(); + applicationReports.add(newApplicationReport); + + ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), + "user2", "queue2", "appname2", "host2", 125, null, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN"); + applicationReports.add(newApplicationReport2); + + ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); + ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( + applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), + "user3", "queue3", "appname3", "host3", 126, null, + YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE"); + applicationReports.add(newApplicationReport3); + return applicationReports; + } + + private GetApplicationsResponse getApplicationReports( + List applicationReports, + GetApplicationsRequest request) { + + List appReports = new ArrayList(); + Set appTypes = request.getApplicationTypes(); + boolean bypassFilter = appTypes.isEmpty(); + + for (ApplicationReport appReport : applicationReports) { + if (!(bypassFilter || appTypes.contains( + appReport.getApplicationType()))) { + continue; + } + appReports.add(appReport); + } + GetApplicationsResponse response = + GetApplicationsResponse.newInstance(appReports); + return response; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index d1f52a4f897..7d03dcfdd90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -32,7 +32,9 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import junit.framework.Assert; @@ -51,6 +53,8 @@ import org.apache.hadoop.yarn.util.Records; import org.junit.Before; import org.junit.Test; +import org.apache.commons.cli.Options; + public class TestYarnCLI { private YarnClient client = mock(YarnClient.class); @@ -105,7 +109,7 @@ public class TestYarnCLI { } @Test - public void testGetAllApplications() throws Exception { + public void testGetApplications() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationReport newApplicationReport = ApplicationReport.newInstance( @@ -115,10 +119,31 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); - when(client.getApplicationList()).thenReturn(applicationReports); - int result = cli.run(new String[] { "-list" }); + + ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), + "user2", "queue2", "appname2", "host2", 125, null, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN"); + applicationReports.add(newApplicationReport2); + + ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); + ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( + applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), + "user3", "queue3", "appname3", "host3", 126, null, + YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE"); + applicationReports.add(newApplicationReport3); + + Set appType1 = new HashSet(); + appType1.add("YARN"); + + when(client.getApplications(appType1)).thenReturn( + getApplicationReports(applicationReports, appType1)); + int result = cli.run(new String[] { "-list", "-appTypes", "YARN" }); assertEquals(0, result); - verify(client).getApplicationList(); + verify(client).getApplications(appType1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); @@ -137,6 +162,138 @@ public class TestYarnCLI { String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType2 = new HashSet(); + appType2.add("YARN"); + appType2.add("FOO-YARN"); + when(client.getApplications(appType2)).thenReturn( + getApplicationReports(applicationReports, appType2)); + cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN", + ",,,,, YARN,," }); + assertEquals(0, result); + verify(client).getApplications(appType2); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:1"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType3 = new HashSet(); + appType3.add("YARN"); + appType3.add("NON-YARN"); + when(client.getApplications(appType3)).thenReturn( + getApplicationReports(applicationReports, appType3)); + + result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" }); + assertEquals(0, result); + verify(client).getApplications(appType3); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:2"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType4 = new HashSet(); + when(client.getApplications(appType4)).thenReturn( + getApplicationReports(applicationReports, appType4)); + result = cli.run(new String[] { "-list" }); + assertEquals(0, result); + verify(client).getApplications(appType4); + + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:3"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t FINISHED\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + } + + private List getApplicationReports( + List applicationReports, + Set appTypes) { + + List appReports = new ArrayList(); + boolean bypassFilter = appTypes.isEmpty(); + + for (ApplicationReport appReport : applicationReports) { + if (!(bypassFilter || appTypes.contains( + appReport.getApplicationType()))) { + continue; + } + appReports.add(appReport); + } + return appReports; + } + + @Test (timeout = 10000) + public void testHelpCommand() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationCLI spyCli = spy(cli); + int result = spyCli.run(new String[] { "-help" }); + Assert.assertTrue(result == 0); + verify(spyCli).printUsage(any(Options.class)); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + result = + cli.run(new String[] { "-kill", applicationId.toString(), "args" }); + verify(spyCli).printUsage(any(Options.class)); + + NodeId nodeId = NodeId.newInstance("host0", 0); + result = cli.run(new String[] { "-status", nodeId.toString(), "args" }); + verify(spyCli).printUsage(any(Options.class)); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index e6cb4024e38..88352eac453 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -33,8 +33,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; @@ -81,7 +81,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationReque import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; @@ -188,13 +188,13 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException, + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException, IOException { - GetAllApplicationsRequestProto requestProto = - ((GetAllApplicationsRequestPBImpl) request).getProto(); + GetApplicationsRequestProto requestProto = + ((GetApplicationsRequestPBImpl) request).getProto(); try { - return new GetAllApplicationsResponsePBImpl(proxy.getAllApplications( + return new GetApplicationsResponsePBImpl(proxy.getApplications( null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index d092edf9337..b38819dfab0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -30,7 +30,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRespo import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; @@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; @@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRe import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; @@ -170,14 +170,14 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient } @Override - public GetAllApplicationsResponseProto getAllApplications( - RpcController controller, GetAllApplicationsRequestProto proto) + public GetApplicationsResponseProto getApplications( + RpcController controller, GetApplicationsRequestProto proto) throws ServiceException { - GetAllApplicationsRequestPBImpl request = - new GetAllApplicationsRequestPBImpl(proto); + GetApplicationsRequestPBImpl request = + new GetApplicationsRequestPBImpl(proto); try { - GetAllApplicationsResponse response = real.getAllApplications(request); - return ((GetAllApplicationsResponsePBImpl)response).getProto(); + GetApplicationsResponse response = real.getApplications(request); + return ((GetApplicationsResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java deleted file mode 100644 index 24b16d052c7..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; - -@Private -@Unstable -public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest { - GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance(); - GetAllApplicationsRequestProto.Builder builder = null; - boolean viaProto = false; - - public GetAllApplicationsRequestPBImpl() { - builder = GetAllApplicationsRequestProto.newBuilder(); - } - - public GetAllApplicationsRequestPBImpl(GetAllApplicationsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetAllApplicationsRequestProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java new file mode 100644 index 00000000000..dda5e2137d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder; + +@Private +@Unstable +public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { + GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance(); + GetApplicationsRequestProto.Builder builder = null; + boolean viaProto = false; + + Set applicationTypes = null; + + public GetApplicationsRequestPBImpl() { + builder = GetApplicationsRequestProto.newBuilder(); + } + + public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetApplicationsRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.applicationTypes != null) { + addLocalApplicationTypesToProto(); + } + } + + private void addLocalApplicationTypesToProto() { + maybeInitBuilder(); + builder.clearApplicationTypes(); + if (this.applicationTypes == null) + return; + builder.addAllApplicationTypes(applicationTypes); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetApplicationsRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void initApplicationTypes() { + if (this.applicationTypes != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + List appTypeList = p.getApplicationTypesList(); + this.applicationTypes = new HashSet(); + this.applicationTypes.addAll(appTypeList); + } + + @Override + public Set getApplicationTypes() { + initApplicationTypes(); + return this.applicationTypes; + } + + @Override + public void setApplicationTypes(Set applicationTypes) { + maybeInitBuilder(); + if (applicationTypes == null) + builder.clearApplicationTypes(); + this.applicationTypes = applicationTypes; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java similarity index 80% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java index a26dccd59bb..b0897c67820 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java @@ -24,36 +24,36 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProtoOrBuilder; @Private @Unstable -public class GetAllApplicationsResponsePBImpl -extends GetAllApplicationsResponse { +public class GetApplicationsResponsePBImpl +extends GetApplicationsResponse { - GetAllApplicationsResponseProto proto = - GetAllApplicationsResponseProto.getDefaultInstance(); - GetAllApplicationsResponseProto.Builder builder = null; + GetApplicationsResponseProto proto = + GetApplicationsResponseProto.getDefaultInstance(); + GetApplicationsResponseProto.Builder builder = null; boolean viaProto = false; List applicationList; - - public GetAllApplicationsResponsePBImpl() { - builder = GetAllApplicationsResponseProto.newBuilder(); + + public GetApplicationsResponsePBImpl() { + builder = GetApplicationsResponseProto.newBuilder(); } - - public GetAllApplicationsResponsePBImpl(GetAllApplicationsResponseProto proto) { + + public GetApplicationsResponsePBImpl(GetApplicationsResponseProto proto) { this.proto = proto; viaProto = true; } @Override - public List getApplicationList() { + public List getApplicationList() { initLocalApplicationsList(); return this.applicationList; } @@ -61,12 +61,12 @@ extends GetAllApplicationsResponse { @Override public void setApplicationList(List applications) { maybeInitBuilder(); - if (applications == null) + if (applications == null) builder.clearApplications(); this.applicationList = applications; } - public GetAllApplicationsResponseProto getProto() { + public GetApplicationsResponseProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -100,7 +100,7 @@ extends GetAllApplicationsResponse { } private void mergeLocalToProto() { - if (viaProto) + if (viaProto) maybeInitBuilder(); mergeLocalToBuilder(); proto = builder.build(); @@ -109,7 +109,7 @@ extends GetAllApplicationsResponse { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = GetAllApplicationsResponseProto.newBuilder(proto); + builder = GetApplicationsResponseProto.newBuilder(proto); } viaProto = false; } @@ -120,7 +120,7 @@ extends GetAllApplicationsResponse { if (this.applicationList != null) { return; } - GetAllApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder; + GetApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder; List list = p.getApplicationsList(); applicationList = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 897d53459b3..882ad5178ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -42,8 +43,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -391,8 +392,8 @@ public class ClientRMService extends AbstractService implements } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException { + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException { UserGroupInformation callerUGI; try { @@ -402,15 +403,21 @@ public class ClientRMService extends AbstractService implements throw RPCUtil.getRemoteException(ie); } + Set applicationTypes = request.getApplicationTypes(); + boolean bypassFilter = applicationTypes.isEmpty(); List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { + if (!(bypassFilter || applicationTypes.contains(application + .getApplicationType()))) { + continue; + } boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); reports.add(application.createAndGetApplicationReport(allowAccess)); } - GetAllApplicationsResponse response = - recordFactory.newRecordInstance(GetAllApplicationsResponse.class); + GetApplicationsResponse response = + recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index 1cf80450974..8c283550c81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -34,7 +34,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -213,8 +213,8 @@ public class TestApplicationACLs { // List apps as owner Assert.assertEquals("App view by owner should list the apps!!", 1, - rmClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + rmClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as owner @@ -244,8 +244,8 @@ public class TestApplicationACLs { // List apps as superUser Assert.assertEquals("App view by super-user should list the apps!!", 2, - superUserClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + superUserClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the superUser @@ -275,8 +275,8 @@ public class TestApplicationACLs { // List apps as friend Assert.assertEquals("App view by a friend should list the apps!!", 3, - friendClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + friendClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the friend @@ -308,8 +308,8 @@ public class TestApplicationACLs { // List apps as enemy List appReports = enemyRmClient - .getAllApplications(recordFactory - .newRecordInstance(GetAllApplicationsRequest.class)) + .getApplications(recordFactory + .newRecordInstance(GetApplicationsRequest.class)) .getApplicationList(); Assert.assertEquals("App view by enemy should list the apps!!", 4, appReports.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 69107290669..cd2e86c5fbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -26,8 +26,10 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; +import java.util.HashSet; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; @@ -42,6 +44,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -49,6 +53,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -88,6 +93,8 @@ public class TestClientRMService { private RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + private String appType = "MockApp"; + private static RMDelegationTokenSecretManager dtsm; @BeforeClass @@ -292,11 +299,18 @@ public class TestClientRMService { new EventHandler() { public void handle(Event event) {} }); + ApplicationId appId1 = getApplicationId(100); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when( + mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); ClientRMService rmService = - new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + new ClientRMService(rmContext, yarnScheduler, appManager, + mockAclsManager, null); // without name and queue - ApplicationId appId1 = getApplicationId(100); + SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest( appId1, null, null); try { @@ -317,6 +331,8 @@ public class TestClientRMService { ApplicationId appId2 = getApplicationId(101); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest( appId2, name, queue); + submitRequest2.getApplicationSubmissionContext().setApplicationType( + "matchType"); try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { @@ -335,6 +351,25 @@ public class TestClientRMService { Assert.assertTrue("The thrown exception is not expected.", e.getMessage().contains("Cannot add a duplicate!")); } + + GetApplicationsRequest getAllAppsRequest = + GetApplicationsRequest.newInstance(new HashSet()); + GetApplicationsResponse getAllApplicationsResponse = + rmService.getApplications(getAllAppsRequest); + Assert.assertEquals(5, + getAllApplicationsResponse.getApplicationList().size()); + + Set appTypes = new HashSet(); + appTypes.add("matchType"); + + getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes); + getAllApplicationsResponse = + rmService.getApplications(getAllAppsRequest); + Assert.assertEquals(1, + getAllApplicationsResponse.getApplicationList().size()); + Assert.assertEquals(appId2, + getAllApplicationsResponse.getApplicationList() + .get(0).getApplicationId()); } @Test(timeout=4000) @@ -416,6 +451,7 @@ public class TestClientRMService { submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); + submissionContext.setApplicationType(appType); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);