Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge -c 1501599 from trunk to branch-2 to fix YARN-727, MAPREDUCE-5325. ClientRMProtocol.getAllApplications should accept ApplicationType as a parameter. Contributed by Xuan Gong. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1501600 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Hitesh Shah 2013-07-09 23:07:15 +00:00
parent 29090d45af
commit 1157061429
26 changed files with 721 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -442,6 +442,9 @@ Release 2.1.0-beta - 2013-07-02
MAPREDUCE-5334. Fix failing unit tests - TestContainerLauncher,
TestContainerLauncherImpl. (Vinod Kumar Vavilapalli via sseth)
MAPREDUCE-5325. MR changes related to YARN-727. ClientRMProtocol.getAllApplications
should accept ApplicationType as a parameter. (Xuan Gong via hitesh)
BREAKDOWN OF HADOOP-8562 SUBTASKS
MAPREDUCE-4739. Some MapReduce tests fail to find winutils.

17
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
View File

@ -20,7 +20,9 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -131,7 +133,10 @@ public class ResourceMgrDelegate extends YarnClient {
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
try {
return TypeConverter.fromYarnApps(client.getApplicationList(), this.conf);
Set<String> appTypes = new HashSet<String>(1);
appTypes.add(MRJobConfig.MR_APPLICATION_TYPE);
return TypeConverter.fromYarnApps(
client.getApplications(appTypes), this.conf);
} catch (YarnException e) {
throw new IOException(e);
}
@ -299,9 +304,15 @@ public class ResourceMgrDelegate extends YarnClient {
}
@Override
public List<ApplicationReport> getApplicationList() throws YarnException,
public List<ApplicationReport> getApplications() throws YarnException,
IOException {
return client.getApplicationList();
return client.getApplications();
}
@Override
public List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException {
return client.getApplications(applicationTypes);
}
@Override

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
View File

@ -72,8 +72,8 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -314,8 +314,8 @@ public class TestClientRedirect {
}
@Override
public GetAllApplicationsResponse getAllApplications(
GetAllApplicationsRequest request) throws IOException {
public GetApplicationsResponse getApplications(
GetApplicationsRequest request) throws IOException {
return null;
}

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
View File

@ -27,8 +27,8 @@ import junit.framework.Assert;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -93,8 +93,8 @@ public class TestResourceMgrDelegate {
@Test
public void tesAllJobs() throws Exception {
final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
GetAllApplicationsResponse allApplicationsResponse = Records
.newRecord(GetAllApplicationsResponse.class);
GetApplicationsResponse allApplicationsResponse = Records
.newRecord(GetApplicationsResponse.class);
List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
applications.add(getApplicationReport(YarnApplicationState.FINISHED,
FinalApplicationStatus.FAILED));
@ -106,8 +106,8 @@ public class TestResourceMgrDelegate {
FinalApplicationStatus.FAILED));
allApplicationsResponse.setApplicationList(applications);
Mockito.when(
applicationsManager.getAllApplications(Mockito
.any(GetAllApplicationsRequest.class))).thenReturn(
applicationsManager.getApplications(Mockito
.any(GetApplicationsRequest.class))).thenReturn(
allApplicationsResponse);
ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
new YarnConfiguration()) {

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
View File

@ -60,8 +60,8 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -213,10 +213,10 @@ public class TestYARNRunner extends TestCase {
verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));
/* make sure getalljobs calls get all applications */
when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class));
when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class));
delegate.getAllJobs();
verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class));
verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class));
/* make sure getapplication report is called */
when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))

3
hadoop-yarn-project/CHANGES.txt
View File

@ -216,6 +216,9 @@ Release 2.1.0-beta - 2013-07-02
YARN-791. Changed RM APIs and web-services related to nodes to ensure that
both are consistent with each other. (Sandy Ryza via vinodkv)
YARN-727. ClientRMProtocol.getAllApplications should accept ApplicationType as
a parameter. (Xuan Gong via hitesh)
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
View File

@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -205,27 +205,30 @@ public interface ApplicationClientProtocol {
throws YarnException, IOException;
/**
* <p>The interface used by clients to get a report of all Applications
* <p>The interface used by clients to get a report of Applications
* matching the filters defined by {@link GetApplicationsRequest}
* in the cluster from the <code>ResourceManager</code>.</p>
*
* <p>The <code>ResourceManager</code> responds with a
* {@link GetAllApplicationsResponse} which includes the
* {@link ApplicationReport} for all the applications.</p>
* {@link GetApplicationsResponse} which includes the
* {@link ApplicationReport} for the applications.</p>
*
* <p>If the user does not have <code>VIEW_APP</code> access for an
* application then the corresponding report will be filtered as
* described in {@link #getApplicationReport(GetApplicationReportRequest)}.
* </p>
*
* @param request request for report on all running applications
* @return report on all running applications
* @param request request for report on applications
* @return report on applications matching the given application types
* defined in the request
* @throws YarnException
* @throws IOException
* @see GetApplicationsRequest
*/
@Public
@Stable
public GetAllApplicationsResponse getAllApplications(
GetAllApplicationsRequest request)
public GetApplicationsResponse getApplications(
GetApplicationsRequest request)
throws YarnException, IOException;
/**

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java
View File

@ -1,44 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.protocolrecords;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.util.Records;
/**
* <p>The request from clients to get a report of all Applications
* in the cluster from the <code>ResourceManager</code>.</p>
*
* <p>Currently, this is empty.</p>
*
* @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest)
*/
@Public
@Stable
public abstract class GetAllApplicationsRequest {
@Public
@Stable
public static GetAllApplicationsRequest newInstance() {
GetAllApplicationsRequest request =
Records.newRecord(GetAllApplicationsRequest.class);
return request;
}
}

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Normal file
View File

@ -0,0 +1,78 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.protocolrecords;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.util.Records;
/**
* <p>The request from clients to get a report of Applications
* in the cluster from the <code>ResourceManager</code>.</p>
*
*
* @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
*/
@Public
@Stable
public abstract class GetApplicationsRequest {
@Public
@Stable
public static GetApplicationsRequest newInstance() {
GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class);
return request;
}
@Public
@Stable
public static GetApplicationsRequest newInstance(
Set<String> applicationTypes) {
GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class);
request.setApplicationTypes(applicationTypes);
return request;
}
/**
* Get the application types to filter applications on
*
* @return Set of Application Types to filter on
*/
@Public
@Stable
public abstract Set<String> getApplicationTypes();
/**
* Set the application types to filter applications on
*
* @param applicationTypes
* A Set of Application Types to filter on.
* If not defined, match all applications
*/
@Private
@Unstable
public abstract void
setApplicationTypes(Set<String> applicationTypes);
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java
View File

@ -30,36 +30,36 @@ import org.apache.hadoop.yarn.util.Records;
/**
* <p>The response sent by the <code>ResourceManager</code> to a client
* requesting an {@link ApplicationReport} for all applications.</p>
*
* <p>The <code>ApplicationReport</code> for each application includes details
* such as user, queue, name, host on which the <code>ApplicationMaster</code>
* requesting an {@link ApplicationReport} for applications.</p>
*
* <p>The <code>ApplicationReport</code> for each application includes details
* such as user, queue, name, host on which the <code>ApplicationMaster</code>
* is running, RPC port, tracking URL, diagnostics, start time etc.</p>
*
*
* @see ApplicationReport
* @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest)
* @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
*/
@Public
@Stable
public abstract class GetAllApplicationsResponse {
public abstract class GetApplicationsResponse {
@Private
@Unstable
public static GetAllApplicationsResponse newInstance(
public static GetApplicationsResponse newInstance(
List<ApplicationReport> applications) {
GetAllApplicationsResponse response =
Records.newRecord(GetAllApplicationsResponse.class);
GetApplicationsResponse response =
Records.newRecord(GetApplicationsResponse.class);
response.setApplicationList(applications);
return response;
}
/**
* Get <code>ApplicationReport</code> for all applications.
* @return <code>ApplicationReport</code> for all applications
* Get <code>ApplicationReport</code> for applications.
* @return <code>ApplicationReport</code> for applications
*/
@Public
@Stable
public abstract List<ApplicationReport> getApplicationList();
@Private
@Unstable
public abstract void setApplicationList(List<ApplicationReport> applications);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
View File

@ -36,7 +36,7 @@ service ApplicationClientProtocolService {
rpc submitApplication (SubmitApplicationRequestProto) returns (SubmitApplicationResponseProto);
rpc forceKillApplication (KillApplicationRequestProto) returns (KillApplicationResponseProto);
rpc getClusterMetrics (GetClusterMetricsRequestProto) returns (GetClusterMetricsResponseProto);
rpc getAllApplications (GetAllApplicationsRequestProto) returns (GetAllApplicationsResponseProto);
rpc getApplications (GetApplicationsRequestProto) returns (GetApplicationsResponseProto);
rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto);
rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto);
rpc getQueueUserAcls (GetQueueUserAclsInfoRequestProto) returns (GetQueueUserAclsInfoResponseProto);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
View File

@ -123,10 +123,11 @@ message GetClusterMetricsResponseProto {
optional YarnClusterMetricsProto cluster_metrics = 1;
}
message GetAllApplicationsRequestProto {
message GetApplicationsRequestProto {
repeated string application_types = 1;
}
message GetAllApplicationsResponseProto {
message GetApplicationsResponseProto {
repeated ApplicationReportProto applications = 1;
}

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -174,8 +175,28 @@ public abstract class YarnClient extends AbstractService {
* @throws YarnException
* @throws IOException
*/
public abstract List<ApplicationReport> getApplicationList() throws YarnException,
IOException;
public abstract List<ApplicationReport> getApplications()
throws YarnException, IOException;
/**
* <p>
* Get a report (ApplicationReport) of Applications
* matching the given application types in the cluster.
* </p>
*
* <p>
* If the user does not have <code>VIEW_APP</code> access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
* </p>
*
* @param applicationTypes
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException;
/**
* <p>

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -32,8 +33,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -206,11 +207,18 @@ public class YarnClientImpl extends YarnClient {
}
@Override
public List<ApplicationReport> getApplicationList()
throws YarnException, IOException {
GetAllApplicationsRequest request =
Records.newRecord(GetAllApplicationsRequest.class);
GetAllApplicationsResponse response = rmClient.getAllApplications(request);
public List<ApplicationReport> getApplications() throws YarnException,
IOException {
return getApplications(null);
}
@Override
public List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException {
GetApplicationsRequest request =
applicationTypes == null ? GetApplicationsRequest.newInstance()
: GetApplicationsRequest.newInstance(applicationTypes);
GetApplicationsResponse response = rmClient.getApplications(request);
return response.getApplicationList();
}

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -21,11 +21,14 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@ -35,6 +38,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
@Private
@Unstable
public class ApplicationCLI extends YarnCLI {
@ -42,6 +47,8 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI();
cli.setSysOutPrintStream(System.out);
@ -56,8 +63,19 @@ public class ApplicationCLI extends YarnCLI {
Options opts = new Options();
opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
opts.addOption(LIST_CMD, false, "Lists all the Applications from RM.");
opts.addOption(LIST_CMD, false, "List applications from the RM. " +
"Supports optional use of --appTypes to filter applications " +
"based on application type.");
opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
Option appTypeOpt = new Option(APP_TYPE_CMD, true,
"Works with --list to filter applications based on their type.");
appTypeOpt.setValueSeparator(',');
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
appTypeOpt.setArgName("Comma-separated list of application types");
opts.addOption(appTypeOpt);
opts.getOption(KILL_CMD).setArgName("Application ID");
opts.getOption(STATUS_CMD).setArgName("Application ID");
CommandLine cliParser = new GnuParser().parse(opts, args);
int exitCode = -1;
@ -68,13 +86,27 @@ public class ApplicationCLI extends YarnCLI {
}
printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
} else if (cliParser.hasOption(LIST_CMD)) {
listAllApplications();
Set<String> appTypes = new HashSet<String>();
if(cliParser.hasOption(APP_TYPE_CMD)) {
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
if (types != null) {
for (String type : types) {
if (!type.trim().isEmpty()) {
appTypes.add(type.trim());
}
}
}
}
listApplications(appTypes);
} else if (cliParser.hasOption(KILL_CMD)) {
if (args.length != 2) {
printUsage(opts);
return exitCode;
}
killApplication(cliParser.getOptionValue(KILL_CMD));
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
} else {
syserr.println("Invalid Command Usage : ");
printUsage(opts);
@ -87,19 +119,24 @@ public class ApplicationCLI extends YarnCLI {
*
* @param opts
*/
private void printUsage(Options opts) {
@VisibleForTesting
void printUsage(Options opts) {
new HelpFormatter().printHelp("application", opts);
}
/**
* Lists all the applications present in the Resource Manager
* Lists the applications matching the given application Types
* present in the Resource Manager
*
* @param appTypes
* @throws YarnException
* @throws IOException
*/
private void listAllApplications() throws YarnException, IOException {
private void listApplications(Set<String> appTypes)
throws YarnException, IOException {
PrintWriter writer = new PrintWriter(sysout);
List<ApplicationReport> appsReport = client.getApplicationList();
List<ApplicationReport> appsReport =
client.getApplications(appTypes);
writer.println("Total Applications:" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id",

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java
View File

@ -33,6 +33,7 @@ public abstract class YarnCLI extends Configured implements Tool {
public static final String STATUS_CMD = "status";
public static final String LIST_CMD = "list";
public static final String KILL_CMD = "kill";
public static final String HELP_CMD = "help";
protected PrintStream sysout;
protected PrintStream syserr;
protected YarnClient client;

109
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
View File

@ -25,7 +25,11 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import junit.framework.Assert;
@ -33,10 +37,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
@ -138,12 +146,44 @@ public class TestYarnClient {
Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType());
rm.stop();
}
@Test (timeout = 10000)
public void testGetApplications() throws YarnException, IOException {
Configuration conf = new Configuration();
final YarnClient client = new MockYarnClient();
client.init(conf);
client.start();
List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports();
Set<String> appTypes = new HashSet<String>();
appTypes.add("YARN");
appTypes.add("NON-YARN");
List<ApplicationReport> reports = client.getApplications(appTypes);
Assert.assertEquals(reports.size(), 2);
Assert
.assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports
.get(1).getApplicationType().equals("NON-YARN"))
|| (reports.get(1).getApplicationType().equals("YARN") && reports
.get(0).getApplicationType().equals("NON-YARN")));
for(ApplicationReport report : reports) {
Assert.assertTrue(expectedReports.contains(report));
}
reports = client.getApplications();
Assert.assertEquals(reports, expectedReports);
client.stop();
}
private static class MockYarnClient extends YarnClientImpl {
private ApplicationReport mockReport;
private List<ApplicationReport> reports;
public MockYarnClient() {
super();
reports = createAppReports();
}
@Override
@ -163,6 +203,20 @@ public class TestYarnClient {
when(mockResponse.getApplicationReport()).thenReturn(mockReport);
}
@Override
public List<ApplicationReport> getApplications(
Set<String> applicationTypes) throws YarnException, IOException {
GetApplicationsRequest request =
applicationTypes == null ? GetApplicationsRequest.newInstance()
: GetApplicationsRequest.newInstance(applicationTypes);
when(rmClient.getApplications(request))
.thenReturn(
getApplicationReports(reports,
request));
GetApplicationsResponse response = rmClient.getApplications(request);
return response.getApplicationList();
}
@Override
public void stop() {
}
@ -172,6 +226,59 @@ public class TestYarnClient {
YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING,
YarnApplicationState.NEW_SAVING, state);
}
public List<ApplicationReport> getReports() {
return this.reports;
}
private List<ApplicationReport> createAppReports() {
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN");
List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2),
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN");
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE");
applicationReports.add(newApplicationReport3);
return applicationReports;
}
private GetApplicationsResponse getApplicationReports(
List<ApplicationReport> applicationReports,
GetApplicationsRequest request) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
Set<String> appTypes = request.getApplicationTypes();
boolean bypassFilter = appTypes.isEmpty();
for (ApplicationReport appReport : applicationReports) {
if (!(bypassFilter || appTypes.contains(
appReport.getApplicationType()))) {
continue;
}
appReports.add(appReport);
}
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(appReports);
return response;
}
}
}

165
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -32,7 +32,9 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import junit.framework.Assert;
@ -51,6 +53,8 @@ import org.apache.hadoop.yarn.util.Records;
import org.junit.Before;
import org.junit.Test;
import org.apache.commons.cli.Options;
public class TestYarnCLI {
private YarnClient client = mock(YarnClient.class);
@ -105,7 +109,7 @@ public class TestYarnCLI {
}
@Test
public void testGetAllApplications() throws Exception {
public void testGetApplications() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
@ -115,10 +119,31 @@ public class TestYarnCLI {
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN");
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
when(client.getApplicationList()).thenReturn(applicationReports);
int result = cli.run(new String[] { "-list" });
ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2),
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN");
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE");
applicationReports.add(newApplicationReport3);
Set<String> appType1 = new HashSet<String>();
appType1.add("YARN");
when(client.getApplications(appType1)).thenReturn(
getApplicationReports(applicationReports, appType1));
int result = cli.run(new String[] { "-list", "-appTypes", "YARN" });
assertEquals(0, result);
verify(client).getApplicationList();
verify(client).getApplications(appType1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
@ -137,6 +162,138 @@ public class TestYarnCLI {
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
Set<String> appType2 = new HashSet<String>();
appType2.add("YARN");
appType2.add("FOO-YARN");
when(client.getApplications(appType2)).thenReturn(
getApplicationReports(applicationReports, appType2));
cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN",
",,,,, YARN,," });
assertEquals(0, result);
verify(client).getApplications(appType2);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total Applications:1");
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
Set<String> appType3 = new HashSet<String>();
appType3.add("YARN");
appType3.add("NON-YARN");
when(client.getApplications(appType3)).thenReturn(
getApplicationReports(applicationReports, appType3));
result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" });
assertEquals(0, result);
verify(client).getApplications(appType3);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total Applications:2");
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.print(" application_1234_0006\t ");
pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue2\t FINISHED\t ");
pw.print("SUCCEEDED\t 63.79%");
pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
Set<String> appType4 = new HashSet<String>();
when(client.getApplications(appType4)).thenReturn(
getApplicationReports(applicationReports, appType4));
result = cli.run(new String[] { "-list" });
assertEquals(0, result);
verify(client).getApplications(appType4);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total Applications:3");
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
pw.print("queue\t FINISHED\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.print(" application_1234_0006\t ");
pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue2\t FINISHED\t ");
pw.print("SUCCEEDED\t 63.79%");
pw.println("\t N/A");
pw.print(" application_1234_0007\t ");
pw.print("appname3\t MAPREDUCE\t user3\t ");
pw.print("queue3\t FINISHED\t ");
pw.print("SUCCEEDED\t 73.79%");
pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
}
private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports,
Set<String> appTypes) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
boolean bypassFilter = appTypes.isEmpty();
for (ApplicationReport appReport : applicationReports) {
if (!(bypassFilter || appTypes.contains(
appReport.getApplicationType()))) {
continue;
}
appReports.add(appReport);
}
return appReports;
}
@Test (timeout = 10000)
public void testHelpCommand() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationCLI spyCli = spy(cli);
int result = spyCli.run(new String[] { "-help" });
Assert.assertTrue(result == 0);
verify(spyCli).printUsage(any(Options.class));
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
result =
cli.run(new String[] { "-kill", applicationId.toString(), "args" });
verify(spyCli).printUsage(any(Options.class));
NodeId nodeId = NodeId.newInstance("host0", 0);
result = cli.run(new String[] { "-status", nodeId.toString(), "args" });
verify(spyCli).printUsage(any(Options.class));
}
@Test

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java
View File

@ -33,8 +33,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
@ -81,7 +81,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationReque
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto;
@ -188,13 +188,13 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP
}
@Override
public GetAllApplicationsResponse getAllApplications(
GetAllApplicationsRequest request) throws YarnException,
public GetApplicationsResponse getApplications(
GetApplicationsRequest request) throws YarnException,
IOException {
GetAllApplicationsRequestProto requestProto =
((GetAllApplicationsRequestPBImpl) request).getProto();
GetApplicationsRequestProto requestProto =
((GetApplicationsRequestPBImpl) request).getProto();
try {
return new GetAllApplicationsResponsePBImpl(proxy.getAllApplications(
return new GetApplicationsResponsePBImpl(proxy.getApplications(
null, requestProto));
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java
View File

@ -30,7 +30,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRespo
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRe
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto;
@ -170,14 +170,14 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient
}
@Override
public GetAllApplicationsResponseProto getAllApplications(
RpcController controller, GetAllApplicationsRequestProto proto)
public GetApplicationsResponseProto getApplications(
RpcController controller, GetApplicationsRequestProto proto)
throws ServiceException {
GetAllApplicationsRequestPBImpl request =
new GetAllApplicationsRequestPBImpl(proto);
GetApplicationsRequestPBImpl request =
new GetApplicationsRequestPBImpl(proto);
try {
GetAllApplicationsResponse response = real.getAllApplications(request);
return ((GetAllApplicationsResponsePBImpl)response).getProto();
GetApplicationsResponse response = real.getApplications(request);
return ((GetApplicationsResponsePBImpl)response).getProto();
} catch (YarnException e) {
throw new ServiceException(e);
} catch (IOException e) {

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java
View File

@ -1,67 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto;
@Private
@Unstable
public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest {
GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance();
GetAllApplicationsRequestProto.Builder builder = null;
boolean viaProto = false;
public GetAllApplicationsRequestPBImpl() {
builder = GetAllApplicationsRequestProto.newBuilder();
}
public GetAllApplicationsRequestPBImpl(GetAllApplicationsRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public GetAllApplicationsRequestProto getProto() {
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
}
}

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Normal file
View File

@ -0,0 +1,128 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
@Private
@Unstable
public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance();
GetApplicationsRequestProto.Builder builder = null;
boolean viaProto = false;
Set<String> applicationTypes = null;
public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder();
}
public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public GetApplicationsRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void mergeLocalToBuilder() {
if (this.applicationTypes != null) {
addLocalApplicationTypesToProto();
}
}
private void addLocalApplicationTypesToProto() {
maybeInitBuilder();
builder.clearApplicationTypes();
if (this.applicationTypes == null)
return;
builder.addAllApplicationTypes(applicationTypes);
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = GetApplicationsRequestProto.newBuilder(proto);
}
viaProto = false;
}
private void initApplicationTypes() {
if (this.applicationTypes != null) {
return;
}
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
List<String> appTypeList = p.getApplicationTypesList();
this.applicationTypes = new HashSet<String>();
this.applicationTypes.addAll(appTypeList);
}
@Override
public Set<String> getApplicationTypes() {
initApplicationTypes();
return this.applicationTypes;
}
@Override
public void setApplicationTypes(Set<String> applicationTypes) {
maybeInitBuilder();
if (applicationTypes == null)
builder.clearApplicationTypes();
this.applicationTypes = applicationTypes;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
}
}

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java
View File

@ -24,36 +24,36 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProtoOrBuilder;
@Private
@Unstable
public class GetAllApplicationsResponsePBImpl
extends GetAllApplicationsResponse {
public class GetApplicationsResponsePBImpl
extends GetApplicationsResponse {
GetAllApplicationsResponseProto proto =
GetAllApplicationsResponseProto.getDefaultInstance();
GetAllApplicationsResponseProto.Builder builder = null;
GetApplicationsResponseProto proto =
GetApplicationsResponseProto.getDefaultInstance();
GetApplicationsResponseProto.Builder builder = null;
boolean viaProto = false;
List<ApplicationReport> applicationList;
public GetAllApplicationsResponsePBImpl() {
builder = GetAllApplicationsResponseProto.newBuilder();
public GetApplicationsResponsePBImpl() {
builder = GetApplicationsResponseProto.newBuilder();
}
public GetAllApplicationsResponsePBImpl(GetAllApplicationsResponseProto proto) {
public GetApplicationsResponsePBImpl(GetApplicationsResponseProto proto) {
this.proto = proto;
viaProto = true;
}
@Override
public List<ApplicationReport> getApplicationList() {
public List<ApplicationReport> getApplicationList() {
initLocalApplicationsList();
return this.applicationList;
}
@ -61,12 +61,12 @@ extends GetAllApplicationsResponse {
@Override
public void setApplicationList(List<ApplicationReport> applications) {
maybeInitBuilder();
if (applications == null)
if (applications == null)
builder.clearApplications();
this.applicationList = applications;
}
public GetAllApplicationsResponseProto getProto() {
public GetApplicationsResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@ -100,7 +100,7 @@ extends GetAllApplicationsResponse {
}
private void mergeLocalToProto() {
if (viaProto)
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
@ -109,7 +109,7 @@ extends GetAllApplicationsResponse {
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = GetAllApplicationsResponseProto.newBuilder(proto);
builder = GetApplicationsResponseProto.newBuilder(proto);
}
viaProto = false;
}
@ -120,7 +120,7 @@ extends GetAllApplicationsResponse {
if (this.applicationList != null) {
return;
}
GetAllApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder;
GetApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder;
List<ApplicationReportProto> list = p.getApplicationsList();
applicationList = new ArrayList<ApplicationReport>();

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@ -42,8 +43,8 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@ -391,8 +392,8 @@ public class ClientRMService extends AbstractService implements
}
@Override
public GetAllApplicationsResponse getAllApplications(
GetAllApplicationsRequest request) throws YarnException {
public GetApplicationsResponse getApplications(
GetApplicationsRequest request) throws YarnException {
UserGroupInformation callerUGI;
try {
@ -402,15 +403,21 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
Set<String> applicationTypes = request.getApplicationTypes();
boolean bypassFilter = applicationTypes.isEmpty();
List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
for (RMApp application : this.rmContext.getRMApps().values()) {
if (!(bypassFilter || applicationTypes.contains(application
.getApplicationType()))) {
continue;
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
ApplicationAccessType.VIEW_APP, application.getApplicationId());
reports.add(application.createAndGetApplicationReport(allowAccess));
}
GetAllApplicationsResponse response =
recordFactory.newRecordInstance(GetAllApplicationsResponse.class);
GetApplicationsResponse response =
recordFactory.newRecordInstance(GetApplicationsResponse.class);
response.setApplicationList(reports);
return response;
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
View File

@ -34,7 +34,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
@ -213,8 +213,8 @@ public class TestApplicationACLs {
// List apps as owner
Assert.assertEquals("App view by owner should list the apps!!", 1,
rmClient.getAllApplications(
recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
rmClient.getApplications(
recordFactory.newRecordInstance(GetApplicationsRequest.class))
.getApplicationList().size());
// Kill app as owner
@ -244,8 +244,8 @@ public class TestApplicationACLs {
// List apps as superUser
Assert.assertEquals("App view by super-user should list the apps!!", 2,
superUserClient.getAllApplications(
recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
superUserClient.getApplications(
recordFactory.newRecordInstance(GetApplicationsRequest.class))
.getApplicationList().size());
// Kill app as the superUser
@ -275,8 +275,8 @@ public class TestApplicationACLs {
// List apps as friend
Assert.assertEquals("App view by a friend should list the apps!!", 3,
friendClient.getAllApplications(
recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
friendClient.getApplications(
recordFactory.newRecordInstance(GetApplicationsRequest.class))
.getApplicationList().size());
// Kill app as the friend
@ -308,8 +308,8 @@ public class TestApplicationACLs {
// List apps as enemy
List<ApplicationReport> appReports = enemyRmClient
.getAllApplications(recordFactory
.newRecordInstance(GetAllApplicationsRequest.class))
.getApplications(recordFactory
.newRecordInstance(GetApplicationsRequest.class))
.getApplicationList();
Assert.assertEquals("App view by enemy should list the apps!!", 4,
appReports.size());

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -26,8 +26,10 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.HashSet;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
@ -42,6 +44,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@ -49,6 +53,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -88,6 +93,8 @@ public class TestClientRMService {
private RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
private String appType = "MockApp";
private static RMDelegationTokenSecretManager dtsm;
@BeforeClass
@ -292,11 +299,18 @@ public class TestClientRMService {
new EventHandler<Event>() {
public void handle(Event event) {}
});
ApplicationId appId1 = getApplicationId(100);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
when(
mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager, null, null);
new ClientRMService(rmContext, yarnScheduler, appManager,
mockAclsManager, null);
// without name and queue
ApplicationId appId1 = getApplicationId(100);
SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(
appId1, null, null);
try {
@ -317,6 +331,8 @@ public class TestClientRMService {
ApplicationId appId2 = getApplicationId(101);
SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(
appId2, name, queue);
submitRequest2.getApplicationSubmissionContext().setApplicationType(
"matchType");
try {
rmService.submitApplication(submitRequest2);
} catch (YarnException e) {
@ -335,6 +351,25 @@ public class TestClientRMService {
Assert.assertTrue("The thrown exception is not expected.",
e.getMessage().contains("Cannot add a duplicate!"));
}
GetApplicationsRequest getAllAppsRequest =
GetApplicationsRequest.newInstance(new HashSet<String>());
GetApplicationsResponse getAllApplicationsResponse =
rmService.getApplications(getAllAppsRequest);
Assert.assertEquals(5,
getAllApplicationsResponse.getApplicationList().size());
Set<String> appTypes = new HashSet<String>();
appTypes.add("matchType");
getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes);
getAllApplicationsResponse =
rmService.getApplications(getAllAppsRequest);
Assert.assertEquals(1,
getAllApplicationsResponse.getApplicationList().size());
Assert.assertEquals(appId2,
getAllApplicationsResponse.getApplicationList()
.get(0).getApplicationId());
}
@Test(timeout=4000)
@ -416,6 +451,7 @@ public class TestClientRMService {
submissionContext.setQueue(queue);
submissionContext.setApplicationId(appId);
submissionContext.setResource(resource);
submissionContext.setApplicationType(appType);
SubmitApplicationRequest submitRequest =
recordFactory.newRecordInstance(SubmitApplicationRequest.class);