From b7a3d0ae9398122639265b5a88e7824c352b9eef Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 28 Jan 2014 20:00:38 +0000 Subject: [PATCH] YARN-967. Added the client and CLI interfaces for obtaining ApplicationHistory data. Contributed by Mayank Bansal. svn merge --ignore-ancestry -c 1556747 ../YARN-321 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562202 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapred/ResourceMgrDelegate.java | 29 ++ hadoop-yarn-project/CHANGES.txt | 3 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 7 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 18 +- .../ApplicationAttemptNotFoundException.java | 49 +++ .../ApplicationNotFoundException.java | 9 +- .../ContainerNotFoundException.java | 49 +++ .../hadoop/yarn/client/api/AHSClient.java | 179 ++++++++ .../hadoop/yarn/client/api/YarnClient.java | 76 ++++ .../yarn/client/api/impl/AHSClientImpl.java | 156 +++++++ .../yarn/client/api/impl/YarnClientImpl.java | 57 ++- .../yarn/client/cli/ApplicationCLI.java | 313 +++++++++++--- .../yarn/client/api/impl/TestAHSClient.java | 394 ++++++++++++++++++ .../yarn/client/api/impl/TestYarnClient.java | 2 - .../hadoop/yarn/client/cli/TestYarnCLI.java | 176 +++++++- .../apache/hadoop/yarn/client/AHSProxy.java | 57 +++ .../ApplicationHistoryClientService.java | 37 +- 17 files changed, 1515 insertions(+), 96 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 74b07c2f3db..fb3e47de353 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -44,9 +44,13 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.NodeState; @@ -371,4 +375,29 @@ public class ResourceMgrDelegate extends YarnClient { IOException { return client.getQueueAclsInfo(); } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId) throws YarnException, IOException { + return client.getApplicationAttemptReport(appAttemptId); + } + + @Override + public List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException { + return client.getApplicationAttempts(appId); + } + + @Override + public ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException { + return client.getContainerReport(containerId); + } + + @Override + public List getContainers( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException { + return client.getContainers(applicationAttemptId); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8ffca797ee1..98c834207b0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -508,6 +508,9 @@ Branch YARN-321: Generic ApplicationHistoryService YARN-954. Implemented web UI for the ApplicationHistoryServer and wired it into the HistoryStorage. (Zhijie Shen via vinodkv) + YARN-967. Added the client and CLI interfaces for obtaining ApplicationHistory + data. (Mayank Bansal via vinodkv) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 41f2c40083b..826c78c9ac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -63,6 +63,8 @@ function print_usage(){ echo " version print the version" echo " jar run a jar file" echo " application prints application(s) report/kill application" + echo " applicationattempt prints applicationattempt(s) report" + echo " container prints container(s) report" echo " node prints node report(s)" echo " logs dump container logs" echo " classpath prints the class path needed to get the" @@ -182,9 +184,12 @@ if [ "$COMMAND" = "classpath" ] ; then elif [ "$COMMAND" = "rmadmin" ] ; then CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI' YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "application" ] ; then +elif [ "$COMMAND" = "application" ] || + [ "$COMMAND" = "applicationattempt" ] || + [ "$COMMAND" = "container" ]; then CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" + set -- $COMMAND $@ elif [ "$COMMAND" = "node" ] ; then CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index bff142e664c..121f864f838 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -141,7 +141,8 @@ if "%1" == "--config" ( goto :eof ) - set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar application node logs daemonlog + set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^ + application applicationattempt container node logs daemonlog historyserver for %%i in ( %yarncommands% ) do ( if %yarn-command% == %%i set yarncommand=true ) @@ -173,8 +174,21 @@ goto :eof :application set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + set yarn-command-arguments=%yarn-command% %yarn-command-arguments% goto :eof +:applicationattempt + set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + set yarn-command-arguments=%yarn-command% %yarn-command-arguments% + goto :eof + +:container + set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + set yarn-command-arguments=%yarn-command% %yarn-command-arguments% + goto :eof + :node set CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% @@ -268,6 +282,8 @@ goto :eof @echo version print the version @echo jar ^ run a jar file @echo application prints application(s) report/kill application + @echo applicationattempt prints applicationattempt(s) report + @echo container prints container(s) report @echo node prints node report(s) @echo logs dump container logs @echo classpath prints the class path needed to get the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java new file mode 100644 index 00000000000..e54f909b144 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; + +/** + * This exception is thrown on + * {@link ApplicationHistoryProtocol#getApplicationAttemptReport + * (GetApplicationAttemptReportRequest)} + * API when the Application Attempt doesn't exist in Application History Server + */ +@Public +@Unstable +public class ApplicationAttemptNotFoundException extends YarnException { + + private static final long serialVersionUID = 8694508L; + + public ApplicationAttemptNotFoundException(Throwable cause) { + super(cause); + } + + public ApplicationAttemptNotFoundException(String message) { + super(message); + } + + public ApplicationAttemptNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java index 8f9a9e25715..da83c397e8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java @@ -18,14 +18,19 @@ package org.apache.hadoop.yarn.exceptions; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; /** * This exception is thrown on - * {@link ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)} API - * when the Application doesn't exist in RM + * {@link ApplicationClientProtocol#getApplicationReport + * (GetApplicationReportRequest)} API + * when the Application doesn't exist in RM and AHS */ +@Public +@Unstable public class ApplicationNotFoundException extends YarnException{ private static final long serialVersionUID = 8694408L; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java new file mode 100644 index 00000000000..aba467a3873 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; + +/** + * This exception is thrown on + * {@link ApplicationHistoryProtocol#getContainerReport + * (GetContainerReportRequest)} + * API when the container doesn't exist in AHS + */ +@Public +@Unstable +public class ContainerNotFoundException extends YarnException { + + private static final long serialVersionUID = 8694608L; + + public ContainerNotFoundException(Throwable cause) { + super(cause); + } + + public ContainerNotFoundException(String message) { + super(message); + } + + public ContainerNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java new file mode 100644 index 00000000000..ebfd95a7e14 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.client.api.impl.AHSClientImpl; +import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@InterfaceAudience.Public +@InterfaceStability.Stable +public abstract class AHSClient extends AbstractService { + + /** + * Create a new instance of AHSClient. + */ + @Public + public static AHSClient createAHSClient() { + AHSClient client = new AHSClientImpl(); + return client; + } + + @Private + public AHSClient(String name) { + super(name); + } + + /** + *

+ * Get a report of the given Application. + *

+ * + *

+ * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

+ * + *

+ * If the user does not have VIEW_APP access then the following + * fields in the report will be set to stubbed values: + *

    + *
  • host - set to "N/A"
    • + *
  • RPC port - set to -1
    • + *
  • client token - set to "N/A"
    • + *
  • diagnostics - set to "N/A"
    • + *
  • tracking URL - set to "N/A"
    • + *
  • original tracking URL - set to "N/A"
    • + *
  • resource usage report - all values are -1
    • + *
+ *

+ * + * @param appId + * {@link ApplicationId} of the application that needs a report + * @return application report + * @throws YarnException + * @throws IOException + */ + public abstract ApplicationReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException; + + /** + *

+ * Get a report (ApplicationReport) of all Applications in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @return a list of reports for all applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplications() + throws YarnException, IOException; + + /** + *

+ * Get a report of the given ApplicationAttempt. + *

+ * + *

+ * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

+ * + * @param applicationAttemptId + * {@link ApplicationAttemptId} of the application attempt that needs + * a report + * @return application attempt report + * @throws YarnException + * @throws {@link ApplicationAttemptNotFoundException} if application attempt + * not found + * @throws IOException + */ + public abstract ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId) throws YarnException, IOException; + + /** + *

+ * Get a report of all (ApplicationAttempts) of Application in the cluster. + *

+ * + * @param applicationId + * @return a list of reports for all application attempts for specified + * application + * @throws YarnException + * @throws IOException + */ + public abstract List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException; + + /** + *

+ * Get a report of the given Container. + *

+ * + *

+ * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

+ * + * @param containerId + * {@link ContainerId} of the container that needs a report + * @return container report + * @throws YarnException + * @throws {@link ContainerNotFoundException} if container not found + * @throws IOException + */ + public abstract ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException; + + /** + *

+ * Get a report of all (Containers) of ApplicationAttempt in the cluster. + *

+ * + * @param applicationAttemptId + * @return a list of reports of all containers for specified application + * attempt + * @throws YarnException + * @throws IOException + */ + public abstract List getContainers( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 155ba5d51a5..d173cdf23a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -29,9 +29,13 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -40,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -360,4 +365,75 @@ public abstract class YarnClient extends AbstractService { */ public abstract List getQueueAclsInfo() throws YarnException, IOException; + + /** + *

+ * Get a report of the given ApplicationAttempt. + *

+ * + *

+ * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

+ * + * @param applicationAttemptId + * {@link ApplicationAttemptId} of the application attempt that needs + * a report + * @return application attempt report + * @throws YarnException + * @throws {@link ApplicationAttemptNotFoundException} if application attempt + * not found + * @throws IOException + */ + public abstract ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId) throws YarnException, IOException; + + /** + *

+ * Get a report of all (ApplicationAttempts) of Application in the cluster. + *

+ * + * @param applicationId + * @return a list of reports for all application attempts for specified + * application. + * @throws YarnException + * @throws IOException + */ + public abstract List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException; + + /** + *

+ * Get a report of the given Container. + *

+ * + *

+ * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

+ * + * @param containerId + * {@link ContainerId} of the container that needs a report + * @return container report + * @throws YarnException + * @throws {@link ContainerNotFoundException} if container not found. + * @throws IOException + */ + public abstract ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException; + + /** + *

+ * Get a report of all (Containers) of ApplicationAttempt in the cluster. + *

+ * + * @param applicationAttemptId + * @return a list of reports of all containers for specified application + * attempts + * @throws YarnException + * @throws IOException + */ + public abstract List getContainers( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java new file mode 100644 index 00000000000..507cc11ef27 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.client.AHSProxy; +import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.util.Records; + +@Private +@Unstable +public class AHSClientImpl extends AHSClient { + + protected ApplicationHistoryProtocol ahsClient; + protected InetSocketAddress ahsAddress; + + public AHSClientImpl() { + super(AHSClientImpl.class.getName()); + } + + private static InetSocketAddress getAHSAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS, + YarnConfiguration.DEFAULT_AHS_ADDRESS, + YarnConfiguration.DEFAULT_AHS_PORT); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.ahsAddress = getAHSAddress(conf); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + try { + ahsClient = AHSProxy.createAHSProxy(getConfig(), + ApplicationHistoryProtocol.class, this.ahsAddress); + } catch (IOException e) { + throw new YarnRuntimeException(e); + } + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.ahsClient != null) { + RPC.stopProxy(this.ahsClient); + } + super.serviceStop(); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + GetApplicationReportRequest request = GetApplicationReportRequest + .newInstance(appId); + GetApplicationReportResponse response = ahsClient + .getApplicationReport(request); + return response.getApplicationReport(); + } + + @Override + public List getApplications() throws YarnException, + IOException { + GetApplicationsRequest request = GetApplicationsRequest.newInstance(null, + null); + GetApplicationsResponse response = ahsClient.getApplications(request); + return response.getApplicationList(); + } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException { + GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest + .newInstance(applicationAttemptId); + GetApplicationAttemptReportResponse response = ahsClient + .getApplicationAttemptReport(request); + return response.getApplicationAttemptReport(); + } + + @Override + public List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException { + GetApplicationAttemptsRequest request = GetApplicationAttemptsRequest + .newInstance(appId); + GetApplicationAttemptsResponse response = ahsClient + .getApplicationAttempts(request); + return response.getApplicationAttemptList(); + } + + @Override + public ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException { + GetContainerReportRequest request = GetContainerReportRequest + .newInstance(containerId); + GetContainerReportResponse response = ahsClient.getContainerReport(request); + return response.getContainerReport(); + } + + @Override + public List getContainers( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException { + GetContainersRequest request = GetContainersRequest + .newInstance(applicationAttemptId); + GetContainersResponse response = ahsClient.getContainers(request); + return response.getContainerList(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index a5ff9f67dc9..23a62de68ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -49,9 +49,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -60,9 +64,11 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.AHSClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -80,6 +86,7 @@ public class YarnClientImpl extends YarnClient { protected ApplicationClientProtocol rmClient; protected long submitPollIntervalMillis; private long asyncApiPollIntervalMillis; + protected AHSClient historyClient; private static final String ROOT = "root"; @@ -100,6 +107,8 @@ public class YarnClientImpl extends YarnClient { YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS); } + historyClient = AHSClientImpl.createAHSClient(); + historyClient.init(getConfig()); super.serviceInit(conf); } @@ -107,7 +116,8 @@ public class YarnClientImpl extends YarnClient { protected void serviceStart() throws Exception { try { rmClient = ClientRMProxy.createRMProxy(getConfig(), - ApplicationClientProtocol.class); + ApplicationClientProtocol.class); + historyClient.start(); } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -119,6 +129,7 @@ public class YarnClientImpl extends YarnClient { if (this.rmClient != null) { RPC.stopProxy(this.rmClient); } + historyClient.stop(); super.serviceStop(); } @@ -207,11 +218,20 @@ public class YarnClientImpl extends YarnClient { @Override public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException { - GetApplicationReportRequest request = - Records.newRecord(GetApplicationReportRequest.class); - request.setApplicationId(appId); - GetApplicationReportResponse response = - rmClient.getApplicationReport(request); + GetApplicationReportResponse response = null; + try { + GetApplicationReportRequest request = Records + .newRecord(GetApplicationReportRequest.class); + request.setApplicationId(appId); + response = rmClient.getApplicationReport(request); + } catch (YarnException e) { + if (!(e.getClass() == ApplicationNotFoundException.class)) { + throw e; + } + } + if (response == null || response.getApplicationReport() == null) { + return historyClient.getApplicationReport(appId); + } return response.getApplicationReport(); } @@ -373,4 +393,29 @@ public class YarnClientImpl extends YarnClient { public void setRMClient(ApplicationClientProtocol rmClient) { this.rmClient = rmClient; } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId) throws YarnException, IOException { + return historyClient.getApplicationAttemptReport(appAttemptId); + } + + @Override + public List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException { + return historyClient.getApplicationAttempts(appId); + } + + @Override + public ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException { + return historyClient.getContainerReport(containerId); + } + + @Override + public List getContainers( + ApplicationAttemptId applicationAttemptId) throws YarnException, + IOException { + return historyClient.getContainers(applicationAttemptId); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index a7b7d654643..9b465b78b28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -35,8 +35,10 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -46,13 +48,22 @@ import com.google.common.annotations.VisibleForTesting; @Private @Unstable public class ApplicationCLI extends YarnCLI { - private static final String APPLICATIONS_PATTERN = - "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + - System.getProperty("line.separator"); + private static final String APPLICATIONS_PATTERN = + "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + + System.getProperty("line.separator"); + private static final String APPLICATION_ATTEMPTS_PATTERN = + "%30s\t%20s\t%35s\t%35s" + + System.getProperty("line.separator"); + private static final String CONTAINER_PATTERN = + "%30s\t%20s\t%20s\t%20s\t%20s\t%35s" + + System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; - private static final String APP_STATE_CMD ="appStates"; + private static final String APP_STATE_CMD = "appStates"; private static final String ALLSTATES_OPTION = "ALL"; + public static final String APPLICATION = "application"; + public static final String APPLICATION_ATTEMPT = "applicationattempt"; + public static final String CONTAINER = "container"; private boolean allAppStates; @@ -69,23 +80,33 @@ public class ApplicationCLI extends YarnCLI { public int run(String[] args) throws Exception { Options opts = new Options(); - opts.addOption(STATUS_CMD, true, "Prints the status of the application."); - opts.addOption(LIST_CMD, false, "List applications from the RM. " + - "Supports optional use of -appTypes to filter applications " + - "based on application type, " + - "and -appStates to filter applications based on application state"); + opts.addOption(STATUS_CMD, true, + "Prints the status of the application."); + if (args.length > 0 + && args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) { + opts.addOption(LIST_CMD, true, + "List application attempts for aplication from AHS. "); + } else if (args.length > 0 && args[0].compareToIgnoreCase("container") == 0) { + opts.addOption(LIST_CMD, true, + "List containers for application attempts from AHS. "); + } else { + opts.addOption(LIST_CMD, false, "List applications from the RM. " + + "Supports optional use of -appTypes to filter applications " + + "based on application type, " + + "and -appStates to filter applications based on application state"); + } opts.addOption(KILL_CMD, true, "Kills the application."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); - Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + - "filter applications based on " + - "input comma-separated list of application types."); + Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + + "filter applications based on " + + "input comma-separated list of application types."); appTypeOpt.setValueSeparator(','); appTypeOpt.setArgs(Option.UNLIMITED_VALUES); appTypeOpt.setArgName("Types"); opts.addOption(appTypeOpt); - Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " + - "to filter applications based on input comma-separated list of " + - "application states. " + getAllValidApplicationStates()); + Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " + + "to filter applications based on input comma-separated list of " + + "application states. " + getAllValidApplicationStates()); appStateOpt.setValueSeparator(','); appStateOpt.setArgs(Option.UNLIMITED_VALUES); appStateOpt.setArgName("States"); @@ -104,50 +125,77 @@ public class ApplicationCLI extends YarnCLI { } if (cliParser.hasOption(STATUS_CMD)) { - if (args.length != 2) { + if ((args[0].compareToIgnoreCase(APPLICATION) == 0) + || (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) + || (args[0].compareToIgnoreCase(CONTAINER) == 0)) { + if (args.length != 3) { + printUsage(opts); + return exitCode; + } + } else if (args.length != 2) { printUsage(opts); return exitCode; } - printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); + if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) { + printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD)); + } else if (args[0].compareToIgnoreCase(CONTAINER) == 0) { + printContainerReport(cliParser.getOptionValue(STATUS_CMD)); + } else { + printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); + } } else if (cliParser.hasOption(LIST_CMD)) { - allAppStates = false; - Set appTypes = new HashSet(); - if(cliParser.hasOption(APP_TYPE_CMD)) { - String[] types = cliParser.getOptionValues(APP_TYPE_CMD); - if (types != null) { - for (String type : types) { - if (!type.trim().isEmpty()) { - appTypes.add(type.toUpperCase().trim()); + if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) { + if (args.length != 3) { + printUsage(opts); + return exitCode; + } + listApplicationAttempts(cliParser.getOptionValue(LIST_CMD)); + } else if (args[0].compareToIgnoreCase(CONTAINER) == 0) { + if (args.length != 3) { + printUsage(opts); + return exitCode; + } + listContainers(cliParser.getOptionValue(LIST_CMD)); + } else { + allAppStates = false; + Set appTypes = new HashSet(); + if (cliParser.hasOption(APP_TYPE_CMD)) { + String[] types = cliParser.getOptionValues(APP_TYPE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + appTypes.add(type.toUpperCase().trim()); + } } } } - } - EnumSet appStates = - EnumSet.noneOf(YarnApplicationState.class); - if (cliParser.hasOption(APP_STATE_CMD)) { - String[] states = cliParser.getOptionValues(APP_STATE_CMD); - if (states != null) { - for (String state : states) { - if (!state.trim().isEmpty()) { - if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) { - allAppStates = true; - break; - } - try { - appStates.add(YarnApplicationState.valueOf(state.toUpperCase() - .trim())); - } catch (IllegalArgumentException ex) { - sysout.println("The application state " + state - + " is invalid."); - sysout.println(getAllValidApplicationStates()); - return exitCode; + EnumSet appStates = EnumSet + .noneOf(YarnApplicationState.class); + if (cliParser.hasOption(APP_STATE_CMD)) { + String[] states = cliParser.getOptionValues(APP_STATE_CMD); + if (states != null) { + for (String state : states) { + if (!state.trim().isEmpty()) { + if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) { + allAppStates = true; + break; + } + try { + appStates.add(YarnApplicationState.valueOf(state + .toUpperCase().trim())); + } catch (IllegalArgumentException ex) { + sysout.println("The application state " + state + + " is invalid."); + sysout.println(getAllValidApplicationStates()); + return exitCode; + } } } } } + listApplications(appTypes, appStates); } - listApplications(appTypes, appStates); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); @@ -175,8 +223,85 @@ public class ApplicationCLI extends YarnCLI { } /** - * Lists the applications matching the given application Types - * And application States present in the Resource Manager + * Prints the application attempt report for an application attempt id. + * + * @param applicationAttemptId + * @throws YarnException + */ + private void printApplicationAttemptReport(String applicationAttemptId) + throws YarnException, IOException { + ApplicationAttemptReport appAttemptReport = client + .getApplicationAttemptReport(ConverterUtils + .toApplicationAttemptId(applicationAttemptId)); + // Use PrintWriter.println, which uses correct platform line ending. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter appAttemptReportStr = new PrintWriter(baos); + if (appAttemptReport != null) { + appAttemptReportStr.println("Application Attempt Report : "); + appAttemptReportStr.print("\tApplicationAttempt-Id : "); + appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId()); + appAttemptReportStr.print("\tState : "); + appAttemptReportStr.println(appAttemptReport + .getYarnApplicationAttemptState()); + appAttemptReportStr.print("\tAMContainer : "); + appAttemptReportStr.println(appAttemptReport.getAMContainerId() + .toString()); + appAttemptReportStr.print("\tTracking-URL : "); + appAttemptReportStr.println(appAttemptReport.getTrackingUrl()); + appAttemptReportStr.print("\tRPC Port : "); + appAttemptReportStr.println(appAttemptReport.getRpcPort()); + appAttemptReportStr.print("\tAM Host : "); + appAttemptReportStr.println(appAttemptReport.getHost()); + appAttemptReportStr.print("\tDiagnostics : "); + appAttemptReportStr.print(appAttemptReport.getDiagnostics()); + } else { + appAttemptReportStr.print("Application Attempt with id '" + + applicationAttemptId + "' doesn't exist in History Server."); + } + appAttemptReportStr.close(); + sysout.println(baos.toString("UTF-8")); + } + + /** + * Prints the container report for an container id. + * + * @param containerId + * @throws YarnException + */ + private void printContainerReport(String containerId) throws YarnException, + IOException { + ContainerReport containerReport = client.getContainerReport((ConverterUtils + .toContainerId(containerId))); + // Use PrintWriter.println, which uses correct platform line ending. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter containerReportStr = new PrintWriter(baos); + if (containerReport != null) { + containerReportStr.println("Container Report : "); + containerReportStr.print("\tContainer-Id : "); + containerReportStr.println(containerReport.getContainerId()); + containerReportStr.print("\tStart-Time : "); + containerReportStr.println(containerReport.getStartTime()); + containerReportStr.print("\tFinish-Time : "); + containerReportStr.println(containerReport.getFinishTime()); + containerReportStr.print("\tState : "); + containerReportStr.println(containerReport.getContainerState()); + containerReportStr.print("\tLOG-URL : "); + containerReportStr.println(containerReport.getLogUrl()); + containerReportStr.print("\tHost : "); + containerReportStr.println(containerReport.getAssignedNode()); + containerReportStr.print("\tDiagnostics : "); + containerReportStr.print(containerReport.getDiagnosticsInfo()); + } else { + containerReportStr.print("Container with id '" + containerId + + "' doesn't exist in Hostory Server."); + } + containerReportStr.close(); + sysout.println(baos.toString("UTF-8")); + } + + /** + * Lists the applications matching the given application Types And application + * States present in the Resource Manager * * @param appTypes * @param appStates @@ -188,7 +313,7 @@ public class ApplicationCLI extends YarnCLI { IOException { PrintWriter writer = new PrintWriter(sysout); if (allAppStates) { - for(YarnApplicationState appState : YarnApplicationState.values()) { + for (YarnApplicationState appState : YarnApplicationState.values()) { appStates.add(appState); } } else { @@ -199,23 +324,24 @@ public class ApplicationCLI extends YarnCLI { } } - List appsReport = - client.getApplications(appTypes, appStates); + List appsReport = client.getApplications(appTypes, + appStates); - writer - .println("Total number of applications (application-types: " + appTypes - + " and states: " + appStates + ")" + ":" + appsReport.size()); - writer.printf(APPLICATIONS_PATTERN, "Application-Id", - "Application-Name","Application-Type", "User", "Queue", - "State", "Final-State","Progress", "Tracking-URL"); + writer.println("Total number of applications (application-types: " + + appTypes + " and states: " + appStates + ")" + ":" + + appsReport.size()); + writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", + "Application-Type", "User", "Queue", "State", "Final-State", + "Progress", "Tracking-URL"); for (ApplicationReport appReport : appsReport) { DecimalFormat formatter = new DecimalFormat("###.##%"); String progress = formatter.format(appReport.getProgress()); writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), - appReport.getName(),appReport.getApplicationType(), appReport.getUser(), - appReport.getQueue(),appReport.getYarnApplicationState(), - appReport.getFinalApplicationStatus(),progress, - appReport.getOriginalTrackingUrl()); + appReport.getName(), appReport.getApplicationType(), appReport + .getUser(), appReport.getQueue(), appReport + .getYarnApplicationState(), + appReport.getFinalApplicationStatus(), progress, appReport + .getOriginalTrackingUrl()); } writer.flush(); } @@ -227,8 +353,8 @@ public class ApplicationCLI extends YarnCLI { * @throws YarnException * @throws IOException */ - private void killApplication(String applicationId) - throws YarnException, IOException { + private void killApplication(String applicationId) throws YarnException, + IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED @@ -296,14 +422,63 @@ public class ApplicationCLI extends YarnCLI { private String getAllValidApplicationStates() { StringBuilder sb = new StringBuilder(); - sb.append("The valid application state can be" - + " one of the following: "); + sb.append("The valid application state can be" + " one of the following: "); sb.append(ALLSTATES_OPTION + ","); - for (YarnApplicationState appState : YarnApplicationState - .values()) { - sb.append(appState+","); + for (YarnApplicationState appState : YarnApplicationState.values()) { + sb.append(appState + ","); } String output = sb.toString(); - return output.substring(0, output.length()-1); + return output.substring(0, output.length() - 1); + } + + /** + * Lists the application attempts matching the given applicationid + * + * @param applicationId + * @throws YarnException + * @throws IOException + */ + private void listApplicationAttempts(String appId) throws YarnException, + IOException { + PrintWriter writer = new PrintWriter(sysout); + + List appAttemptsReport = client + .getApplicationAttempts(ConverterUtils.toApplicationId(appId)); + writer.println("Total number of application attempts " + ":" + + appAttemptsReport.size()); + writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id", + "State", "AM-Container-Id", "Tracking-URL"); + for (ApplicationAttemptReport appAttemptReport : appAttemptsReport) { + writer.printf(APPLICATION_ATTEMPTS_PATTERN, appAttemptReport + .getApplicationAttemptId(), appAttemptReport + .getYarnApplicationAttemptState(), appAttemptReport + .getAMContainerId().toString(), appAttemptReport.getTrackingUrl()); + } + writer.flush(); + } + + /** + * Lists the containers matching the given application attempts + * + * @param appAttemptId + * @throws YarnException + * @throws IOException + */ + private void listContainers(String appAttemptId) throws YarnException, + IOException { + PrintWriter writer = new PrintWriter(sysout); + + List appsReport = client + .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId)); + writer.println("Total number of containers " + ":" + appsReport.size()); + writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time", + "Finish Time", "State", "Host", "LOG-URL"); + for (ContainerReport containerReport : appsReport) { + writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(), + containerReport.getStartTime(), containerReport.getFinishTime(), + containerReport.getContainerState(), containerReport + .getAssignedNode(), containerReport.getLogUrl()); + } + writer.flush(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java new file mode 100644 index 00000000000..2122b421c77 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -0,0 +1,394 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.junit.Test; + +public class TestAHSClient { + + @Test + public void testClientStop() { + Configuration conf = new Configuration(); + AHSClient client = AHSClient.createAHSClient(); + client.init(conf); + client.start(); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetApplications() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + List expectedReports = ((MockAHSClient) client) + .getReports(); + + List reports = client.getApplications(); + Assert.assertEquals(reports, expectedReports); + + reports = client.getApplications(); + Assert.assertEquals(reports.size(), 4); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetApplicationReport() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + List expectedReports = ((MockAHSClient) client) + .getReports(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationReport report = client.getApplicationReport(applicationId); + Assert.assertEquals(report, expectedReports.get(0)); + Assert.assertEquals(report.getApplicationId().toString(), expectedReports + .get(0).getApplicationId().toString()); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetApplicationAttempts() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + List reports = client + .getApplicationAttempts(applicationId); + Assert.assertNotNull(reports); + Assert.assertEquals(reports.get(0).getApplicationAttemptId(), + ApplicationAttemptId.newInstance(applicationId, 1)); + Assert.assertEquals(reports.get(1).getApplicationAttemptId(), + ApplicationAttemptId.newInstance(applicationId, 2)); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetApplicationAttempt() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + List expectedReports = ((MockAHSClient) client) + .getReports(); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ApplicationAttemptReport report = client + .getApplicationAttemptReport(appAttemptId); + Assert.assertNotNull(report); + Assert.assertEquals(report.getApplicationAttemptId().toString(), + expectedReports.get(0).getCurrentApplicationAttemptId().toString()); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetContainers() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + List reports = client.getContainers(appAttemptId); + Assert.assertNotNull(reports); + Assert.assertEquals(reports.get(0).getContainerId(), (ContainerId + .newInstance(appAttemptId, 1))); + Assert.assertEquals(reports.get(1).getContainerId(), (ContainerId + .newInstance(appAttemptId, 2))); + client.stop(); + } + + @Test(timeout = 10000) + public void testGetContainerReport() throws YarnException, IOException { + Configuration conf = new Configuration(); + final AHSClient client = new MockAHSClient(); + client.init(conf); + client.start(); + + List expectedReports = ((MockAHSClient) client) + .getReports(); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 1); + ContainerReport report = client.getContainerReport(containerId); + Assert.assertNotNull(report); + Assert.assertEquals(report.getContainerId().toString(), + (ContainerId.newInstance(expectedReports.get(0) + .getCurrentApplicationAttemptId(), 1)).toString()); + client.stop(); + } + + private static class MockAHSClient extends AHSClientImpl { + // private ApplicationReport mockReport; + private List reports = new ArrayList(); + private HashMap> attempts = + new HashMap>(); + private HashMap> containers = + new HashMap>(); + GetApplicationsResponse mockAppResponse = + mock(GetApplicationsResponse.class); + GetApplicationReportResponse mockResponse = + mock(GetApplicationReportResponse.class); + GetApplicationAttemptsResponse mockAppAttemptsResponse = + mock(GetApplicationAttemptsResponse.class); + GetApplicationAttemptReportResponse mockAttemptResponse = + mock(GetApplicationAttemptReportResponse.class); + GetContainersResponse mockContainersResponse = + mock(GetContainersResponse.class); + GetContainerReportResponse mockContainerResponse = + mock(GetContainerReportResponse.class); + + public MockAHSClient() { + super(); + createAppReports(); + } + + @Override + public void start() { + ahsClient = mock(ApplicationHistoryProtocol.class); + + try { + when( + ahsClient + .getApplicationReport(any(GetApplicationReportRequest.class))) + .thenReturn(mockResponse); + when(ahsClient.getApplications(any(GetApplicationsRequest.class))) + .thenReturn(mockAppResponse); + when( + ahsClient + .getApplicationAttemptReport(any(GetApplicationAttemptReportRequest.class))) + .thenReturn(mockAttemptResponse); + when( + ahsClient + .getApplicationAttempts(any(GetApplicationAttemptsRequest.class))) + .thenReturn(mockAppAttemptsResponse); + when(ahsClient.getContainers(any(GetContainersRequest.class))) + .thenReturn(mockContainersResponse); + + when(ahsClient.getContainerReport(any(GetContainerReportRequest.class))) + .thenReturn(mockContainerResponse); + + } catch (YarnException e) { + Assert.fail("Exception is not expected."); + } catch (IOException e) { + Assert.fail("Exception is not expected."); + } + } + + @Override + public List getApplications() throws YarnException, + IOException { + when(mockAppResponse.getApplicationList()).thenReturn(reports); + return super.getApplications(); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + when(mockResponse.getApplicationReport()).thenReturn(getReport(appId)); + return super.getApplicationReport(appId); + } + + @Override + public List getApplicationAttempts( + ApplicationId appId) throws YarnException, IOException { + when(mockAppAttemptsResponse.getApplicationAttemptList()).thenReturn( + getAttempts(appId)); + return super.getApplicationAttempts(appId); + } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId) throws YarnException, IOException { + when(mockAttemptResponse.getApplicationAttemptReport()).thenReturn( + getAttempt(appAttemptId)); + return super.getApplicationAttemptReport(appAttemptId); + } + + @Override + public List getContainers(ApplicationAttemptId appAttemptId) + throws YarnException, IOException { + when(mockContainersResponse.getContainerList()).thenReturn( + getContainersReport(appAttemptId)); + return super.getContainers(appAttemptId); + } + + @Override + public ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException { + when(mockContainerResponse.getContainerReport()).thenReturn( + getContainer(containerId)); + return super.getContainerReport(containerId); + } + + @Override + public void stop() { + } + + public ApplicationReport getReport(ApplicationId appId) { + for (int i = 0; i < reports.size(); ++i) { + if (appId.toString().equalsIgnoreCase( + reports.get(i).getApplicationId().toString())) { + return reports.get(i); + } + } + return null; + } + + public List getAttempts(ApplicationId appId) { + return attempts.get(appId); + } + + public ApplicationAttemptReport getAttempt(ApplicationAttemptId appAttemptId) { + return attempts.get(appAttemptId.getApplicationId()).get(0); + } + + public List getContainersReport( + ApplicationAttemptId appAttemptId) { + return containers.get(appAttemptId); + } + + public ContainerReport getContainer(ContainerId containerId) { + return containers.get(containerId.getApplicationAttemptId()).get(0); + } + + public List getReports() { + return this.reports; + } + + private void createAppReports() { + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationReport newApplicationReport = ApplicationReport + .newInstance(applicationId, ApplicationAttemptId.newInstance( + applicationId, 1), "user", "queue", "appname", "host", 124, null, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", + null); + List applicationReports = new ArrayList(); + applicationReports.add(newApplicationReport); + List appAttempts = new ArrayList(); + ApplicationAttemptReport attempt = ApplicationAttemptReport.newInstance( + ApplicationAttemptId.newInstance(applicationId, 1), "host", 124, + "url", "diagnostics", YarnApplicationAttemptState.FINISHED, + ContainerId.newInstance(newApplicationReport + .getCurrentApplicationAttemptId(), 1)); + appAttempts.add(attempt); + ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( + ApplicationAttemptId.newInstance(applicationId, 2), "host", 124, + "url", "diagnostics", YarnApplicationAttemptState.FINISHED, + ContainerId.newInstance(newApplicationReport + .getCurrentApplicationAttemptId(), 2)); + appAttempts.add(attempt1); + attempts.put(applicationId, appAttempts); + + List containerReports = new ArrayList(); + ContainerReport container = ContainerReport.newInstance(ContainerId + .newInstance(attempt.getApplicationAttemptId(), 1), null, NodeId + .newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + containerReports.add(container); + + ContainerReport container1 = ContainerReport.newInstance(ContainerId + .newInstance(attempt.getApplicationAttemptId(), 2), null, NodeId + .newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + containerReports.add(container1); + containers.put(attempt.getApplicationAttemptId(), containerReports); + + ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), + "user2", "queue2", "appname2", "host2", 125, null, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", + null); + applicationReports.add(newApplicationReport2); + + ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); + ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( + applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), + "user3", "queue3", "appname3", "host3", 126, null, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", + null); + applicationReports.add(newApplicationReport3); + + ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); + ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( + applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), + "user4", "queue4", "appname4", "host4", 127, null, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, + "NON-MAPREDUCE", null); + applicationReports.add(newApplicationReport4); + reports = applicationReports; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 00ab7895d67..dc6d98e6920 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -58,12 +58,10 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; import org.apache.log4j.LogManager; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1d08f245d06..dd6be0d8a57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -43,19 +43,26 @@ import junit.framework.Assert; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.util.Records; import org.junit.Before; import org.junit.Test; +import org.mortbay.log.Log; import org.apache.commons.cli.Options; @@ -113,20 +120,181 @@ public class TestYarnCLI { verify(sysOut, times(1)).println(isA(String.class)); } + @Test + public void testGetApplicationAttemptReport() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ApplicationAttemptReport attemptReport = ApplicationAttemptReport + .newInstance(attemptId, "host", 124, "url", "diagnostics", + YarnApplicationAttemptState.FINISHED, ContainerId.newInstance( + attemptId, 1)); + when( + client + .getApplicationAttemptReport(any(ApplicationAttemptId.class))) + .thenReturn(attemptReport); + int result = cli.run(new String[] { "applicationattempt", "-status", + attemptId.toString() }); + assertEquals(0, result); + verify(client).getApplicationAttemptReport(attemptId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Application Attempt Report : "); + pw.println("\tApplicationAttempt-Id : appattempt_1234_0005_000001"); + pw.println("\tState : FINISHED"); + pw.println("\tAMContainer : container_1234_0005_01_000001"); + pw.println("\tTracking-URL : url"); + pw.println("\tRPC Port : 124"); + pw.println("\tAM Host : host"); + pw.println("\tDiagnostics : diagnostics"); + pw.close(); + String appReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appReportStr, sysOutStream.toString()); + verify(sysOut, times(1)).println(isA(String.class)); + } + + @Test + public void testGetApplicationAttempts() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance( + applicationId, 2); + ApplicationAttemptReport attemptReport = ApplicationAttemptReport + .newInstance(attemptId, "host", 124, "url", "diagnostics", + YarnApplicationAttemptState.FINISHED, ContainerId.newInstance( + attemptId, 1)); + ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport + .newInstance(attemptId1, "host", 124, "url", "diagnostics", + YarnApplicationAttemptState.FINISHED, ContainerId.newInstance( + attemptId1, 1)); + List reports = new ArrayList(); + reports.add(attemptReport); + reports.add(attemptReport1); + when(client.getApplicationAttempts(any(ApplicationId.class))) + .thenReturn(reports); + int result = cli.run(new String[] { "applicationattempt", "-list", + applicationId.toString() }); + assertEquals(0, result); + verify(client).getApplicationAttempts(applicationId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Total number of application attempts :2"); + pw.print(" ApplicationAttempt-Id"); + pw.print("\t State"); + pw.print("\t AM-Container-Id"); + pw.println("\t Tracking-URL"); + pw.print(" appattempt_1234_0005_000001"); + pw.print("\t FINISHED"); + pw.print("\t container_1234_0005_01_000001"); + pw.println("\t url"); + pw.print(" appattempt_1234_0005_000002"); + pw.print("\t FINISHED"); + pw.print("\t container_1234_0005_02_000001"); + pw.println("\t url"); + pw.close(); + String appReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appReportStr, sysOutStream.toString()); + } + + @Test + public void testGetContainerReport() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ContainerId containerId = ContainerId.newInstance(attemptId, 1); + ContainerReport container = ContainerReport.newInstance(containerId, null, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + when(client.getContainerReport(any(ContainerId.class))).thenReturn( + container); + int result = cli.run(new String[] { "container", "-status", + containerId.toString() }); + assertEquals(0, result); + verify(client).getContainerReport(containerId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Container Report : "); + pw.println("\tContainer-Id : container_1234_0005_01_000001"); + pw.println("\tStart-Time : 1234"); + pw.println("\tFinish-Time : 5678"); + pw.println("\tState : COMPLETE"); + pw.println("\tLOG-URL : logURL"); + pw.println("\tHost : host:1234"); + pw.println("\tDiagnostics : diagnosticInfo"); + pw.close(); + String appReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appReportStr, sysOutStream.toString()); + verify(sysOut, times(1)).println(isA(String.class)); + } + + @Test + public void testGetContainers() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( + applicationId, 1); + ContainerId containerId = ContainerId.newInstance(attemptId, 1); + ContainerId containerId1 = ContainerId.newInstance(attemptId, 2); + ContainerReport container = ContainerReport.newInstance(containerId, null, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + ContainerReport container1 = ContainerReport.newInstance(containerId1, null, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + List reports = new ArrayList(); + reports.add(container); + reports.add(container1); + when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn( + reports); + int result = cli.run(new String[] { "container", "-list", + attemptId.toString() }); + assertEquals(0, result); + verify(client).getContainers(attemptId); + Log.info(sysOutStream.toString()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Total number of containers :2"); + pw.print(" Container-Id"); + pw.print("\t Start Time"); + pw.print("\t Finish Time"); + pw.print("\t State"); + pw.print("\t Host"); + pw.println("\t LOG-URL"); + pw.print(" container_1234_0005_01_000001"); + pw.print("\t 1234"); + pw.print("\t 5678"); + pw.print("\t COMPLETE"); + pw.print("\t host:1234"); + pw.println("\t logURL"); + pw.print(" container_1234_0005_01_000002"); + pw.print("\t 1234"); + pw.print("\t 5678"); + pw.print("\t COMPLETE"); + pw.print("\t host:1234"); + pw.println("\t logURL"); + pw.close(); + String appReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appReportStr, sysOutStream.toString()); + } + @Test public void testGetApplicationReportException() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); when(client.getApplicationReport(any(ApplicationId.class))).thenThrow( - new ApplicationNotFoundException("Application with id '" - + applicationId + "' doesn't exist in RM.")); + new ApplicationNotFoundException("History file for application" + + applicationId + " is not found")); try { cli.run(new String[] { "-status", applicationId.toString() }); Assert.fail(); } catch (Exception ex) { Assert.assertTrue(ex instanceof ApplicationNotFoundException); - Assert.assertEquals("Application with id '" + applicationId - + "' doesn't exist in RM.", ex.getMessage()); + Assert.assertEquals("History file for application" + + applicationId + " is not found", ex.getMessage()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java new file mode 100644 index 00000000000..b254f073e57 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.PrivilegedAction; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.ipc.YarnRPC; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +@SuppressWarnings("unchecked") +public class AHSProxy { + + private static final Log LOG = LogFactory.getLog(AHSProxy.class); + + public static T createAHSProxy(final Configuration conf, + final Class protocol, InetSocketAddress ahsAddress) throws IOException { + LOG.info("Connecting to Application History server at " + ahsAddress); + return (T) getProxy(conf, protocol, ahsAddress); + } + + protected static T getProxy(final Configuration conf, + final Class protocol, final InetSocketAddress rmAddress) + throws IOException { + return UserGroupInformation.getCurrentUser().doAs( + new PrivilegedAction() { + @Override + public T run() { + return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf); + } + }); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 45b20c3f65b..0901d9d57cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -52,6 +52,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -122,10 +125,14 @@ public class ApplicationHistoryClientService extends AbstractService { public GetApplicationAttemptReportResponse getApplicationAttemptReport( GetApplicationAttemptReportRequest request) throws YarnException, IOException { - GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse - .newInstance(history.getApplicationAttempt(request - .getApplicationAttemptId())); - return response; + try { + GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse + .newInstance(history.getApplicationAttempt(request + .getApplicationAttemptId())); + return response; + } catch (IOException e) { + throw new ApplicationAttemptNotFoundException(e.getMessage()); + } } @Override @@ -141,10 +148,14 @@ public class ApplicationHistoryClientService extends AbstractService { @Override public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnException, IOException { - ApplicationId applicationId = request.getApplicationId(); - GetApplicationReportResponse response = GetApplicationReportResponse - .newInstance(history.getApplication(applicationId)); - return response; + try { + ApplicationId applicationId = request.getApplicationId(); + GetApplicationReportResponse response = GetApplicationReportResponse + .newInstance(history.getApplication(applicationId)); + return response; + } catch (IOException e) { + throw new ApplicationNotFoundException(e.getMessage()); + } } @Override @@ -159,9 +170,13 @@ public class ApplicationHistoryClientService extends AbstractService { @Override public GetContainerReportResponse getContainerReport( GetContainerReportRequest request) throws YarnException, IOException { - GetContainerReportResponse response = GetContainerReportResponse - .newInstance(history.getContainer(request.getContainerId())); - return response; + try { + GetContainerReportResponse response = GetContainerReportResponse + .newInstance(history.getContainer(request.getContainerId())); + return response; + } catch (IOException e) { + throw new ContainerNotFoundException(e.getMessage()); + } } @Override