YARN-967. Added the client and CLI interfaces for obtaining ApplicationHistory data. Contributed by Mayank Bansal.
svn merge --ignore-ancestry -c 1556747 ../YARN-321 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562202 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2de75edd46
commit
b7a3d0ae93
|
@ -44,9 +44,13 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
|
@ -371,4 +375,29 @@ public class ResourceMgrDelegate extends YarnClient {
|
||||||
IOException {
|
IOException {
|
||||||
return client.getQueueAclsInfo();
|
return client.getQueueAclsInfo();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
|
||||||
|
return client.getApplicationAttemptReport(appAttemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException {
|
||||||
|
return client.getApplicationAttempts(appId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return client.getContainerReport(containerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContainerReport> getContainers(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
return client.getContainers(applicationAttemptId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -508,6 +508,9 @@ Branch YARN-321: Generic ApplicationHistoryService
|
||||||
YARN-954. Implemented web UI for the ApplicationHistoryServer and wired it into
|
YARN-954. Implemented web UI for the ApplicationHistoryServer and wired it into
|
||||||
the HistoryStorage. (Zhijie Shen via vinodkv)
|
the HistoryStorage. (Zhijie Shen via vinodkv)
|
||||||
|
|
||||||
|
YARN-967. Added the client and CLI interfaces for obtaining ApplicationHistory
|
||||||
|
data. (Mayank Bansal via vinodkv)
|
||||||
|
|
||||||
Release 2.2.0 - 2013-10-13
|
Release 2.2.0 - 2013-10-13
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -63,6 +63,8 @@ function print_usage(){
|
||||||
echo " version print the version"
|
echo " version print the version"
|
||||||
echo " jar <jar> run a jar file"
|
echo " jar <jar> run a jar file"
|
||||||
echo " application prints application(s) report/kill application"
|
echo " application prints application(s) report/kill application"
|
||||||
|
echo " applicationattempt prints applicationattempt(s) report"
|
||||||
|
echo " container prints container(s) report"
|
||||||
echo " node prints node report(s)"
|
echo " node prints node report(s)"
|
||||||
echo " logs dump container logs"
|
echo " logs dump container logs"
|
||||||
echo " classpath prints the class path needed to get the"
|
echo " classpath prints the class path needed to get the"
|
||||||
|
@ -182,9 +184,12 @@ if [ "$COMMAND" = "classpath" ] ; then
|
||||||
elif [ "$COMMAND" = "rmadmin" ] ; then
|
elif [ "$COMMAND" = "rmadmin" ] ; then
|
||||||
CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
|
CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
|
||||||
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
||||||
elif [ "$COMMAND" = "application" ] ; then
|
elif [ "$COMMAND" = "application" ] ||
|
||||||
|
[ "$COMMAND" = "applicationattempt" ] ||
|
||||||
|
[ "$COMMAND" = "container" ]; then
|
||||||
CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
||||||
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
||||||
|
set -- $COMMAND $@
|
||||||
elif [ "$COMMAND" = "node" ] ; then
|
elif [ "$COMMAND" = "node" ] ; then
|
||||||
CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
|
CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
|
||||||
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
||||||
|
|
|
@ -141,7 +141,8 @@ if "%1" == "--config" (
|
||||||
goto :eof
|
goto :eof
|
||||||
)
|
)
|
||||||
|
|
||||||
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar application node logs daemonlog
|
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
|
||||||
|
application applicationattempt container node logs daemonlog historyserver
|
||||||
for %%i in ( %yarncommands% ) do (
|
for %%i in ( %yarncommands% ) do (
|
||||||
if %yarn-command% == %%i set yarncommand=true
|
if %yarn-command% == %%i set yarncommand=true
|
||||||
)
|
)
|
||||||
|
@ -173,6 +174,19 @@ goto :eof
|
||||||
:application
|
:application
|
||||||
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
||||||
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||||
|
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
|
||||||
|
goto :eof
|
||||||
|
|
||||||
|
:applicationattempt
|
||||||
|
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
||||||
|
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||||
|
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
|
||||||
|
goto :eof
|
||||||
|
|
||||||
|
:container
|
||||||
|
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
||||||
|
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||||
|
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
|
||||||
goto :eof
|
goto :eof
|
||||||
|
|
||||||
:node
|
:node
|
||||||
|
@ -268,6 +282,8 @@ goto :eof
|
||||||
@echo version print the version
|
@echo version print the version
|
||||||
@echo jar ^<jar^> run a jar file
|
@echo jar ^<jar^> run a jar file
|
||||||
@echo application prints application(s) report/kill application
|
@echo application prints application(s) report/kill application
|
||||||
|
@echo applicationattempt prints applicationattempt(s) report
|
||||||
|
@echo container prints container(s) report
|
||||||
@echo node prints node report(s)
|
@echo node prints node report(s)
|
||||||
@echo logs dump container logs
|
@echo logs dump container logs
|
||||||
@echo classpath prints the class path needed to get the
|
@echo classpath prints the class path needed to get the
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.exceptions;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This exception is thrown on
|
||||||
|
* {@link ApplicationHistoryProtocol#getApplicationAttemptReport
|
||||||
|
* (GetApplicationAttemptReportRequest)}
|
||||||
|
* API when the Application Attempt doesn't exist in Application History Server
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public class ApplicationAttemptNotFoundException extends YarnException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 8694508L;
|
||||||
|
|
||||||
|
public ApplicationAttemptNotFoundException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplicationAttemptNotFoundException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplicationAttemptNotFoundException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
}
|
|
@ -18,14 +18,19 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.exceptions;
|
package org.apache.hadoop.yarn.exceptions;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This exception is thrown on
|
* This exception is thrown on
|
||||||
* {@link ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)} API
|
* {@link ApplicationClientProtocol#getApplicationReport
|
||||||
* when the Application doesn't exist in RM
|
* (GetApplicationReportRequest)} API
|
||||||
|
* when the Application doesn't exist in RM and AHS
|
||||||
*/
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
public class ApplicationNotFoundException extends YarnException{
|
public class ApplicationNotFoundException extends YarnException{
|
||||||
|
|
||||||
private static final long serialVersionUID = 8694408L;
|
private static final long serialVersionUID = 8694408L;
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.exceptions;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This exception is thrown on
|
||||||
|
* {@link ApplicationHistoryProtocol#getContainerReport
|
||||||
|
* (GetContainerReportRequest)}
|
||||||
|
* API when the container doesn't exist in AHS
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public class ContainerNotFoundException extends YarnException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 8694608L;
|
||||||
|
|
||||||
|
public ContainerNotFoundException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ContainerNotFoundException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ContainerNotFoundException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,179 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.client.api;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.service.AbstractService;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.client.api.impl.AHSClientImpl;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
|
||||||
|
@InterfaceAudience.Public
|
||||||
|
@InterfaceStability.Stable
|
||||||
|
public abstract class AHSClient extends AbstractService {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new instance of AHSClient.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
public static AHSClient createAHSClient() {
|
||||||
|
AHSClient client = new AHSClientImpl();
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Private
|
||||||
|
public AHSClient(String name) {
|
||||||
|
super(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of the given Application.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* In secure mode, <code>YARN</code> verifies access to the application, queue
|
||||||
|
* etc. before accepting the request.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* If the user does not have <code>VIEW_APP</code> access then the following
|
||||||
|
* fields in the report will be set to stubbed values:
|
||||||
|
* <ul>
|
||||||
|
* <li>host - set to "N/A"</li>
|
||||||
|
* <li>RPC port - set to -1</li>
|
||||||
|
* <li>client token - set to "N/A"</li>
|
||||||
|
* <li>diagnostics - set to "N/A"</li>
|
||||||
|
* <li>tracking URL - set to "N/A"</li>
|
||||||
|
* <li>original tracking URL - set to "N/A"</li>
|
||||||
|
* <li>resource usage report - all values are -1</li>
|
||||||
|
* </ul>
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param appId
|
||||||
|
* {@link ApplicationId} of the application that needs a report
|
||||||
|
* @return application report
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract ApplicationReport getApplicationReport(ApplicationId appId)
|
||||||
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report (ApplicationReport) of all Applications in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* If the user does not have <code>VIEW_APP</code> access for an application
|
||||||
|
* then the corresponding report will be filtered as described in
|
||||||
|
* {@link #getApplicationReport(ApplicationId)}.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @return a list of reports for all applications
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract List<ApplicationReport> getApplications()
|
||||||
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of the given ApplicationAttempt.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* In secure mode, <code>YARN</code> verifies access to the application, queue
|
||||||
|
* etc. before accepting the request.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId
|
||||||
|
* {@link ApplicationAttemptId} of the application attempt that needs
|
||||||
|
* a report
|
||||||
|
* @return application attempt report
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws {@link ApplicationAttemptNotFoundException} if application attempt
|
||||||
|
* not found
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId appAttemptId) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of all (ApplicationAttempts) of Application in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationId
|
||||||
|
* @return a list of reports for all application attempts for specified
|
||||||
|
* application
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of the given Container.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* In secure mode, <code>YARN</code> verifies access to the application, queue
|
||||||
|
* etc. before accepting the request.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param containerId
|
||||||
|
* {@link ContainerId} of the container that needs a report
|
||||||
|
* @return container report
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws {@link ContainerNotFoundException} if container not found
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of all (Containers) of ApplicationAttempt in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId
|
||||||
|
* @return a list of reports of all containers for specified application
|
||||||
|
* attempt
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract List<ContainerReport> getContainers(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException;
|
||||||
|
}
|
|
@ -29,9 +29,13 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.service.AbstractService;
|
import org.apache.hadoop.service.AbstractService;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
|
@ -40,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Token;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
||||||
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
|
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||||
|
|
||||||
|
@ -360,4 +365,75 @@ public abstract class YarnClient extends AbstractService {
|
||||||
*/
|
*/
|
||||||
public abstract List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException,
|
public abstract List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException,
|
||||||
IOException;
|
IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of the given ApplicationAttempt.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* In secure mode, <code>YARN</code> verifies access to the application, queue
|
||||||
|
* etc. before accepting the request.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId
|
||||||
|
* {@link ApplicationAttemptId} of the application attempt that needs
|
||||||
|
* a report
|
||||||
|
* @return application attempt report
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws {@link ApplicationAttemptNotFoundException} if application attempt
|
||||||
|
* not found
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId appAttemptId) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of all (ApplicationAttempts) of Application in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationId
|
||||||
|
* @return a list of reports for all application attempts for specified
|
||||||
|
* application.
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of the given Container.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* In secure mode, <code>YARN</code> verifies access to the application, queue
|
||||||
|
* etc. before accepting the request.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param containerId
|
||||||
|
* {@link ContainerId} of the container that needs a report
|
||||||
|
* @return container report
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws {@link ContainerNotFoundException} if container not found.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Get a report of all (Containers) of ApplicationAttempt in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId
|
||||||
|
* @return a list of reports of all containers for specified application
|
||||||
|
* attempts
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public abstract List<ContainerReport> getContainers(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,156 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.client.api.impl;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.ipc.RPC;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.client.AHSProxy;
|
||||||
|
import org.apache.hadoop.yarn.client.api.AHSClient;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class AHSClientImpl extends AHSClient {
|
||||||
|
|
||||||
|
protected ApplicationHistoryProtocol ahsClient;
|
||||||
|
protected InetSocketAddress ahsAddress;
|
||||||
|
|
||||||
|
public AHSClientImpl() {
|
||||||
|
super(AHSClientImpl.class.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static InetSocketAddress getAHSAddress(Configuration conf) {
|
||||||
|
return conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS,
|
||||||
|
YarnConfiguration.DEFAULT_AHS_ADDRESS,
|
||||||
|
YarnConfiguration.DEFAULT_AHS_PORT);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void serviceInit(Configuration conf) throws Exception {
|
||||||
|
this.ahsAddress = getAHSAddress(conf);
|
||||||
|
super.serviceInit(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void serviceStart() throws Exception {
|
||||||
|
try {
|
||||||
|
ahsClient = AHSProxy.createAHSProxy(getConfig(),
|
||||||
|
ApplicationHistoryProtocol.class, this.ahsAddress);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new YarnRuntimeException(e);
|
||||||
|
}
|
||||||
|
super.serviceStart();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void serviceStop() throws Exception {
|
||||||
|
if (this.ahsClient != null) {
|
||||||
|
RPC.stopProxy(this.ahsClient);
|
||||||
|
}
|
||||||
|
super.serviceStop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationReport getApplicationReport(ApplicationId appId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
GetApplicationReportRequest request = GetApplicationReportRequest
|
||||||
|
.newInstance(appId);
|
||||||
|
GetApplicationReportResponse response = ahsClient
|
||||||
|
.getApplicationReport(request);
|
||||||
|
return response.getApplicationReport();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationReport> getApplications() throws YarnException,
|
||||||
|
IOException {
|
||||||
|
GetApplicationsRequest request = GetApplicationsRequest.newInstance(null,
|
||||||
|
null);
|
||||||
|
GetApplicationsResponse response = ahsClient.getApplications(request);
|
||||||
|
return response.getApplicationList();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest
|
||||||
|
.newInstance(applicationAttemptId);
|
||||||
|
GetApplicationAttemptReportResponse response = ahsClient
|
||||||
|
.getApplicationAttemptReport(request);
|
||||||
|
return response.getApplicationAttemptReport();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException {
|
||||||
|
GetApplicationAttemptsRequest request = GetApplicationAttemptsRequest
|
||||||
|
.newInstance(appId);
|
||||||
|
GetApplicationAttemptsResponse response = ahsClient
|
||||||
|
.getApplicationAttempts(request);
|
||||||
|
return response.getApplicationAttemptList();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
GetContainerReportRequest request = GetContainerReportRequest
|
||||||
|
.newInstance(containerId);
|
||||||
|
GetContainerReportResponse response = ahsClient.getContainerReport(request);
|
||||||
|
return response.getContainerReport();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContainerReport> getContainers(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
GetContainersRequest request = GetContainersRequest
|
||||||
|
.newInstance(applicationAttemptId);
|
||||||
|
GetContainersResponse response = ahsClient.getContainers(request);
|
||||||
|
return response.getContainerList();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -49,9 +49,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
|
@ -60,9 +64,11 @@ import org.apache.hadoop.yarn.api.records.Token;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
||||||
import org.apache.hadoop.yarn.client.ClientRMProxy;
|
import org.apache.hadoop.yarn.client.ClientRMProxy;
|
||||||
|
import org.apache.hadoop.yarn.client.api.AHSClient;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||||
|
@ -80,6 +86,7 @@ public class YarnClientImpl extends YarnClient {
|
||||||
protected ApplicationClientProtocol rmClient;
|
protected ApplicationClientProtocol rmClient;
|
||||||
protected long submitPollIntervalMillis;
|
protected long submitPollIntervalMillis;
|
||||||
private long asyncApiPollIntervalMillis;
|
private long asyncApiPollIntervalMillis;
|
||||||
|
protected AHSClient historyClient;
|
||||||
|
|
||||||
private static final String ROOT = "root";
|
private static final String ROOT = "root";
|
||||||
|
|
||||||
|
@ -100,6 +107,8 @@ public class YarnClientImpl extends YarnClient {
|
||||||
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
|
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
|
||||||
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
|
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
historyClient = AHSClientImpl.createAHSClient();
|
||||||
|
historyClient.init(getConfig());
|
||||||
super.serviceInit(conf);
|
super.serviceInit(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -108,6 +117,7 @@ public class YarnClientImpl extends YarnClient {
|
||||||
try {
|
try {
|
||||||
rmClient = ClientRMProxy.createRMProxy(getConfig(),
|
rmClient = ClientRMProxy.createRMProxy(getConfig(),
|
||||||
ApplicationClientProtocol.class);
|
ApplicationClientProtocol.class);
|
||||||
|
historyClient.start();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new YarnRuntimeException(e);
|
throw new YarnRuntimeException(e);
|
||||||
}
|
}
|
||||||
|
@ -119,6 +129,7 @@ public class YarnClientImpl extends YarnClient {
|
||||||
if (this.rmClient != null) {
|
if (this.rmClient != null) {
|
||||||
RPC.stopProxy(this.rmClient);
|
RPC.stopProxy(this.rmClient);
|
||||||
}
|
}
|
||||||
|
historyClient.stop();
|
||||||
super.serviceStop();
|
super.serviceStop();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -207,11 +218,20 @@ public class YarnClientImpl extends YarnClient {
|
||||||
@Override
|
@Override
|
||||||
public ApplicationReport getApplicationReport(ApplicationId appId)
|
public ApplicationReport getApplicationReport(ApplicationId appId)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
GetApplicationReportRequest request =
|
GetApplicationReportResponse response = null;
|
||||||
Records.newRecord(GetApplicationReportRequest.class);
|
try {
|
||||||
|
GetApplicationReportRequest request = Records
|
||||||
|
.newRecord(GetApplicationReportRequest.class);
|
||||||
request.setApplicationId(appId);
|
request.setApplicationId(appId);
|
||||||
GetApplicationReportResponse response =
|
response = rmClient.getApplicationReport(request);
|
||||||
rmClient.getApplicationReport(request);
|
} catch (YarnException e) {
|
||||||
|
if (!(e.getClass() == ApplicationNotFoundException.class)) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (response == null || response.getApplicationReport() == null) {
|
||||||
|
return historyClient.getApplicationReport(appId);
|
||||||
|
}
|
||||||
return response.getApplicationReport();
|
return response.getApplicationReport();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -373,4 +393,29 @@ public class YarnClientImpl extends YarnClient {
|
||||||
public void setRMClient(ApplicationClientProtocol rmClient) {
|
public void setRMClient(ApplicationClientProtocol rmClient) {
|
||||||
this.rmClient = rmClient;
|
this.rmClient = rmClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
|
||||||
|
return historyClient.getApplicationAttemptReport(appAttemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException {
|
||||||
|
return historyClient.getApplicationAttempts(appId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return historyClient.getContainerReport(containerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContainerReport> getContainers(
|
||||||
|
ApplicationAttemptId applicationAttemptId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
return historyClient.getContainers(applicationAttemptId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,8 +35,10 @@ import org.apache.commons.cli.Options;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
@ -47,12 +49,21 @@ import com.google.common.annotations.VisibleForTesting;
|
||||||
@Unstable
|
@Unstable
|
||||||
public class ApplicationCLI extends YarnCLI {
|
public class ApplicationCLI extends YarnCLI {
|
||||||
private static final String APPLICATIONS_PATTERN =
|
private static final String APPLICATIONS_PATTERN =
|
||||||
"%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
|
"%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s"
|
||||||
System.getProperty("line.separator");
|
+ System.getProperty("line.separator");
|
||||||
|
private static final String APPLICATION_ATTEMPTS_PATTERN =
|
||||||
|
"%30s\t%20s\t%35s\t%35s"
|
||||||
|
+ System.getProperty("line.separator");
|
||||||
|
private static final String CONTAINER_PATTERN =
|
||||||
|
"%30s\t%20s\t%20s\t%20s\t%20s\t%35s"
|
||||||
|
+ System.getProperty("line.separator");
|
||||||
|
|
||||||
private static final String APP_TYPE_CMD = "appTypes";
|
private static final String APP_TYPE_CMD = "appTypes";
|
||||||
private static final String APP_STATE_CMD ="appStates";
|
private static final String APP_STATE_CMD = "appStates";
|
||||||
private static final String ALLSTATES_OPTION = "ALL";
|
private static final String ALLSTATES_OPTION = "ALL";
|
||||||
|
public static final String APPLICATION = "application";
|
||||||
|
public static final String APPLICATION_ATTEMPT = "applicationattempt";
|
||||||
|
public static final String CONTAINER = "container";
|
||||||
|
|
||||||
private boolean allAppStates;
|
private boolean allAppStates;
|
||||||
|
|
||||||
|
@ -69,23 +80,33 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
public int run(String[] args) throws Exception {
|
public int run(String[] args) throws Exception {
|
||||||
|
|
||||||
Options opts = new Options();
|
Options opts = new Options();
|
||||||
opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
|
opts.addOption(STATUS_CMD, true,
|
||||||
opts.addOption(LIST_CMD, false, "List applications from the RM. " +
|
"Prints the status of the application.");
|
||||||
"Supports optional use of -appTypes to filter applications " +
|
if (args.length > 0
|
||||||
"based on application type, " +
|
&& args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
|
||||||
"and -appStates to filter applications based on application state");
|
opts.addOption(LIST_CMD, true,
|
||||||
|
"List application attempts for aplication from AHS. ");
|
||||||
|
} else if (args.length > 0 && args[0].compareToIgnoreCase("container") == 0) {
|
||||||
|
opts.addOption(LIST_CMD, true,
|
||||||
|
"List containers for application attempts from AHS. ");
|
||||||
|
} else {
|
||||||
|
opts.addOption(LIST_CMD, false, "List applications from the RM. "
|
||||||
|
+ "Supports optional use of -appTypes to filter applications "
|
||||||
|
+ "based on application type, "
|
||||||
|
+ "and -appStates to filter applications based on application state");
|
||||||
|
}
|
||||||
opts.addOption(KILL_CMD, true, "Kills the application.");
|
opts.addOption(KILL_CMD, true, "Kills the application.");
|
||||||
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
|
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
|
||||||
Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " +
|
Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to "
|
||||||
"filter applications based on " +
|
+ "filter applications based on "
|
||||||
"input comma-separated list of application types.");
|
+ "input comma-separated list of application types.");
|
||||||
appTypeOpt.setValueSeparator(',');
|
appTypeOpt.setValueSeparator(',');
|
||||||
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
|
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
|
||||||
appTypeOpt.setArgName("Types");
|
appTypeOpt.setArgName("Types");
|
||||||
opts.addOption(appTypeOpt);
|
opts.addOption(appTypeOpt);
|
||||||
Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " +
|
Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list "
|
||||||
"to filter applications based on input comma-separated list of " +
|
+ "to filter applications based on input comma-separated list of "
|
||||||
"application states. " + getAllValidApplicationStates());
|
+ "application states. " + getAllValidApplicationStates());
|
||||||
appStateOpt.setValueSeparator(',');
|
appStateOpt.setValueSeparator(',');
|
||||||
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
|
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
|
||||||
appStateOpt.setArgName("States");
|
appStateOpt.setArgName("States");
|
||||||
|
@ -104,15 +125,41 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cliParser.hasOption(STATUS_CMD)) {
|
if (cliParser.hasOption(STATUS_CMD)) {
|
||||||
if (args.length != 2) {
|
if ((args[0].compareToIgnoreCase(APPLICATION) == 0)
|
||||||
|
|| (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0)
|
||||||
|
|| (args[0].compareToIgnoreCase(CONTAINER) == 0)) {
|
||||||
|
if (args.length != 3) {
|
||||||
printUsage(opts);
|
printUsage(opts);
|
||||||
return exitCode;
|
return exitCode;
|
||||||
}
|
}
|
||||||
|
} else if (args.length != 2) {
|
||||||
|
printUsage(opts);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
|
||||||
|
printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
|
||||||
|
} else if (args[0].compareToIgnoreCase(CONTAINER) == 0) {
|
||||||
|
printContainerReport(cliParser.getOptionValue(STATUS_CMD));
|
||||||
|
} else {
|
||||||
printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
|
printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
|
||||||
|
}
|
||||||
} else if (cliParser.hasOption(LIST_CMD)) {
|
} else if (cliParser.hasOption(LIST_CMD)) {
|
||||||
|
if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
|
||||||
|
if (args.length != 3) {
|
||||||
|
printUsage(opts);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
|
||||||
|
} else if (args[0].compareToIgnoreCase(CONTAINER) == 0) {
|
||||||
|
if (args.length != 3) {
|
||||||
|
printUsage(opts);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
listContainers(cliParser.getOptionValue(LIST_CMD));
|
||||||
|
} else {
|
||||||
allAppStates = false;
|
allAppStates = false;
|
||||||
Set<String> appTypes = new HashSet<String>();
|
Set<String> appTypes = new HashSet<String>();
|
||||||
if(cliParser.hasOption(APP_TYPE_CMD)) {
|
if (cliParser.hasOption(APP_TYPE_CMD)) {
|
||||||
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
|
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
|
||||||
if (types != null) {
|
if (types != null) {
|
||||||
for (String type : types) {
|
for (String type : types) {
|
||||||
|
@ -123,8 +170,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
EnumSet<YarnApplicationState> appStates =
|
EnumSet<YarnApplicationState> appStates = EnumSet
|
||||||
EnumSet.noneOf(YarnApplicationState.class);
|
.noneOf(YarnApplicationState.class);
|
||||||
if (cliParser.hasOption(APP_STATE_CMD)) {
|
if (cliParser.hasOption(APP_STATE_CMD)) {
|
||||||
String[] states = cliParser.getOptionValues(APP_STATE_CMD);
|
String[] states = cliParser.getOptionValues(APP_STATE_CMD);
|
||||||
if (states != null) {
|
if (states != null) {
|
||||||
|
@ -135,8 +182,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
appStates.add(YarnApplicationState.valueOf(state.toUpperCase()
|
appStates.add(YarnApplicationState.valueOf(state
|
||||||
.trim()));
|
.toUpperCase().trim()));
|
||||||
} catch (IllegalArgumentException ex) {
|
} catch (IllegalArgumentException ex) {
|
||||||
sysout.println("The application state " + state
|
sysout.println("The application state " + state
|
||||||
+ " is invalid.");
|
+ " is invalid.");
|
||||||
|
@ -148,6 +195,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
listApplications(appTypes, appStates);
|
listApplications(appTypes, appStates);
|
||||||
|
}
|
||||||
} else if (cliParser.hasOption(KILL_CMD)) {
|
} else if (cliParser.hasOption(KILL_CMD)) {
|
||||||
if (args.length != 2) {
|
if (args.length != 2) {
|
||||||
printUsage(opts);
|
printUsage(opts);
|
||||||
|
@ -175,8 +223,85 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Lists the applications matching the given application Types
|
* Prints the application attempt report for an application attempt id.
|
||||||
* And application States present in the Resource Manager
|
*
|
||||||
|
* @param applicationAttemptId
|
||||||
|
* @throws YarnException
|
||||||
|
*/
|
||||||
|
private void printApplicationAttemptReport(String applicationAttemptId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
ApplicationAttemptReport appAttemptReport = client
|
||||||
|
.getApplicationAttemptReport(ConverterUtils
|
||||||
|
.toApplicationAttemptId(applicationAttemptId));
|
||||||
|
// Use PrintWriter.println, which uses correct platform line ending.
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter appAttemptReportStr = new PrintWriter(baos);
|
||||||
|
if (appAttemptReport != null) {
|
||||||
|
appAttemptReportStr.println("Application Attempt Report : ");
|
||||||
|
appAttemptReportStr.print("\tApplicationAttempt-Id : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId());
|
||||||
|
appAttemptReportStr.print("\tState : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport
|
||||||
|
.getYarnApplicationAttemptState());
|
||||||
|
appAttemptReportStr.print("\tAMContainer : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport.getAMContainerId()
|
||||||
|
.toString());
|
||||||
|
appAttemptReportStr.print("\tTracking-URL : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport.getTrackingUrl());
|
||||||
|
appAttemptReportStr.print("\tRPC Port : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport.getRpcPort());
|
||||||
|
appAttemptReportStr.print("\tAM Host : ");
|
||||||
|
appAttemptReportStr.println(appAttemptReport.getHost());
|
||||||
|
appAttemptReportStr.print("\tDiagnostics : ");
|
||||||
|
appAttemptReportStr.print(appAttemptReport.getDiagnostics());
|
||||||
|
} else {
|
||||||
|
appAttemptReportStr.print("Application Attempt with id '"
|
||||||
|
+ applicationAttemptId + "' doesn't exist in History Server.");
|
||||||
|
}
|
||||||
|
appAttemptReportStr.close();
|
||||||
|
sysout.println(baos.toString("UTF-8"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prints the container report for an container id.
|
||||||
|
*
|
||||||
|
* @param containerId
|
||||||
|
* @throws YarnException
|
||||||
|
*/
|
||||||
|
private void printContainerReport(String containerId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
ContainerReport containerReport = client.getContainerReport((ConverterUtils
|
||||||
|
.toContainerId(containerId)));
|
||||||
|
// Use PrintWriter.println, which uses correct platform line ending.
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter containerReportStr = new PrintWriter(baos);
|
||||||
|
if (containerReport != null) {
|
||||||
|
containerReportStr.println("Container Report : ");
|
||||||
|
containerReportStr.print("\tContainer-Id : ");
|
||||||
|
containerReportStr.println(containerReport.getContainerId());
|
||||||
|
containerReportStr.print("\tStart-Time : ");
|
||||||
|
containerReportStr.println(containerReport.getStartTime());
|
||||||
|
containerReportStr.print("\tFinish-Time : ");
|
||||||
|
containerReportStr.println(containerReport.getFinishTime());
|
||||||
|
containerReportStr.print("\tState : ");
|
||||||
|
containerReportStr.println(containerReport.getContainerState());
|
||||||
|
containerReportStr.print("\tLOG-URL : ");
|
||||||
|
containerReportStr.println(containerReport.getLogUrl());
|
||||||
|
containerReportStr.print("\tHost : ");
|
||||||
|
containerReportStr.println(containerReport.getAssignedNode());
|
||||||
|
containerReportStr.print("\tDiagnostics : ");
|
||||||
|
containerReportStr.print(containerReport.getDiagnosticsInfo());
|
||||||
|
} else {
|
||||||
|
containerReportStr.print("Container with id '" + containerId
|
||||||
|
+ "' doesn't exist in Hostory Server.");
|
||||||
|
}
|
||||||
|
containerReportStr.close();
|
||||||
|
sysout.println(baos.toString("UTF-8"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lists the applications matching the given application Types And application
|
||||||
|
* States present in the Resource Manager
|
||||||
*
|
*
|
||||||
* @param appTypes
|
* @param appTypes
|
||||||
* @param appStates
|
* @param appStates
|
||||||
|
@ -188,7 +313,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
IOException {
|
IOException {
|
||||||
PrintWriter writer = new PrintWriter(sysout);
|
PrintWriter writer = new PrintWriter(sysout);
|
||||||
if (allAppStates) {
|
if (allAppStates) {
|
||||||
for(YarnApplicationState appState : YarnApplicationState.values()) {
|
for (YarnApplicationState appState : YarnApplicationState.values()) {
|
||||||
appStates.add(appState);
|
appStates.add(appState);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -199,23 +324,24 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
List<ApplicationReport> appsReport =
|
List<ApplicationReport> appsReport = client.getApplications(appTypes,
|
||||||
client.getApplications(appTypes, appStates);
|
appStates);
|
||||||
|
|
||||||
writer
|
writer.println("Total number of applications (application-types: "
|
||||||
.println("Total number of applications (application-types: " + appTypes
|
+ appTypes + " and states: " + appStates + ")" + ":"
|
||||||
+ " and states: " + appStates + ")" + ":" + appsReport.size());
|
+ appsReport.size());
|
||||||
writer.printf(APPLICATIONS_PATTERN, "Application-Id",
|
writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name",
|
||||||
"Application-Name","Application-Type", "User", "Queue",
|
"Application-Type", "User", "Queue", "State", "Final-State",
|
||||||
"State", "Final-State","Progress", "Tracking-URL");
|
"Progress", "Tracking-URL");
|
||||||
for (ApplicationReport appReport : appsReport) {
|
for (ApplicationReport appReport : appsReport) {
|
||||||
DecimalFormat formatter = new DecimalFormat("###.##%");
|
DecimalFormat formatter = new DecimalFormat("###.##%");
|
||||||
String progress = formatter.format(appReport.getProgress());
|
String progress = formatter.format(appReport.getProgress());
|
||||||
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
|
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
|
||||||
appReport.getName(),appReport.getApplicationType(), appReport.getUser(),
|
appReport.getName(), appReport.getApplicationType(), appReport
|
||||||
appReport.getQueue(),appReport.getYarnApplicationState(),
|
.getUser(), appReport.getQueue(), appReport
|
||||||
appReport.getFinalApplicationStatus(),progress,
|
.getYarnApplicationState(),
|
||||||
appReport.getOriginalTrackingUrl());
|
appReport.getFinalApplicationStatus(), progress, appReport
|
||||||
|
.getOriginalTrackingUrl());
|
||||||
}
|
}
|
||||||
writer.flush();
|
writer.flush();
|
||||||
}
|
}
|
||||||
|
@ -227,8 +353,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
* @throws YarnException
|
* @throws YarnException
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
private void killApplication(String applicationId)
|
private void killApplication(String applicationId) throws YarnException,
|
||||||
throws YarnException, IOException {
|
IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
||||||
ApplicationReport appReport = client.getApplicationReport(appId);
|
ApplicationReport appReport = client.getApplicationReport(appId);
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
||||||
|
@ -296,14 +422,63 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
|
|
||||||
private String getAllValidApplicationStates() {
|
private String getAllValidApplicationStates() {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
sb.append("The valid application state can be"
|
sb.append("The valid application state can be" + " one of the following: ");
|
||||||
+ " one of the following: ");
|
|
||||||
sb.append(ALLSTATES_OPTION + ",");
|
sb.append(ALLSTATES_OPTION + ",");
|
||||||
for (YarnApplicationState appState : YarnApplicationState
|
for (YarnApplicationState appState : YarnApplicationState.values()) {
|
||||||
.values()) {
|
sb.append(appState + ",");
|
||||||
sb.append(appState+",");
|
|
||||||
}
|
}
|
||||||
String output = sb.toString();
|
String output = sb.toString();
|
||||||
return output.substring(0, output.length()-1);
|
return output.substring(0, output.length() - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lists the application attempts matching the given applicationid
|
||||||
|
*
|
||||||
|
* @param applicationId
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private void listApplicationAttempts(String appId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
PrintWriter writer = new PrintWriter(sysout);
|
||||||
|
|
||||||
|
List<ApplicationAttemptReport> appAttemptsReport = client
|
||||||
|
.getApplicationAttempts(ConverterUtils.toApplicationId(appId));
|
||||||
|
writer.println("Total number of application attempts " + ":"
|
||||||
|
+ appAttemptsReport.size());
|
||||||
|
writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
|
||||||
|
"State", "AM-Container-Id", "Tracking-URL");
|
||||||
|
for (ApplicationAttemptReport appAttemptReport : appAttemptsReport) {
|
||||||
|
writer.printf(APPLICATION_ATTEMPTS_PATTERN, appAttemptReport
|
||||||
|
.getApplicationAttemptId(), appAttemptReport
|
||||||
|
.getYarnApplicationAttemptState(), appAttemptReport
|
||||||
|
.getAMContainerId().toString(), appAttemptReport.getTrackingUrl());
|
||||||
|
}
|
||||||
|
writer.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lists the containers matching the given application attempts
|
||||||
|
*
|
||||||
|
* @param appAttemptId
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private void listContainers(String appAttemptId) throws YarnException,
|
||||||
|
IOException {
|
||||||
|
PrintWriter writer = new PrintWriter(sysout);
|
||||||
|
|
||||||
|
List<ContainerReport> appsReport = client
|
||||||
|
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
|
||||||
|
writer.println("Total number of containers " + ":" + appsReport.size());
|
||||||
|
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
|
||||||
|
"Finish Time", "State", "Host", "LOG-URL");
|
||||||
|
for (ContainerReport containerReport : appsReport) {
|
||||||
|
writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(),
|
||||||
|
containerReport.getStartTime(), containerReport.getFinishTime(),
|
||||||
|
containerReport.getContainerState(), containerReport
|
||||||
|
.getAssignedNode(), containerReport.getLogUrl());
|
||||||
|
}
|
||||||
|
writer.flush();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,394 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.client.api.impl;
|
||||||
|
|
||||||
|
import static org.mockito.Matchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import junit.framework.Assert;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerState;
|
||||||
|
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
|
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
|
||||||
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
|
import org.apache.hadoop.yarn.client.api.AHSClient;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestAHSClient {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testClientStop() {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
AHSClient client = AHSClient.createAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetApplications() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
List<ApplicationReport> expectedReports = ((MockAHSClient) client)
|
||||||
|
.getReports();
|
||||||
|
|
||||||
|
List<ApplicationReport> reports = client.getApplications();
|
||||||
|
Assert.assertEquals(reports, expectedReports);
|
||||||
|
|
||||||
|
reports = client.getApplications();
|
||||||
|
Assert.assertEquals(reports.size(), 4);
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetApplicationReport() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
List<ApplicationReport> expectedReports = ((MockAHSClient) client)
|
||||||
|
.getReports();
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationReport report = client.getApplicationReport(applicationId);
|
||||||
|
Assert.assertEquals(report, expectedReports.get(0));
|
||||||
|
Assert.assertEquals(report.getApplicationId().toString(), expectedReports
|
||||||
|
.get(0).getApplicationId().toString());
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetApplicationAttempts() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
List<ApplicationAttemptReport> reports = client
|
||||||
|
.getApplicationAttempts(applicationId);
|
||||||
|
Assert.assertNotNull(reports);
|
||||||
|
Assert.assertEquals(reports.get(0).getApplicationAttemptId(),
|
||||||
|
ApplicationAttemptId.newInstance(applicationId, 1));
|
||||||
|
Assert.assertEquals(reports.get(1).getApplicationAttemptId(),
|
||||||
|
ApplicationAttemptId.newInstance(applicationId, 2));
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetApplicationAttempt() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
List<ApplicationReport> expectedReports = ((MockAHSClient) client)
|
||||||
|
.getReports();
|
||||||
|
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ApplicationAttemptReport report = client
|
||||||
|
.getApplicationAttemptReport(appAttemptId);
|
||||||
|
Assert.assertNotNull(report);
|
||||||
|
Assert.assertEquals(report.getApplicationAttemptId().toString(),
|
||||||
|
expectedReports.get(0).getCurrentApplicationAttemptId().toString());
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetContainers() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
List<ContainerReport> reports = client.getContainers(appAttemptId);
|
||||||
|
Assert.assertNotNull(reports);
|
||||||
|
Assert.assertEquals(reports.get(0).getContainerId(), (ContainerId
|
||||||
|
.newInstance(appAttemptId, 1)));
|
||||||
|
Assert.assertEquals(reports.get(1).getContainerId(), (ContainerId
|
||||||
|
.newInstance(appAttemptId, 2)));
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testGetContainerReport() throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
final AHSClient client = new MockAHSClient();
|
||||||
|
client.init(conf);
|
||||||
|
client.start();
|
||||||
|
|
||||||
|
List<ApplicationReport> expectedReports = ((MockAHSClient) client)
|
||||||
|
.getReports();
|
||||||
|
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
|
||||||
|
ContainerReport report = client.getContainerReport(containerId);
|
||||||
|
Assert.assertNotNull(report);
|
||||||
|
Assert.assertEquals(report.getContainerId().toString(),
|
||||||
|
(ContainerId.newInstance(expectedReports.get(0)
|
||||||
|
.getCurrentApplicationAttemptId(), 1)).toString());
|
||||||
|
client.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class MockAHSClient extends AHSClientImpl {
|
||||||
|
// private ApplicationReport mockReport;
|
||||||
|
private List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
|
||||||
|
private HashMap<ApplicationId, List<ApplicationAttemptReport>> attempts =
|
||||||
|
new HashMap<ApplicationId, List<ApplicationAttemptReport>>();
|
||||||
|
private HashMap<ApplicationAttemptId, List<ContainerReport>> containers =
|
||||||
|
new HashMap<ApplicationAttemptId, List<ContainerReport>>();
|
||||||
|
GetApplicationsResponse mockAppResponse =
|
||||||
|
mock(GetApplicationsResponse.class);
|
||||||
|
GetApplicationReportResponse mockResponse =
|
||||||
|
mock(GetApplicationReportResponse.class);
|
||||||
|
GetApplicationAttemptsResponse mockAppAttemptsResponse =
|
||||||
|
mock(GetApplicationAttemptsResponse.class);
|
||||||
|
GetApplicationAttemptReportResponse mockAttemptResponse =
|
||||||
|
mock(GetApplicationAttemptReportResponse.class);
|
||||||
|
GetContainersResponse mockContainersResponse =
|
||||||
|
mock(GetContainersResponse.class);
|
||||||
|
GetContainerReportResponse mockContainerResponse =
|
||||||
|
mock(GetContainerReportResponse.class);
|
||||||
|
|
||||||
|
public MockAHSClient() {
|
||||||
|
super();
|
||||||
|
createAppReports();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void start() {
|
||||||
|
ahsClient = mock(ApplicationHistoryProtocol.class);
|
||||||
|
|
||||||
|
try {
|
||||||
|
when(
|
||||||
|
ahsClient
|
||||||
|
.getApplicationReport(any(GetApplicationReportRequest.class)))
|
||||||
|
.thenReturn(mockResponse);
|
||||||
|
when(ahsClient.getApplications(any(GetApplicationsRequest.class)))
|
||||||
|
.thenReturn(mockAppResponse);
|
||||||
|
when(
|
||||||
|
ahsClient
|
||||||
|
.getApplicationAttemptReport(any(GetApplicationAttemptReportRequest.class)))
|
||||||
|
.thenReturn(mockAttemptResponse);
|
||||||
|
when(
|
||||||
|
ahsClient
|
||||||
|
.getApplicationAttempts(any(GetApplicationAttemptsRequest.class)))
|
||||||
|
.thenReturn(mockAppAttemptsResponse);
|
||||||
|
when(ahsClient.getContainers(any(GetContainersRequest.class)))
|
||||||
|
.thenReturn(mockContainersResponse);
|
||||||
|
|
||||||
|
when(ahsClient.getContainerReport(any(GetContainerReportRequest.class)))
|
||||||
|
.thenReturn(mockContainerResponse);
|
||||||
|
|
||||||
|
} catch (YarnException e) {
|
||||||
|
Assert.fail("Exception is not expected.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
Assert.fail("Exception is not expected.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationReport> getApplications() throws YarnException,
|
||||||
|
IOException {
|
||||||
|
when(mockAppResponse.getApplicationList()).thenReturn(reports);
|
||||||
|
return super.getApplications();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationReport getApplicationReport(ApplicationId appId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
when(mockResponse.getApplicationReport()).thenReturn(getReport(appId));
|
||||||
|
return super.getApplicationReport(appId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ApplicationAttemptReport> getApplicationAttempts(
|
||||||
|
ApplicationId appId) throws YarnException, IOException {
|
||||||
|
when(mockAppAttemptsResponse.getApplicationAttemptList()).thenReturn(
|
||||||
|
getAttempts(appId));
|
||||||
|
return super.getApplicationAttempts(appId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ApplicationAttemptReport getApplicationAttemptReport(
|
||||||
|
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
|
||||||
|
when(mockAttemptResponse.getApplicationAttemptReport()).thenReturn(
|
||||||
|
getAttempt(appAttemptId));
|
||||||
|
return super.getApplicationAttemptReport(appAttemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContainerReport> getContainers(ApplicationAttemptId appAttemptId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
when(mockContainersResponse.getContainerList()).thenReturn(
|
||||||
|
getContainersReport(appAttemptId));
|
||||||
|
return super.getContainers(appAttemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ContainerReport getContainerReport(ContainerId containerId)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
when(mockContainerResponse.getContainerReport()).thenReturn(
|
||||||
|
getContainer(containerId));
|
||||||
|
return super.getContainerReport(containerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stop() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplicationReport getReport(ApplicationId appId) {
|
||||||
|
for (int i = 0; i < reports.size(); ++i) {
|
||||||
|
if (appId.toString().equalsIgnoreCase(
|
||||||
|
reports.get(i).getApplicationId().toString())) {
|
||||||
|
return reports.get(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ApplicationAttemptReport> getAttempts(ApplicationId appId) {
|
||||||
|
return attempts.get(appId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplicationAttemptReport getAttempt(ApplicationAttemptId appAttemptId) {
|
||||||
|
return attempts.get(appAttemptId.getApplicationId()).get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ContainerReport> getContainersReport(
|
||||||
|
ApplicationAttemptId appAttemptId) {
|
||||||
|
return containers.get(appAttemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ContainerReport getContainer(ContainerId containerId) {
|
||||||
|
return containers.get(containerId.getApplicationAttemptId()).get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ApplicationReport> getReports() {
|
||||||
|
return this.reports;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createAppReports() {
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationReport newApplicationReport = ApplicationReport
|
||||||
|
.newInstance(applicationId, ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1), "user", "queue", "appname", "host", 124, null,
|
||||||
|
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
|
||||||
|
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
|
||||||
|
null);
|
||||||
|
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
|
||||||
|
applicationReports.add(newApplicationReport);
|
||||||
|
List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
|
||||||
|
ApplicationAttemptReport attempt = ApplicationAttemptReport.newInstance(
|
||||||
|
ApplicationAttemptId.newInstance(applicationId, 1), "host", 124,
|
||||||
|
"url", "diagnostics", YarnApplicationAttemptState.FINISHED,
|
||||||
|
ContainerId.newInstance(newApplicationReport
|
||||||
|
.getCurrentApplicationAttemptId(), 1));
|
||||||
|
appAttempts.add(attempt);
|
||||||
|
ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance(
|
||||||
|
ApplicationAttemptId.newInstance(applicationId, 2), "host", 124,
|
||||||
|
"url", "diagnostics", YarnApplicationAttemptState.FINISHED,
|
||||||
|
ContainerId.newInstance(newApplicationReport
|
||||||
|
.getCurrentApplicationAttemptId(), 2));
|
||||||
|
appAttempts.add(attempt1);
|
||||||
|
attempts.put(applicationId, appAttempts);
|
||||||
|
|
||||||
|
List<ContainerReport> containerReports = new ArrayList<ContainerReport>();
|
||||||
|
ContainerReport container = ContainerReport.newInstance(ContainerId
|
||||||
|
.newInstance(attempt.getApplicationAttemptId(), 1), null, NodeId
|
||||||
|
.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
|
||||||
|
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
|
||||||
|
containerReports.add(container);
|
||||||
|
|
||||||
|
ContainerReport container1 = ContainerReport.newInstance(ContainerId
|
||||||
|
.newInstance(attempt.getApplicationAttemptId(), 2), null, NodeId
|
||||||
|
.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
|
||||||
|
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
|
||||||
|
containerReports.add(container1);
|
||||||
|
containers.put(attempt.getApplicationAttemptId(), containerReports);
|
||||||
|
|
||||||
|
ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
|
||||||
|
ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
|
||||||
|
applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2),
|
||||||
|
"user2", "queue2", "appname2", "host2", 125, null,
|
||||||
|
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
|
||||||
|
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
|
||||||
|
null);
|
||||||
|
applicationReports.add(newApplicationReport2);
|
||||||
|
|
||||||
|
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
|
||||||
|
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
|
||||||
|
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
|
||||||
|
"user3", "queue3", "appname3", "host3", 126, null,
|
||||||
|
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
|
||||||
|
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
|
||||||
|
null);
|
||||||
|
applicationReports.add(newApplicationReport3);
|
||||||
|
|
||||||
|
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
|
||||||
|
ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(
|
||||||
|
applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
|
||||||
|
"user4", "queue4", "appname4", "host4", 127, null,
|
||||||
|
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
|
||||||
|
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
|
||||||
|
"NON-MAPREDUCE", null);
|
||||||
|
applicationReports.add(newApplicationReport4);
|
||||||
|
reports = applicationReports;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -58,12 +58,10 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
|
||||||
import org.apache.hadoop.yarn.server.MiniYARNCluster;
|
import org.apache.hadoop.yarn.server.MiniYARNCluster;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
|
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
|
|
|
@ -43,19 +43,26 @@ import junit.framework.Assert;
|
||||||
|
|
||||||
import org.apache.commons.lang.time.DateFormatUtils;
|
import org.apache.commons.lang.time.DateFormatUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerState;
|
||||||
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||||
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.mortbay.log.Log;
|
||||||
|
|
||||||
import org.apache.commons.cli.Options;
|
import org.apache.commons.cli.Options;
|
||||||
|
|
||||||
|
@ -113,20 +120,181 @@ public class TestYarnCLI {
|
||||||
verify(sysOut, times(1)).println(isA(String.class));
|
verify(sysOut, times(1)).println(isA(String.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetApplicationAttemptReport() throws Exception {
|
||||||
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
|
||||||
|
.newInstance(attemptId, "host", 124, "url", "diagnostics",
|
||||||
|
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
|
||||||
|
attemptId, 1));
|
||||||
|
when(
|
||||||
|
client
|
||||||
|
.getApplicationAttemptReport(any(ApplicationAttemptId.class)))
|
||||||
|
.thenReturn(attemptReport);
|
||||||
|
int result = cli.run(new String[] { "applicationattempt", "-status",
|
||||||
|
attemptId.toString() });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getApplicationAttemptReport(attemptId);
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Application Attempt Report : ");
|
||||||
|
pw.println("\tApplicationAttempt-Id : appattempt_1234_0005_000001");
|
||||||
|
pw.println("\tState : FINISHED");
|
||||||
|
pw.println("\tAMContainer : container_1234_0005_01_000001");
|
||||||
|
pw.println("\tTracking-URL : url");
|
||||||
|
pw.println("\tRPC Port : 124");
|
||||||
|
pw.println("\tAM Host : host");
|
||||||
|
pw.println("\tDiagnostics : diagnostics");
|
||||||
|
pw.close();
|
||||||
|
String appReportStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||||
|
verify(sysOut, times(1)).println(isA(String.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetApplicationAttempts() throws Exception {
|
||||||
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 2);
|
||||||
|
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
|
||||||
|
.newInstance(attemptId, "host", 124, "url", "diagnostics",
|
||||||
|
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
|
||||||
|
attemptId, 1));
|
||||||
|
ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport
|
||||||
|
.newInstance(attemptId1, "host", 124, "url", "diagnostics",
|
||||||
|
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
|
||||||
|
attemptId1, 1));
|
||||||
|
List<ApplicationAttemptReport> reports = new ArrayList<ApplicationAttemptReport>();
|
||||||
|
reports.add(attemptReport);
|
||||||
|
reports.add(attemptReport1);
|
||||||
|
when(client.getApplicationAttempts(any(ApplicationId.class)))
|
||||||
|
.thenReturn(reports);
|
||||||
|
int result = cli.run(new String[] { "applicationattempt", "-list",
|
||||||
|
applicationId.toString() });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getApplicationAttempts(applicationId);
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Total number of application attempts :2");
|
||||||
|
pw.print(" ApplicationAttempt-Id");
|
||||||
|
pw.print("\t State");
|
||||||
|
pw.print("\t AM-Container-Id");
|
||||||
|
pw.println("\t Tracking-URL");
|
||||||
|
pw.print(" appattempt_1234_0005_000001");
|
||||||
|
pw.print("\t FINISHED");
|
||||||
|
pw.print("\t container_1234_0005_01_000001");
|
||||||
|
pw.println("\t url");
|
||||||
|
pw.print(" appattempt_1234_0005_000002");
|
||||||
|
pw.print("\t FINISHED");
|
||||||
|
pw.print("\t container_1234_0005_02_000001");
|
||||||
|
pw.println("\t url");
|
||||||
|
pw.close();
|
||||||
|
String appReportStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetContainerReport() throws Exception {
|
||||||
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ContainerId containerId = ContainerId.newInstance(attemptId, 1);
|
||||||
|
ContainerReport container = ContainerReport.newInstance(containerId, null,
|
||||||
|
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
|
||||||
|
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
|
||||||
|
when(client.getContainerReport(any(ContainerId.class))).thenReturn(
|
||||||
|
container);
|
||||||
|
int result = cli.run(new String[] { "container", "-status",
|
||||||
|
containerId.toString() });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getContainerReport(containerId);
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Container Report : ");
|
||||||
|
pw.println("\tContainer-Id : container_1234_0005_01_000001");
|
||||||
|
pw.println("\tStart-Time : 1234");
|
||||||
|
pw.println("\tFinish-Time : 5678");
|
||||||
|
pw.println("\tState : COMPLETE");
|
||||||
|
pw.println("\tLOG-URL : logURL");
|
||||||
|
pw.println("\tHost : host:1234");
|
||||||
|
pw.println("\tDiagnostics : diagnosticInfo");
|
||||||
|
pw.close();
|
||||||
|
String appReportStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||||
|
verify(sysOut, times(1)).println(isA(String.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetContainers() throws Exception {
|
||||||
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ContainerId containerId = ContainerId.newInstance(attemptId, 1);
|
||||||
|
ContainerId containerId1 = ContainerId.newInstance(attemptId, 2);
|
||||||
|
ContainerReport container = ContainerReport.newInstance(containerId, null,
|
||||||
|
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
|
||||||
|
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
|
||||||
|
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
|
||||||
|
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
|
||||||
|
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
|
||||||
|
List<ContainerReport> reports = new ArrayList<ContainerReport>();
|
||||||
|
reports.add(container);
|
||||||
|
reports.add(container1);
|
||||||
|
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
|
||||||
|
reports);
|
||||||
|
int result = cli.run(new String[] { "container", "-list",
|
||||||
|
attemptId.toString() });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getContainers(attemptId);
|
||||||
|
Log.info(sysOutStream.toString());
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Total number of containers :2");
|
||||||
|
pw.print(" Container-Id");
|
||||||
|
pw.print("\t Start Time");
|
||||||
|
pw.print("\t Finish Time");
|
||||||
|
pw.print("\t State");
|
||||||
|
pw.print("\t Host");
|
||||||
|
pw.println("\t LOG-URL");
|
||||||
|
pw.print(" container_1234_0005_01_000001");
|
||||||
|
pw.print("\t 1234");
|
||||||
|
pw.print("\t 5678");
|
||||||
|
pw.print("\t COMPLETE");
|
||||||
|
pw.print("\t host:1234");
|
||||||
|
pw.println("\t logURL");
|
||||||
|
pw.print(" container_1234_0005_01_000002");
|
||||||
|
pw.print("\t 1234");
|
||||||
|
pw.print("\t 5678");
|
||||||
|
pw.print("\t COMPLETE");
|
||||||
|
pw.print("\t host:1234");
|
||||||
|
pw.println("\t logURL");
|
||||||
|
pw.close();
|
||||||
|
String appReportStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetApplicationReportException() throws Exception {
|
public void testGetApplicationReportException() throws Exception {
|
||||||
ApplicationCLI cli = createAndGetAppCLI();
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
|
when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
|
||||||
new ApplicationNotFoundException("Application with id '"
|
new ApplicationNotFoundException("History file for application"
|
||||||
+ applicationId + "' doesn't exist in RM."));
|
+ applicationId + " is not found"));
|
||||||
try {
|
try {
|
||||||
cli.run(new String[] { "-status", applicationId.toString() });
|
cli.run(new String[] { "-status", applicationId.toString() });
|
||||||
Assert.fail();
|
Assert.fail();
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
Assert.assertTrue(ex instanceof ApplicationNotFoundException);
|
Assert.assertTrue(ex instanceof ApplicationNotFoundException);
|
||||||
Assert.assertEquals("Application with id '" + applicationId
|
Assert.assertEquals("History file for application"
|
||||||
+ "' doesn't exist in RM.", ex.getMessage());
|
+ applicationId + " is not found", ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,57 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.client;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.security.PrivilegedAction;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
|
|
||||||
|
@InterfaceAudience.Public
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public class AHSProxy<T> {
|
||||||
|
|
||||||
|
private static final Log LOG = LogFactory.getLog(AHSProxy.class);
|
||||||
|
|
||||||
|
public static <T> T createAHSProxy(final Configuration conf,
|
||||||
|
final Class<T> protocol, InetSocketAddress ahsAddress) throws IOException {
|
||||||
|
LOG.info("Connecting to Application History server at " + ahsAddress);
|
||||||
|
return (T) getProxy(conf, protocol, ahsAddress);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static <T> T getProxy(final Configuration conf,
|
||||||
|
final Class<T> protocol, final InetSocketAddress rmAddress)
|
||||||
|
throws IOException {
|
||||||
|
return UserGroupInformation.getCurrentUser().doAs(
|
||||||
|
new PrivilegedAction<T>() {
|
||||||
|
@Override
|
||||||
|
public T run() {
|
||||||
|
return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
|
@ -52,6 +52,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
|
|
||||||
|
@ -122,10 +125,14 @@ public class ApplicationHistoryClientService extends AbstractService {
|
||||||
public GetApplicationAttemptReportResponse getApplicationAttemptReport(
|
public GetApplicationAttemptReportResponse getApplicationAttemptReport(
|
||||||
GetApplicationAttemptReportRequest request) throws YarnException,
|
GetApplicationAttemptReportRequest request) throws YarnException,
|
||||||
IOException {
|
IOException {
|
||||||
|
try {
|
||||||
GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse
|
GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse
|
||||||
.newInstance(history.getApplicationAttempt(request
|
.newInstance(history.getApplicationAttempt(request
|
||||||
.getApplicationAttemptId()));
|
.getApplicationAttemptId()));
|
||||||
return response;
|
return response;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ApplicationAttemptNotFoundException(e.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -141,10 +148,14 @@ public class ApplicationHistoryClientService extends AbstractService {
|
||||||
@Override
|
@Override
|
||||||
public GetApplicationReportResponse getApplicationReport(
|
public GetApplicationReportResponse getApplicationReport(
|
||||||
GetApplicationReportRequest request) throws YarnException, IOException {
|
GetApplicationReportRequest request) throws YarnException, IOException {
|
||||||
|
try {
|
||||||
ApplicationId applicationId = request.getApplicationId();
|
ApplicationId applicationId = request.getApplicationId();
|
||||||
GetApplicationReportResponse response = GetApplicationReportResponse
|
GetApplicationReportResponse response = GetApplicationReportResponse
|
||||||
.newInstance(history.getApplication(applicationId));
|
.newInstance(history.getApplication(applicationId));
|
||||||
return response;
|
return response;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ApplicationNotFoundException(e.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -159,9 +170,13 @@ public class ApplicationHistoryClientService extends AbstractService {
|
||||||
@Override
|
@Override
|
||||||
public GetContainerReportResponse getContainerReport(
|
public GetContainerReportResponse getContainerReport(
|
||||||
GetContainerReportRequest request) throws YarnException, IOException {
|
GetContainerReportRequest request) throws YarnException, IOException {
|
||||||
|
try {
|
||||||
GetContainerReportResponse response = GetContainerReportResponse
|
GetContainerReportResponse response = GetContainerReportResponse
|
||||||
.newInstance(history.getContainer(request.getContainerId()));
|
.newInstance(history.getContainer(request.getContainerId()));
|
||||||
return response;
|
return response;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ContainerNotFoundException(e.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
Loading…
Reference in New Issue