diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 24f5ff15404..f4c89f68be0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -41,7 +41,6 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -59,7 +59,7 @@ public class ResourceMgrDelegate extends YarnClient { private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); private YarnConfiguration conf; - private GetNewApplicationResponse application; + private ApplicationSubmissionContext application; private ApplicationId applicationId; @Private @VisibleForTesting @@ -178,7 +178,7 @@ public class ResourceMgrDelegate extends YarnClient { public JobID getNewJobID() throws IOException, InterruptedException { try { - this.application = client.getNewApplication(); + this.application = client.createApplication().getApplicationSubmissionContext(); this.applicationId = this.application.getApplicationId(); return TypeConverter.fromYarn(applicationId); } catch (YarnException e) { @@ -272,9 +272,9 @@ public class ResourceMgrDelegate extends YarnClient { } @Override - public GetNewApplicationResponse getNewApplication() throws YarnException, - IOException { - return client.getNewApplication(); + public YarnClientApplication createApplication() throws + YarnException, IOException { + return client.createApplication(); } @Override diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8573daac517..74d6df5f02b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -182,6 +182,11 @@ Release 2.1.0-beta - UNRELEASED YARN-694. Starting to use NMTokens to authenticate all communication with NodeManagers. (Omkar Vinit Joshi via vinodkv) + YARN-553. Replaced YarnClient.getNewApplication with + YarnClient.createApplication which provides a directly usable + ApplicationSubmissionContext to simplify the api. (Karthik Kambatla via + acmurthy) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 9e14ca44046..43507256e36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -217,7 +218,6 @@ public class Client { /** * Helper function to print out usage - * @param opts Parsed command line options */ private void printUsage() { new HelpFormatter().printHelp("Client", opts); @@ -351,16 +351,15 @@ public class Client { } } - // Get a new application id - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - + // Get a new application id + YarnClientApplication app = yarnClient.createApplication(); + GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager - int maxMem = newApp.getMaximumResourceCapability().getMemory(); + int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. @@ -371,13 +370,9 @@ public class Client { amMemory = maxMem; } - // Create launch context for app master - LOG.info("Setting up application submission context for ASM"); - ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); - - // set the application id - appContext.setApplicationId(appId); // set the application name + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up the container launch context for the application master diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b209d95a739..fbfe50669b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -274,17 +274,12 @@ public class UnmanagedAMLauncher { // Connect to ResourceManager rmClient.start(); try { - // Get a new application id - GetNewApplicationResponse newApp = rmClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - // Create launch context for app master LOG.info("Setting up application submission context for ASM"); - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = rmClient.createApplication() + .getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); + // set the application name appContext.setApplicationName(appName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 751c4ba809c..869a52bccc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -28,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -79,27 +78,18 @@ public abstract class YarnClient extends AbstractService { /** *

- * Obtain a new {@link ApplicationId} for submitting new applications. + * Obtain a {@link YarnClientApplication} for a new application, + * which in turn contains the {@link ApplicationSubmissionContext} and + * {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse} + * objects. *

- * - *

- * Returns a response which contains {@link ApplicationId} that can be used to - * submit a new application. See - * {@link #submitApplication(ApplicationSubmissionContext)}. - *

- * - *

- * See {@link GetNewApplicationResponse} for other information that is - * returned. - *

- * - * @return response containing the new ApplicationId to be used - * to submit an application + * + * @return {@link YarnClientApplication} built for a new application * @throws YarnException * @throws IOException */ - public abstract GetNewApplicationResponse getNewApplication() throws YarnException, - IOException; + public abstract YarnClientApplication createApplication() + throws YarnException, IOException; /** *

@@ -114,7 +104,7 @@ public abstract class YarnClient extends AbstractService { * @return {@link ApplicationId} of the accepted application * @throws YarnException * @throws IOException - * @see #getNewApplication() + * @see #createApplication() */ public abstract ApplicationId submitApplication(ApplicationSubmissionContext appContext) throws YarnException, IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java new file mode 100644 index 00000000000..7f9f47347b1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; + +@InterfaceAudience.Public +@InterfaceStability.Stable + +/** + * Holder for the {@link GetNewApplicationResponse} and {@link + * ApplicationSubmissionContext} objects created via {@link org.apache.hadoop + * .yarn.client.api.YarnClient#createApplication()} + */ +public class YarnClientApplication { + private final GetNewApplicationResponse newAppResponse; + private final ApplicationSubmissionContext appSubmissionContext; + + public YarnClientApplication(GetNewApplicationResponse newAppResponse, + ApplicationSubmissionContext appContext) { + this.newAppResponse = newAppResponse; + this.appSubmissionContext = appContext; + } + + public GetNewApplicationResponse getNewApplicationResponse() { + return newAppResponse; + } + + public ApplicationSubmissionContext getApplicationSubmissionContext() { + return appSubmissionContext; + } +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 3e1d5793461..cbe8120eac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -125,14 +126,24 @@ public class YarnClientImpl extends YarnClient { super.serviceStop(); } - @Override - public GetNewApplicationResponse getNewApplication() + private GetNewApplicationResponse getNewApplication() throws YarnException, IOException { GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); return rmClient.getNewApplication(request); } + @Override + public YarnClientApplication createApplication() + throws YarnException, IOException { + ApplicationSubmissionContext context = Records.newRecord + (ApplicationSubmissionContext.class); + GetNewApplicationResponse newApp = getNewApplication(); + ApplicationId appId = newApp.getApplicationId(); + context.setApplicationId(appId); + return new YarnClientApplication(newApp, context); + } + @Override public ApplicationId submitApplication(ApplicationSubmissionContext appContext) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 9398f984d97..5955f26b0bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -39,7 +39,6 @@ import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -61,7 +60,6 @@ import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest; -import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; @@ -117,13 +115,9 @@ public class TestAMRMClient { @Before public void startApp() throws Exception { // submit new app - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = + yarnClient.createApplication().getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java index 309b5afb882..5bcb428c3e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java @@ -36,7 +36,6 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -58,9 +57,6 @@ import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; -import org.apache.hadoop.yarn.client.api.impl.NMClientImpl; -import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; @@ -102,13 +98,9 @@ public class TestNMClient { nodeReports = yarnClient.getNodeReports(); // submit new app - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = + yarnClient.createApplication().getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master