From 075ff276ca9e8c192717a50b0e18485afc8571a6 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 2 Jul 2014 00:23:07 +0000 Subject: [PATCH] YARN-1713. Added get-new-app and submit-app functionality to RM web services. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607216 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../records/ApplicationSubmissionContext.java | 2 +- .../yarn/webapp/GenericExceptionHandler.java | 5 + .../resourcemanager/webapp/RMWebServices.java | 275 +++++ .../dao/ApplicationSubmissionContextInfo.java | 186 +++ .../dao/ContainerLaunchContextInfo.java | 117 ++ .../webapp/dao/CredentialsInfo.java | 59 + .../webapp/dao/LocalResourceInfo.java | 96 ++ .../webapp/dao/NewApplication.java | 54 + .../webapp/dao/ResourceInfo.java | 10 +- .../TestRMWebServicesAppsModification.java | 368 +++++- .../src/site/apt/ResourceManagerRest.apt.vm | 1006 ++++++++++++----- 12 files changed, 1887 insertions(+), 294 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 03efcb0b3f7..f1a00cfcdc2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -51,6 +51,9 @@ Release 2.5.0 - UNRELEASED YARN-2052. Embedded an epoch number in container id to ensure the uniqueness of container id after RM restarts. (Tsuyoshi OZAWA via jianhe) + YARN-1713. Added get-new-app and submit-app functionality to RM web services. + (Varun Vasudev via vinodkv) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 529df113c27..1ee04f04ca3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -88,7 +88,7 @@ public static ApplicationSubmissionContext newInstance( int maxAppAttempts, Resource resource, String applicationType) { return newInstance(applicationId, applicationName, queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, - resource, null, false); + resource, applicationType, false); } @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java index 74180f3f3fa..1e53e022dc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java @@ -21,10 +21,12 @@ import java.io.IOException; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; +import javax.xml.bind.UnmarshalException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -87,6 +89,9 @@ public Response toResponse(Exception e) { s = Response.Status.BAD_REQUEST; } else if (e instanceof BadRequestException) { s = Response.Status.BAD_REQUEST; + } else if (e instanceof WebApplicationException + && e.getCause() instanceof UnmarshalException) { + s = Response.Status.BAD_REQUEST; } else { LOG.warn("INTERNAL_SERVER_ERROR", e); s = Response.Status.INTERNAL_SERVER_ERROR; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 62e7f9f8f3d..2c2a7aaed9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.security.AccessControlException; +import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; @@ -36,6 +37,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -47,22 +49,38 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -81,17 +99,22 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; @@ -758,4 +781,256 @@ private UserGroupInformation getCallerUserGroupInformation( return callerUGI; } + + /** + * Generates a new ApplicationId which is then sent to the client + * + * @param hsr + * the servlet request + * @return Response containing the app id and the maximum resource + * capabilities + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps/new-application") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response createNewApplication(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + NewApplication appId = createNewApplication(); + return Response.status(Status.OK).entity(appId).build(); + + } + + // reuse the code in ClientRMService to create new app + // get the new app id and submit app + // set location header with new app location + /** + * Function to submit an app to the RM + * + * @param newApp + * structure containing information to construct the + * ApplicationSubmissionContext + * @param hsr + * the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitApplication(ApplicationSubmissionContextInfo newApp, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + ApplicationSubmissionContext appContext = + createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = + SubmitApplicationRequest.newInstance(appContext); + + try { + callerUGI + .doAs(new PrivilegedExceptionAction() { + @Override + public SubmitApplicationResponse run() throws IOException, + YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Submit app request failed", ue); + throw ue; + } + + String url = hsr.getRequestURL() + "/" + newApp.getApplicationId(); + return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url) + .build(); + } + + /** + * Function that actually creates the ApplicationId by calling the + * ClientRMService + * + * @return returns structure containing the app-id and maximum resource + * capabilities + */ + private NewApplication createNewApplication() { + GetNewApplicationRequest req = + recordFactory.newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } catch (YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + NewApplication appId = + new NewApplication(resp.getApplicationId().toString(), new ResourceInfo( + resp.getMaximumResourceCapability())); + return appId; + } + + /** + * Create the actual ApplicationSubmissionContext to be submitted to the RM + * from the information provided by the user. + * + * @param newApp + * the information provided by the user + * @return returns the constructed ApplicationSubmissionContext + * @throws IOException + */ + protected ApplicationSubmissionContext createAppSubmissionContext( + ApplicationSubmissionContextInfo newApp) throws IOException { + + // create local resources and app submission context + + ApplicationId appid; + String error = + "Could not parse application id " + newApp.getApplicationId(); + try { + appid = + ConverterUtils.toApplicationId(recordFactory, + newApp.getApplicationId()); + } catch (Exception e) { + throw new BadRequestException(error); + } + ApplicationSubmissionContext appContext = + ApplicationSubmissionContext.newInstance(appid, + newApp.getApplicationName(), newApp.getQueue(), + Priority.newInstance(newApp.getPriority()), + createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), + newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), + createAppSubmissionContextResource(newApp), + newApp.getApplicationType(), + newApp.getKeepContainersAcrossApplicationAttempts()); + appContext.setApplicationTags(newApp.getApplicationTags()); + + return appContext; + } + + protected Resource createAppSubmissionContextResource( + ApplicationSubmissionContextInfo newApp) throws BadRequestException { + if (newApp.getResource().getvCores() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + String msg = "Requested more cores than configured max"; + throw new BadRequestException(msg); + } + if (newApp.getResource().getMemory() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + String msg = "Requested more memory than configured max"; + throw new BadRequestException(msg); + } + Resource r = + Resource.newInstance(newApp.getResource().getMemory(), newApp + .getResource().getvCores()); + return r; + } + + /** + * Create the ContainerLaunchContext required for the + * ApplicationSubmissionContext. This function takes the user information and + * generates the ByteBuffer structures required by the ContainerLaunchContext + * + * @param newApp + * the information provided by the user + * @return + * @throws BadRequestException + * @throws IOException + */ + protected ContainerLaunchContext createContainerLaunchContext( + ApplicationSubmissionContextInfo newApp) throws BadRequestException, IOException { + + // create container launch context + + HashMap hmap = new HashMap(); + for (Map.Entry entry : newApp + .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) { + if (entry.getValue().isEmpty() == false) { + Base64 decoder = new Base64(0, null, true); + byte[] data = decoder.decode(entry.getValue()); + hmap.put(entry.getKey(), ByteBuffer.wrap(data)); + } + } + + HashMap hlr = new HashMap(); + for (Map.Entry entry : newApp + .getContainerLaunchContextInfo().getResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + LocalResource lr = + LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(), + l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + + DataOutputBuffer out = new DataOutputBuffer(); + Credentials cs = + createCredentials(newApp.getContainerLaunchContextInfo() + .getCredentials()); + cs.writeTokenStorageToStream(out); + ByteBuffer tokens = ByteBuffer.wrap(out.getData()); + + ContainerLaunchContext ctx = + ContainerLaunchContext.newInstance(hlr, newApp + .getContainerLaunchContextInfo().getEnvironment(), newApp + .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp + .getContainerLaunchContextInfo().getAcls()); + + return ctx; + } + + /** + * Generate a Credentials object from the information in the CredentialsInfo + * object. + * + * @param credentials + * the CredentialsInfo provided by the user. + * @return + */ + private Credentials createCredentials(CredentialsInfo credentials) { + Credentials ret = new Credentials(); + try { + for (Map.Entry entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Token token = new Token(); + token.decodeFromUrlString(entry.getValue()); + ret.addToken(alias, token); + } + for (Map.Entry entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Base64 decoder = new Base64(0, null, true); + byte[] secret = decoder.decode(entry.getValue()); + ret.addSecretKey(alias, secret); + } + } catch (IOException ie) { + throw new BadRequestException( + "Could not parse credentials data; exception message = " + + ie.getMessage()); + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java new file mode 100644 index 00000000000..f7233e6e3af --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashSet; +import java.util.Set; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +/** + * Simple class to allow users to send information required to create an + * ApplicationSubmissionContext which can then be used to submit an app + * + */ +@XmlRootElement(name = "application-submission-context") +@XmlAccessorType(XmlAccessType.FIELD) +public class ApplicationSubmissionContextInfo { + + @XmlElement(name = "application-id") + String applicationId; + + @XmlElement(name = "application-name") + String applicationName; + + String queue; + int priority; + + @XmlElement(name = "am-container-spec") + ContainerLaunchContextInfo containerInfo; + + @XmlElement(name = "unmanaged-AM") + boolean isUnmanagedAM; + + @XmlElement(name = "cancel-tokens-when-complete") + boolean cancelTokensWhenComplete; + + @XmlElement(name = "max-app-attempts") + int maxAppAttempts; + + @XmlElement(name = "resource") + ResourceInfo resource; + + @XmlElement(name = "application-type") + String applicationType; + + @XmlElement(name = "keep-containers-across-application-attempts") + boolean keepContainers; + + @XmlElementWrapper(name = "application-tags") + @XmlElement(name = "tag") + Set tags; + + public ApplicationSubmissionContextInfo() { + applicationId = ""; + applicationName = ""; + containerInfo = new ContainerLaunchContextInfo(); + resource = new ResourceInfo(); + priority = Priority.UNDEFINED.getPriority(); + isUnmanagedAM = false; + cancelTokensWhenComplete = true; + keepContainers = false; + applicationType = ""; + tags = new HashSet(); + } + + public String getApplicationId() { + return applicationId; + } + + public String getApplicationName() { + return applicationName; + } + + public String getQueue() { + return queue; + } + + public int getPriority() { + return priority; + } + + public ContainerLaunchContextInfo getContainerLaunchContextInfo() { + return containerInfo; + } + + public boolean getUnmanagedAM() { + return isUnmanagedAM; + } + + public boolean getCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + + public int getMaxAppAttempts() { + return maxAppAttempts; + } + + public ResourceInfo getResource() { + return resource; + } + + public String getApplicationType() { + return applicationType; + } + + public boolean getKeepContainersAcrossApplicationAttempts() { + return keepContainers; + } + + public Set getApplicationTags() { + return tags; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public void setContainerLaunchContextInfo( + ContainerLaunchContextInfo containerLaunchContext) { + this.containerInfo = containerLaunchContext; + } + + public void setUnmanagedAM(boolean isUnmanagedAM) { + this.isUnmanagedAM = isUnmanagedAM; + } + + public void setCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + } + + public void setMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + + public void setResource(ResourceInfo resource) { + this.resource = resource; + } + + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + + public void + setKeepContainersAcrossApplicationAttempts(boolean keepContainers) { + this.keepContainers = keepContainers; + } + + public void setApplicationTags(Set tags) { + this.tags = tags; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java new file mode 100644 index 00000000000..e296c3991e3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; + +/** + * Simple class to allow users to send information required to create a + * ContainerLaunchContext which can then be used as part of the + * ApplicationSubmissionContext + * + */ +@XmlRootElement(name = "container-launch-context-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLaunchContextInfo { + + @XmlElementWrapper(name = "local-resources") + HashMap local_resources; + HashMap environment; + + @XmlElementWrapper(name = "commands") + @XmlElement(name = "command", type = String.class) + List commands; + + @XmlElementWrapper(name = "service-data") + HashMap servicedata; + + @XmlElement(name = "credentials") + CredentialsInfo credentials; + + @XmlElementWrapper(name = "application-acls") + HashMap acls; + + public ContainerLaunchContextInfo() { + local_resources = new HashMap(); + environment = new HashMap(); + commands = new ArrayList(); + servicedata = new HashMap(); + credentials = new CredentialsInfo(); + acls = new HashMap(); + } + + public Map getResources() { + return local_resources; + } + + public Map getEnvironment() { + return environment; + } + + public List getCommands() { + return commands; + } + + public Map getAuxillaryServiceData() { + return servicedata; + } + + public CredentialsInfo getCredentials() { + return credentials; + } + + public Map getAcls() { + return acls; + } + + public void setResources(HashMap resources) { + this.local_resources = resources; + } + + public void setEnvironment(HashMap environment) { + this.environment = environment; + } + + public void setCommands(List commands) { + this.commands = commands; + } + + public void setAuxillaryServiceData(HashMap serviceData) { + this.servicedata = serviceData; + } + + public void setCredentials(CredentialsInfo credentials) { + this.credentials = credentials; + } + + public void setAcls(HashMap acls) { + this.acls = acls; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java new file mode 100644 index 00000000000..76c815a19a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashMap; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "credentials-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class CredentialsInfo { + + @XmlElementWrapper(name = "tokens") + HashMap tokens; + + @XmlElementWrapper(name = "secrets") + HashMap secrets; + + public CredentialsInfo() { + tokens = new HashMap(); + secrets = new HashMap(); + } + + public HashMap getTokens() { + return tokens; + } + + public HashMap getSecrets() { + return secrets; + } + + public void setTokens(HashMap tokens) { + this.tokens = tokens; + } + + public void setSecrets(HashMap secrets) { + this.secrets = secrets; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java new file mode 100644 index 00000000000..6e73b76fb4a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.net.URI; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; + +@XmlRootElement(name = "localresources") +@XmlAccessorType(XmlAccessType.FIELD) +public class LocalResourceInfo { + + @XmlElement(name = "resource") + URI url; + LocalResourceType type; + LocalResourceVisibility visibility; + long size; + long timestamp; + String pattern; + + public URI getUrl() { + return url; + } + + public LocalResourceType getType() { + return type; + } + + public LocalResourceVisibility getVisibility() { + return visibility; + } + + public long getSize() { + return size; + } + + public long getTimestamp() { + return timestamp; + } + + public String getPattern() { + return pattern; + } + + public void setUrl(URI url) { + this.url = url; + } + + public void setType(LocalResourceType type) { + this.type = type; + } + + public void setVisibility(LocalResourceVisibility visibility) { + this.visibility = visibility; + } + + public void setSize(long size) { + if (size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.size = size; + } + + public void setTimestamp(long timestamp) { + if (timestamp <= 0) { + throw new IllegalArgumentException("timestamp must be greater than 0"); + } + this.timestamp = timestamp; + } + + public void setPattern(String pattern) { + this.pattern = pattern; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java new file mode 100644 index 00000000000..b5064723199 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java @@ -0,0 +1,54 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name="NewApplication") +@XmlAccessorType(XmlAccessType.FIELD) +public class NewApplication { + + @XmlElement(name="application-id") + String applicationId; + + @XmlElement(name="maximum-resource-capability") + ResourceInfo maximumResourceCapability; + + public NewApplication() { + applicationId = ""; + maximumResourceCapability = new ResourceInfo(); + } + + public NewApplication(String appId, ResourceInfo maxResources) { + applicationId = appId; + maximumResourceCapability = maxResources; + } + + public String getApplicationId() { + return applicationId; + } + + public ResourceInfo getMaximumResourceCapability() { + return maximumResourceCapability; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 6b4422ce61a..9510f5f5f04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -30,7 +30,7 @@ public class ResourceInfo { int memory; int vCores; - public ResourceInfo() { + public ResourceInfo() { } public ResourceInfo(Resource res) { @@ -50,4 +50,12 @@ public int getvCores() { public String toString() { return ""; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 76281d47b80..4c8eeb306b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -25,10 +25,17 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; +import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; import javax.servlet.FilterConfig; import javax.servlet.ServletException; @@ -38,9 +45,15 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -54,7 +67,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONException; @@ -80,6 +97,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.filter.LoggingFilter; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; @@ -461,11 +479,7 @@ public void testSingleAppKillUnauthorized() throws Exception { .constructWebResource("apps", app.getApplicationId().toString(), "state").accept(mediaType) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); - if (!isAuthenticationEnabled()) { - assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); - } else { - assertEquals(Status.FORBIDDEN, response.getClientResponseStatus()); - } + validateResponseStatus(response, Status.FORBIDDEN); } rm.stop(); return; @@ -502,4 +516,348 @@ public void tearDown() throws Exception { } super.tearDown(); } + + /** + * Helper function to wrap frequently used code. It checks the response status + * and checks if it UNAUTHORIZED if we are running with authorization turned + * off or the param passed if we are running with authorization turned on. + * + * @param response + * the ClientResponse object to be checked + * @param expectedAuthorizedMode + * the expected Status in authorized mode. + */ + public void validateResponseStatus(ClientResponse response, + Status expectedAuthorizedMode) { + validateResponseStatus(response, Status.UNAUTHORIZED, + expectedAuthorizedMode); + } + + /** + * Helper function to wrap frequently used code. It checks the response status + * and checks if it is the param expectedUnauthorizedMode if we are running + * with authorization turned off or the param expectedAuthorizedMode passed if + * we are running with authorization turned on. + * + * @param response + * the ClientResponse object to be checked + * @param expectedUnauthorizedMode + * the expected Status in unauthorized mode. + * @param expectedAuthorizedMode + * the expected Status in authorized mode. + */ + public void validateResponseStatus(ClientResponse response, + Status expectedUnauthorizedMode, Status expectedAuthorizedMode) { + if (!isAuthenticationEnabled()) { + assertEquals(expectedUnauthorizedMode, response.getClientResponseStatus()); + } else { + assertEquals(expectedAuthorizedMode, response.getClientResponseStatus()); + } + } + + // Simple test - just post to /apps/id and validate the response + @Test + public void testGetNewApplication() throws Exception { + // client().addFilter(new LoggingFilter(System.out)); + rm.start(); + String mediaTypes[] = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + for (String acceptMedia : mediaTypes) { + testGetNewApplication(acceptMedia); + } + rm.stop(); + return; + } + + protected String testGetNewApplication(String mediaType) throws JSONException, + ParserConfigurationException, IOException, SAXException { + ClientResponse response = + this.constructWebResource("apps", "new-application").accept(mediaType) + .post(ClientResponse.class); + validateResponseStatus(response, Status.OK); + if (!isAuthenticationEnabled()) { + return ""; + } + return validateGetNewApplicationResponse(response); + } + + protected String validateGetNewApplicationResponse(ClientResponse resp) + throws JSONException, ParserConfigurationException, IOException, + SAXException { + String ret = ""; + if (resp.getType().equals(MediaType.APPLICATION_JSON_TYPE)) { + JSONObject json = resp.getEntity(JSONObject.class); + ret = validateGetNewApplicationJsonResponse(json); + } else if (resp.getType().equals(MediaType.APPLICATION_XML_TYPE)) { + String xml = resp.getEntity(String.class); + ret = validateGetNewApplicationXMLResponse(xml); + } else { + // we should not be here + assertTrue(false); + } + return ret; + } + + protected String validateGetNewApplicationJsonResponse(JSONObject json) + throws JSONException { + String appId = json.getString("application-id"); + assertTrue(appId.isEmpty() == false); + JSONObject maxResources = json.getJSONObject("maximum-resource-capability"); + long memory = maxResources.getLong("memory"); + long vCores = maxResources.getLong("vCores"); + assertTrue(memory != 0); + assertTrue(vCores != 0); + return appId; + } + + protected String validateGetNewApplicationXMLResponse(String response) + throws ParserConfigurationException, IOException, SAXException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(response)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("NewApplication"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + String appId = WebServicesTestUtils.getXmlString(element, "application-id"); + assertTrue(appId.isEmpty() == false); + NodeList maxResourceNodes = + element.getElementsByTagName("maximum-resource-capability"); + assertEquals(1, maxResourceNodes.getLength()); + Element maxResourceCapability = (Element) maxResourceNodes.item(0); + long memory = + WebServicesTestUtils.getXmlLong(maxResourceCapability, "memory"); + long vCores = + WebServicesTestUtils.getXmlLong(maxResourceCapability, "vCores"); + assertTrue(memory != 0); + assertTrue(vCores != 0); + return appId; + } + + // Test to validate the process of submitting apps - test for appropriate + // errors as well + @Test + public void testGetNewApplicationAndSubmit() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String mediaTypes[] = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + for (String acceptMedia : mediaTypes) { + for (String contentMedia : mediaTypes) { + testAppSubmit(acceptMedia, contentMedia); + testAppSubmitErrors(acceptMedia, contentMedia); + } + } + rm.stop(); + return; + } + + public void testAppSubmit(String acceptMedia, String contentMedia) + throws Exception { + + // create a test app and submit it via rest(after getting an app-id) then + // get the app details from the rmcontext and check that everything matches + + // client().addFilter(new LoggingFilter(System.out)); + String lrKey = "example"; + String queueName = "testqueue"; + String appName = "test"; + String appType = "test-type"; + String urlPath = "apps"; + String appId = testGetNewApplication(acceptMedia); + List commands = new ArrayList(); + commands.add("/bin/sleep 5"); + HashMap environment = new HashMap(); + environment.put("APP_VAR", "ENV_SETTING"); + HashMap acls = + new HashMap(); + acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2"); + acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag 2"); + CredentialsInfo credentials = new CredentialsInfo(); + HashMap tokens = new HashMap(); + HashMap secrets = new HashMap(); + secrets.put("secret1", Base64.encodeBase64URLSafeString("secret1".getBytes("UTF8"))); + credentials.setSecrets(secrets); + credentials.setTokens(tokens); + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + appInfo.setApplicationId(appId); + appInfo.setApplicationName(appName); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue(queueName); + appInfo.setApplicationType(appType); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put(lrKey, y); + appInfo.getContainerLaunchContextInfo().setResources(lr); + appInfo.getContainerLaunchContextInfo().setCommands(commands); + appInfo.getContainerLaunchContextInfo().setEnvironment(environment); + appInfo.getContainerLaunchContextInfo().setAcls(acls); + appInfo.getContainerLaunchContextInfo().getAuxillaryServiceData() + .put("test", Base64.encodeBase64URLSafeString("value12".getBytes("UTF8"))); + appInfo.getContainerLaunchContextInfo().setCredentials(credentials); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + appInfo.setApplicationTags(tags); + + ClientResponse response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + + if (this.isAuthenticationEnabled() == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + assertTrue(response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty() == false); + String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION); + assertTrue(locURL.indexOf("/apps/application") != -1); + appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length()); + + WebResource res = resource().uri(new URI(locURL)); + res = res.queryParam("user.name", webserviceUserName); + response = res.get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + + RMApp app = + rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appId)); + assertEquals(appName, app.getName()); + assertEquals(webserviceUserName, app.getUser()); + assertEquals(2, app.getMaxAppAttempts()); + assertEquals(queueName, app.getQueue()); + assertEquals(appType, app.getApplicationType()); + assertEquals(tags, app.getApplicationTags()); + ContainerLaunchContext ctx = + app.getApplicationSubmissionContext().getAMContainerSpec(); + assertEquals(commands, ctx.getCommands()); + assertEquals(environment, ctx.getEnvironment()); + assertEquals(acls, ctx.getApplicationACLs()); + Map appLRs = ctx.getLocalResources(); + assertTrue(appLRs.containsKey(lrKey)); + LocalResource exampleLR = appLRs.get(lrKey); + assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()), + exampleLR.getResource()); + assertEquals(y.getSize(), exampleLR.getSize()); + assertEquals(y.getTimestamp(), exampleLR.getTimestamp()); + assertEquals(y.getType(), exampleLR.getType()); + assertEquals(y.getPattern(), exampleLR.getPattern()); + assertEquals(y.getVisibility(), exampleLR.getVisibility()); + + response = + this.constructWebResource("apps", appId).accept(acceptMedia) + .get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + return; + } + + public void testAppSubmitErrors(String acceptMedia, String contentMedia) + throws Exception { + + // submit a bunch of bad requests(correct format but bad values) via the + // REST API and make sure we get the right error response codes + + String urlPath = "apps"; + String appId = ""; + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + ClientResponse response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + appId = "random"; + appInfo.setApplicationId(appId); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + appId = "random_junk"; + appInfo.setApplicationId(appId); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + // bad resource info + appInfo.getResource().setMemory( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1); + appInfo.getResource().setvCores(1); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + + validateResponseStatus(response, Status.BAD_REQUEST); + + appInfo.getResource().setvCores( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1); + appInfo.getResource().setMemory(CONTAINER_MB); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + return; + } + + @Test + public void testAppSubmitBadJsonAndXML() throws Exception { + + // submit a bunch of bad XML and JSON via the + // REST API and make sure we get error response codes + + String urlPath = "apps"; + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContextInfo().setResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + String body = + ""; + ClientResponse response = + this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_XML).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + body = "{\"a\" : \"b\"}"; + response = + this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_JSON).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + rm.stop(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index f17b1df6e1a..e419ceef577 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1105,9 +1105,9 @@ ResourceManager REST API's. +---+ -* Cluster Applications API +* {Cluster Applications API} - With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. ** URI @@ -1123,7 +1123,7 @@ ResourceManager REST API's. ** Query Parameters Supported - Multiple parameters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + Multiple parameters can be specified for GET operations. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. ------ * state [deprecated] - state of the application @@ -1295,7 +1295,6 @@ _01_000001 0 - +---+ * Cluster Application Statistics API @@ -1427,7 +1426,7 @@ _01_000001 ** URI - Use the following URI to obtain an app object, from a application identified by the {appid} value. + Use the following URI to obtain an app object, from a application identified by the {appid} value. ------ * http:///ws/v1/cluster/apps/{appid} @@ -1538,7 +1537,7 @@ _01_000001 } +---+ - <> + <> HTTP Request: @@ -1580,287 +1579,6 @@ _01_000001 +---+ -* Cluster Application State API - - With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response. - - Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. - -** URI - ------ - * http:///ws/v1/cluster/apps/{appid}/state ------ - -** HTTP Operations Supported - ------- - * GET - * PUT ------- - -** Query Parameters Supported - ------- - None ------- - -** Elements of object - - When you make a request for the state of an app, the information returned has the following fields - -*---------------+--------------+-------------------------------+ -|| Item || Data Type || Description | -*---------------+--------------+-------------------------------+ -| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" | -*---------------+--------------+--------------------------------+ - - -** Response Examples - - <> - - HTTP Request - ------ - GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state ------ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"ACCEPTED" -} -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - Response Header: - -+---+ -HTTP/1.1 202 Accepted -Content-Type: application/json -Transfer-Encoding: chunked -Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"ACCEPTED" -} -+---+ - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - <> - - HTTP Request - ------ - GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state ------ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/xml -Content-Length: 99 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - ACCEPTED - -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 202 Accepted -Content-Type: application/json -Content-Length: 794 -Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - ACCEPTED - -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/xml -Content-Length: 917 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - KILLED - -+---+ - - <> - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 403 Unauthorized -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - - <> - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - RUNNING - -+---+ - - Response Header: - -+---+ -HTTP/1.1 400 -Content-Length: 295 -Content-Type: application/xml -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - BadRequestException - java.lang.Exception: Only 'KILLED' is allowed as a target state. - org.apache.hadoop.yarn.webapp.BadRequestException - -+---+ - * Cluster Application Attempts API With the application attempts API, you can obtain a collection of resources that represent an application attempt. When you run a GET operation on this resource, you obtain a collection of App Attempt Objects. @@ -2275,3 +1993,717 @@ Server: Jetty(6.1.26) +---+ +* {Cluster Writeable APIs} + + The setions below refer to APIs which allow to create and modify applications. These APIs are currently in alpha and may change in the future. + +* {Cluster New Application API} + + With the New Application API, you can obtain an application-id which can then be used as part of the {{{Cluster_Applications_API(Submit_Application)}Cluster Submit Applications API}} to submit applications. The response also includes the maximum resource capabilities available on the cluster. + + This feature is currently in the alpha stage and may change in the future. + +** URI + +------ + * http:///ws/v1/cluster/apps/new-application +------ + +** HTTP Operations Supported + +------ + * POST +------ + +** Query Parameters Supported + +------ + * None +------ + +** Elements of the NewApplication object + + The NewApplication response contains the following elements: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| application-id | string | The newly created application id | +*---------------+--------------+--------------------------------+ +| maximum-resource-capabilities | object | The maximum resource capabilities available on this cluster | +*---------------+--------------+--------------------------------+ + + The object contains the following elements: + + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| memory | int | The maxiumim memory available for a container | +*---------------+--------------+--------------------------------+ +| vCores | int | The maximum number of cores available for a container | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + POST http:///ws/v1/cluster/apps/new-application +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "application-id":"application_1404198295326_0001", + "maximum-resource-capability": + { + "memory":"8192", + "vCores":"32" + } +} ++---+ + + <> + + HTTP Request: + +------ + POST http:///ws/v1/cluster/apps/new-application +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 248 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1404198295326_0003 + + 8192 + 32 + + ++---+ + +* {Cluster Applications API(Submit Application)} + + The Submit Applications API can be used to submit applications. In case of submitting applications, you must first obtain an application-id using the {{{Cluster_New_Application_API}Cluster New Application API}}. The application-id must be part of the request body. The response contains a URL to the application page which can be used to track the state and progress of your application. + +** URI + +------ + * http:///ws/v1/cluster/apps +------ + +** HTTP Operations Supported + +------ + * POST +------ + +** POST Response Examples + + POST requests can be used to submit apps to the ResourceManager. As mentioned above, an application-id must be obtained first. Successful submissions result in a 202 response code and a Location header specifying where to get information about the app. Please note that in order to submit an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. + + Please note that this feature is currently in the alpha stage and may change in the future. + +*** Elements of the POST request object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| application-id | string | The application id | +*---------------+--------------+-------------------------------+ +| application-name | string | The application name | +*---------------+--------------+-------------------------------+ +| queue | string | The name of the queue to which the application should be submitted | +*---------------+--------------+-------------------------------+ +| priority | int | The priority of the application | +*---------------+--------------+-------------------------------+ +| am-container-spec | object | The application master container launch context, described below | +*---------------+--------------+-------------------------------+ +| unmanaged-AM | boolean | Is the application using an unmanaged application master | +*---------------+--------------+-------------------------------+ +| max-app-attempts | int | The max number of attempts for this application | +*---------------+--------------+-------------------------------+ +| resource | object | The resources the application master requires, described below | +*---------------+--------------+-------------------------------+ +| application-type | string | The application type(MapReduce, Pig, Hive, etc) | +*---------------+--------------+-------------------------------+ +| keep-containers-across-application-attempts | boolean | Should YARN keep the containers used by this application instead of destroying them | +*---------------+--------------+-------------------------------+ +| application-tags | object | List of application tags, please see the request examples on how to speciy the tags | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The am-container-spec object should be used to provide the container launch context for the application master. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| local-resources | object | Object describing the resources that need to be localized, described below | +*---------------+--------------+-------------------------------+ +| environment | object | Environment variables for your containers, specified as key value pairs | +*---------------+--------------+-------------------------------+ +| commands | object | The commands for launching your container, in the order in which they should be executed | +*---------------+--------------+-------------------------------+ +| service-data | object | Application specific service data; key is the name of the auxiliary servce, value is base-64 encoding of the data you wish to pass | +*---------------+--------------+-------------------------------+ +| credentials | object | The credentials required for your application to run, described below | +*---------------+--------------+-------------------------------+ +| application-acls | objec | ACLs for your application; the key can be "VIEW_APP" or "MODIFY_APP", the value is the list of users with the permissions | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The object is a collection of key-value pairs. They key is an identifier for the resources to be localized and the value is the details of the resource. The elements of the value are described below: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| resource | string | Location of the resource to be localized | +*---------------+--------------+-------------------------------+ +| type | string | Type of the resource; options are "ARCHIVE", "FILE", and "PATTERN" | +*---------------+--------------+-------------------------------+ +| visibility | string | Visibility the resource to be localized; options are "PUBLIC", "PRIVATE", and "APPLICATION" | +*---------------+--------------+-------------------------------+ +| size | long | Size of the resource to be localized | +*---------------+--------------+-------------------------------+ +| timestamp | long | Timestamp of the resource to be localized | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The credentials object should be used to pass data required for the application to authenticate itself such as delegation-tokens and secrets. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| tokens | object | Tokens that you wish to pass to your application, specified as key-value pairs. The key is an identifier for the token and the value is the token(which should be obtained using the respective web-services) | +*---------------+--------------+-------------------------------+ +| secrets | object | Secrets that you wish to use in your application, specified as key-value pairs. They key is an identifier and the value is the base-64 encoding of the secret | +*---------------+--------------+-------------------------------+ + + + Elements of the POST request body object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| memory | int | Memory required for each container | +*---------------+--------------+-------------------------------+ +| vCores | int | Virtual cores required for each container | +*---------------+--------------+-------------------------------+ + + <> + + HTTP Request: + ++---+ + POST http:///ws/v1/cluster/apps + Accept: application/json + Content-Type: application/json + { + "application-id":"application_1404203615263_0001", + "application-name":"test", + "queue":"testqueue", + "priority":"3", + "am-container-spec": + { + "local-resources": + { + "entry": + { + "key":"example", + "value": + { + "resource":"http://www.test.com/file.txt", + "type":"FILE", + "visibility":"APPLICATION", + "size":"100", + "timestamp":"1404203616003" + } + } + }, + "environment": + { + "entry": + { + "key":"APP_VAR", + "value":"ENV_SETTING" + } + }, + "commands": + { + "command":"/bin/sleep 5" + }, + "service-data": + { + "entry": + { + "key":"test", + "value":"dmFsdWUxMg" + } + }, + "credentials": + { + "tokens":null, + "secrets": + { + "entry": + { + "key":"secret1", + "value":"c2VjcmV0MQ" + } + } + }, + "application-acls": + { + "entry": + [ + { + "key":"VIEW_APP", + "value":"testuser3, testuser4" + }, + { + "key":"MODIFY_APP", + "value":"testuser1, testuser2" + } + ] + } + }, + "unmanaged-AM":"false", + "max-app-attempts":"2", + "resource": + { + "memory":"1024", + "vCores":"1" + }, + "application-type":"YARN", + "keep-containers-across-application-attempts":"false", + "application-tags": + { + "tag": + [ + "tag 2", + "tag1" + ] + } + } ++---+ + + Response Header: + ++---+ + HTTP/1.1 202 + Transfer-Encoding: chunked + Location: http:///ws/v1/cluster/apps/application_1404203615263_0001 + Content-Type: application/json + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + No response body ++---+ + + <> + + HTTP Request: + ++---+ + POST http:///ws/v1/cluster/apps + Accept: application/xml + Content-Type: application/xml + + + application_1404204891930_0002 + test + testqueue + 3 + + + + example + + http://www.test.com/file.txt + FILE + APPLICATION + 100 + 1404204892877 + + + + + + APP_VAR + ENV_SETTING + + + + /bin/sleep 5 + + + + test + dmFsdWUxMg + + + + + + + secret1 + c2VjcmV0MQ + + + + + + VIEW_APP + testuser3, testuser4 + + + MODIFY_APP + testuser1, testuser2 + + + + false + 2 + + 1024 + 1 + + YARN + false + + tag 2 + tag1 + + ++---+ + + Response Header: + ++---+ + HTTP/1.1 202 + Transfer-Encoding: chunked + Location: http:///ws/v1/cluster/apps/application_1404204891930_0002 + Content-Type: application/xml + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + No response body ++---+ + +* Cluster Application State API + + With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response. + + Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. + + This feature is currently in the alpha stage and may change in the future. + +** URI + +----- + * http:///ws/v1/cluster/apps/{appid}/state +----- + +** HTTP Operations Supported + +------ + * GET + * PUT +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of object + + When you make a request for the state of an app, the information returned has the following fields + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" | +*---------------+--------------+--------------------------------+ + + +** Response Examples + + <> + + HTTP Request + +----- + GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state +----- + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"ACCEPTED" +} ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + Response Header: + ++---+ +HTTP/1.1 202 Accepted +Content-Type: application/json +Transfer-Encoding: chunked +Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"ACCEPTED" +} ++---+ + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + <> + + HTTP Request + +----- + GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state +----- + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/xml +Content-Length: 99 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + ACCEPTED + ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 202 Accepted +Content-Type: application/json +Content-Length: 794 +Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + ACCEPTED + ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/xml +Content-Length: 917 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + KILLED + ++---+ + + <> + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 403 Unauthorized +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + + <> + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + RUNNING + ++---+ + + Response Header: + ++---+ +HTTP/1.1 400 +Content-Length: 295 +Content-Type: application/xml +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + BadRequestException + java.lang.Exception: Only 'KILLED' is allowed as a target state. + org.apache.hadoop.yarn.webapp.BadRequestException + ++---+ + +