From 7b676c5632adc9a8de204240b434529388f42a78 Mon Sep 17 00:00:00 2001 From: Sunil Date: Thu, 24 Nov 2016 17:37:36 +0530 Subject: [PATCH] YARN-5649. Add REST endpoints for updating application timeouts. Contributed by Rohith Sharma K S --- .../resourcemanager/ClientRMService.java | 2 +- .../server/resourcemanager/RMAuditLogger.java | 1 + .../server/resourcemanager/RMServerUtils.java | 2 +- .../webapp/JAXBContextResolver.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 185 ++++++++++++++++++ .../resourcemanager/webapp/dao/AppInfo.java | 24 +++ .../webapp/dao/AppTimeoutInfo.java | 71 +++++++ .../webapp/dao/AppTimeoutsInfo.java | 47 +++++ .../resourcemanager/rmapp/MockRMApp.java | 3 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../TestRMWebServicesAppsModification.java | 132 ++++++++++++- 11 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 21a9d13f154..8a0f373e00b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1734,7 +1734,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_TIMEOUTS, "UNKNOWN", "ClientRMService", ex.getMessage()); - throw RPCUtil.getRemoteException(ex); + throw ex; } RMAuditLogger.logSuccess(callerUGI.getShortUserName(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index d52e002f863..051d979e12e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -66,6 +66,7 @@ public static class AuditConstants { "Update Application Priority"; public static final String UPDATE_APP_TIMEOUTS = "Update Application Timeouts"; + public static final String GET_APP_TIMEOUTS = "Get Application Timeouts"; public static final String CHANGE_CONTAINER_RESOURCE = "AM Changed Container Resource"; public static final String SIGNAL_CONTAINER = "Signal Container Request"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 98a34d9e6a4..45a1f30afdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -515,7 +515,7 @@ public static Map validateISO8601AndConvertToLocal String message = "Expire time is not in ISO8601 format. ISO8601 supported " + "format is yyyy-MM-dd'T'HH:mm:ss.SSSZ"; - throw new YarnException(message); + throw new YarnException(message, ex); } if (expireTime < currentTimeMillis) { String message = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index b61072d5388..2f50a24eac3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -54,7 +54,8 @@ public JAXBContextResolver() throws Exception { CapacitySchedulerQueueInfoList.class, ResourceInfo.class, UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, - FairSchedulerQueueInfoList.class}; + FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, + AppTimeoutInfo.class }; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 4c692fe2376..6023ccb0d17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -24,9 +24,11 @@ import java.security.AccessControlException; import java.security.Principal; import java.security.PrivilegedExceptionAction; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -95,10 +97,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -147,6 +151,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; @@ -189,6 +195,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.AdHocLogDumper; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -2388,4 +2395,182 @@ public ReservationListResponse run() throws IOException, return Response.status(Status.OK).entity(resResponse).build(); } + @GET + @Path("/apps/{appid}/timeout/{type}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppTimeoutInfo getAppTimeout(@Context HttpServletRequest hsr, + @PathParam("appid") String appId, @PathParam("type") String type) + throws AuthorizationException { + init(); + RMApp app = validateAppTimeoutRequest(hsr, appId); + + ApplicationTimeoutType appTimeoutType = parseTimeoutType(type); + Long timeoutValue = app.getApplicationTimeouts().get(appTimeoutType); + AppTimeoutInfo timeout = + constructAppTimeoutDao(appTimeoutType, timeoutValue); + return timeout; + } + + private RMApp validateAppTimeoutRequest(HttpServletRequest hsr, + String appId) { + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + String userName = "UNKNOWN-USER"; + if (callerUGI != null) { + userName = callerUGI.getUserName(); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_TIMEOUTS, + "UNKNOWN", "RMWebService", msg); + throw new ForbiddenException(msg); + } + + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_TIMEOUTS, + "UNKNOWN", "RMWebService", + "Trying to get timeouts of an absent application " + appId); + throw e; + } + return app; + } + + @GET + @Path("/apps/{appid}/timeouts") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppTimeoutsInfo getAppTimeouts(@Context HttpServletRequest hsr, + @PathParam("appid") String appId) throws AuthorizationException { + init(); + + RMApp app = validateAppTimeoutRequest(hsr, appId); + + AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); + Map applicationTimeouts = + app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent as default + // with expiryTime=UNLIMITED and remainingTime=-1 + timeouts + .add(constructAppTimeoutDao(ApplicationTimeoutType.LIFETIME, null)); + } else { + for (Entry timeout : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeoutInfo = + constructAppTimeoutDao(timeout.getKey(), timeout.getValue()); + timeouts.add(timeoutInfo); + } + } + return timeouts; + } + + private ApplicationTimeoutType parseTimeoutType(String type) { + try { + // enum string is in the uppercase + return ApplicationTimeoutType + .valueOf(StringUtils.toUpperCase(type.trim())); + } catch (RuntimeException e) { + ApplicationTimeoutType[] typeArray = ApplicationTimeoutType.values(); + String allAppTimeoutTypes = Arrays.toString(typeArray); + throw new BadRequestException("Invalid application-state " + type.trim() + + " specified. It should be one of " + allAppTimeoutTypes); + } + } + + private AppTimeoutInfo constructAppTimeoutDao(ApplicationTimeoutType type, + Long timeoutInMillis) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(type); + if (timeoutInMillis != null) { + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis.longValue())); + timeout.setRemainingTime( + Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + return timeout; + } + + @PUT + @Path("/apps/{appid}/timeout") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, + @Context HttpServletRequest hsr, @PathParam("appid") String appId) + throws AuthorizationException, YarnException, InterruptedException, + IOException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException( + "Unable to obtain user name, user not authenticated"); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + return Response.status(Status.FORBIDDEN) + .entity("The default static user cannot carry out this operation.") + .build(); + } + + String userName = callerUGI.getUserName(); + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.UPDATE_APP_TIMEOUTS, + "UNKNOWN", "RMWebService", + "Trying to update timeout of an absent application " + appId); + throw e; + } + + return updateApplicationTimeouts(app, callerUGI, appTimeout); + } + + private Response updateApplicationTimeouts(final RMApp app, + UserGroupInformation callerUGI, final AppTimeoutInfo appTimeout) + throws IOException, InterruptedException { + + if (appTimeout.getTimeoutType() == null) { + return Response.status(Status.BAD_REQUEST).entity("Timeout type is null.") + .build(); + } + + String userName = callerUGI.getUserName(); + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + UpdateApplicationTimeoutsRequest request = + UpdateApplicationTimeoutsRequest + .newInstance(app.getApplicationId(), Collections.singletonMap( + appTimeout.getTimeoutType(), appTimeout.getExpireTime())); + rm.getClientRMService().updateApplicationTimeouts(request); + return null; + } + }); + } catch (UndeclaredThrowableException ue) { + // if the root cause is a permissions issue + // bubble that up to the user + if (ue.getCause() instanceof YarnException) { + YarnException ye = (YarnException) ue.getCause(); + if (ye.getCause() instanceof AccessControlException) { + String appId = app.getApplicationId().toString(); + String msg = "Unauthorized attempt to change timeout of app " + appId + + " by remote user " + userName; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } else if (ye.getCause() instanceof ParseException) { + return Response.status(Status.BAD_REQUEST).entity(ye.getMessage()) + .build(); + } else { + throw ue; + } + } else { + throw ue; + } + } + AppTimeoutInfo timeout = constructAppTimeoutDao(appTimeout.getTimeoutType(), + app.getApplicationTimeouts().get(appTimeout.getTimeoutType())); + return Response.status(Status.OK).entity(timeout).build(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index d3ddf80394e..2d364f474c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; @@ -114,6 +116,7 @@ public class AppInfo { protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; + protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); public AppInfo() { } // JAXB needs this @@ -240,6 +243,27 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, : null; } } + + Map applicationTimeouts = + app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent as default + // with expiryTime=UNLIMITED and remainingTime=-1 + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeouts.add(timeoutInfo); + } else { + for (Map.Entry entry : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); + timeout.setRemainingTime(Math + .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + timeouts.add(timeout); + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java new file mode 100644 index 00000000000..140857e4063 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; + +/** + * DAO object to display Application timeout information. + */ +@XmlRootElement(name = "timeout") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppTimeoutInfo { + + @XmlElement(name = "type") + private ApplicationTimeoutType timeoutType; + + @XmlElement(name = "expiryTime") + private String expiryTime; + + @XmlElement(name = "remainingTimeInSeconds") + private long remainingTimeInSec; + + public AppTimeoutInfo() { + expiryTime = "UNLIMITED"; + remainingTimeInSec = -1; + } + + public ApplicationTimeoutType getTimeoutType() { + return timeoutType; + } + + public String getExpireTime() { + return expiryTime; + } + + public long getRemainingTimeInSec() { + return remainingTimeInSec; + } + + public void setTimeoutType(ApplicationTimeoutType type) { + this.timeoutType = type; + } + + public void setExpiryTime(String expiryTime) { + this.expiryTime = expiryTime; + } + + public void setRemainingTime(long remainingTime) { + this.remainingTimeInSec = remainingTime; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java new file mode 100644 index 00000000000..256e55caa38 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * This class hosts a set of AppTimeout DAO objects. + */ +@XmlRootElement(name = "timeouts") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppTimeoutsInfo { + + @XmlElement(name = "timeout") + private ArrayList timeouts = new ArrayList(); + + public AppTimeoutsInfo() { + } // JAXB needs this + + public void add(AppTimeoutInfo timeoutInfo) { + timeouts.add(timeoutInfo); + } + + public ArrayList getAppTimeouts() { + return timeouts; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 5dbe8582882..bcc4be1ca5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -305,7 +306,7 @@ public CallerContext getCallerContext() { @Override public Map getApplicationTimeouts() { - throw new UnsupportedOperationException("Not supported yet."); + return Collections.emptyMap(); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 8db633f4598..c1815bf2b33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1370,7 +1370,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - int expectedNumberOfElements = 34; + int expectedNumberOfElements = 35; String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index d2810f78d44..23853534cb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -81,14 +82,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo; -import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.After; @@ -115,6 +118,7 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.LoggingFilter; +import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; @@ -1282,4 +1286,130 @@ protected static void verifyAppPriorityXML(ClientResponse response, assertEquals(queue, responseQueue); } + @Test(timeout = 90000) + public void testUpdateAppTimeout() throws Exception { + client().addFilter(new LoggingFilter(System.out)); + + rm.start(); + rm.registerNode("127.0.0.1:1234", 2048); + String[] mediaTypes = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + MediaType[] contentTypes = + { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; + for (String mediaType : mediaTypes) { + for (MediaType contentType : contentTypes) { + // application submitted without timeout + RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + + ClientResponse response = + this.constructWebResource("apps", app.getApplicationId().toString(), + "timeouts").accept(mediaType).get(ClientResponse.class); + if (mediaType.contains(MediaType.APPLICATION_JSON)) { + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject js = + response.getEntity(JSONObject.class).getJSONObject("timeouts"); + JSONArray entity = js.getJSONArray("timeout"); + verifyAppTimeoutJson(entity.getJSONObject(0), + ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1); + } + + AppTimeoutInfo timeoutUpdate = new AppTimeoutInfo(); + long timeOutFromNow = 60; + String expireTime = Times + .formatISO8601(System.currentTimeMillis() + timeOutFromNow * 1000); + timeoutUpdate.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeoutUpdate.setExpiryTime(expireTime); + + Object entity; + if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) { + entity = appTimeoutToJSON(timeoutUpdate); + } else { + entity = timeoutUpdate; + } + response = this + .constructWebResource("apps", app.getApplicationId().toString(), + "timeout") + .entity(entity, contentType).accept(mediaType) + .put(ClientResponse.class); + + if (!isAuthenticationEnabled()) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + continue; + } + assertEquals(Status.OK, response.getClientResponseStatus()); + if (mediaType.contains(MediaType.APPLICATION_JSON)) { + verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } else { + verifyAppTimeoutXML(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } + + // invoke get + response = + this.constructWebResource("apps", app.getApplicationId().toString(), + "timeout", ApplicationTimeoutType.LIFETIME.toString()) + .accept(mediaType).get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + if (mediaType.contains(MediaType.APPLICATION_JSON)) { + verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } + } + } + rm.stop(); + } + + protected static void verifyAppTimeoutJson(ClientResponse response, + ApplicationTimeoutType type, String expireTime, long timeOutFromNow) + throws JSONException { + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject jsonTimeout = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, jsonTimeout.length()); + JSONObject json = jsonTimeout.getJSONObject("timeout"); + verifyAppTimeoutJson(json, type, expireTime, timeOutFromNow); + } + + protected static void verifyAppTimeoutJson(JSONObject json, + ApplicationTimeoutType type, String expireTime, long timeOutFromNow) + throws JSONException { + assertEquals("incorrect number of elements", 3, json.length()); + assertEquals(type.toString(), json.getString("type")); + assertEquals(expireTime, json.getString("expiryTime")); + assertTrue(json.getLong("remainingTimeInSeconds") <= timeOutFromNow); + } + + protected static void verifyAppTimeoutXML(ClientResponse response, + ApplicationTimeoutType type, String expireTime, long timeOutFromNow) + throws ParserConfigurationException, IOException, SAXException { + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("timeout"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + assertEquals(type.toString(), + WebServicesTestUtils.getXmlString(element, "type")); + assertEquals(expireTime, + WebServicesTestUtils.getXmlString(element, "expiryTime")); + assertTrue(WebServicesTestUtils.getXmlLong(element, + "remainingTimeInSeconds") < timeOutFromNow); + } + + protected static String appTimeoutToJSON(AppTimeoutInfo timeout) + throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext( + JSONConfiguration.natural().rootUnwrapping(false).build(), + AppTimeoutInfo.class); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(timeout, sw); + jm.marshallToJSON(timeout, System.out); + return sw.toString(); + } + }