From 9a4e0d343e9e891c10ef6682e7b2231a59e69ade Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Fri, 7 Nov 2014 20:35:46 -0800 Subject: [PATCH] YARN-2505. Supported get/add/remove/change labels in RM REST API. Contributed by Craig Welch. --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/util/ConverterUtils.java | 7 + .../hadoop/yarn/util/TestConverterUtils.java | 14 + .../resourcemanager/webapp/RMWebServices.java | 180 ++++++++- .../dao/ApplicationSubmissionContextInfo.java | 23 ++ .../webapp/dao/NodeLabelsInfo.java | 52 +++ .../webapp/dao/NodeToLabelsInfo.java | 41 ++ .../webapp/TestRMWebServicesNodeLabels.java | 357 ++++++++++++++++++ 8 files changed, 676 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1e6406a9a2b..748ffe078a7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -198,6 +198,9 @@ Release 2.6.0 - UNRELEASED YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar Vavilapalli via jlowe) + YARN-2505. Supported get/add/remove/change labels in RM REST API. (Craig Welch + via zjshen) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index 27f7bc107e7..012d7999f9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -151,6 +151,13 @@ private static ApplicationId toApplicationId( public static String toString(ContainerId cId) { return cId == null ? null : cId.toString(); } + + public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) { + if (nodeIdStr.indexOf(":") < 0) { + return toNodeId(nodeIdStr + ":0"); + } + return toNodeId(nodeIdStr); + } public static NodeId toNodeId(String nodeIdStr) { String[] parts = nodeIdStr.split(":"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java index 824e6c003b1..7d537855348 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.TestContainerId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.api.records.NodeId; import org.junit.Test; public class TestConverterUtils { @@ -85,4 +86,17 @@ public void testContainerIdWithEpoch() throws URISyntaxException { public void testContainerIdNull() throws URISyntaxException { assertNull(ConverterUtils.toString((ContainerId)null)); } + + @Test + public void testNodeIdWithDefaultPort() throws URISyntaxException { + NodeId nid; + + nid = ConverterUtils.toNodeIdWithDefaultPort("node:10"); + assertEquals(nid.getPort(), 10); + assertEquals(nid.getHost(), "node"); + + nid = ConverterUtils.toNodeIdWithDefaultPort("node"); + assertEquals(nid.getPort(), 0); + assertEquals(nid.getHost(), "node"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 87c895ac14b..cf0a83ac860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import java.security.Principal; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; @@ -133,6 +134,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -715,6 +718,179 @@ public Response updateAppState(AppState targetState, return Response.status(Status.OK).entity(ret).build(); } + + @GET + @Path("/get-node-to-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr) + throws IOException { + init(); + + NodeToLabelsInfo ntl = new NodeToLabelsInfo(); + HashMap ntlMap = ntl.getNodeToLabels(); + Map> nodeIdToLabels = + rm.getRMContext().getNodeLabelManager().getNodeLabels(); + + for (Map.Entry> nitle : nodeIdToLabels.entrySet()) { + ntlMap.put(nitle.getKey().toString(), + new NodeLabelsInfo(nitle.getValue())); + } + + return ntl; + } + + @POST + @Path("/replace-node-to-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response replaceLabelsOnNodes( + final NodeToLabelsInfo newNodeToLabels, + @Context HttpServletRequest hsr) + throws IOException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated for" + + " post to .../replace-node-to-labels"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User " + callerUGI.getShortUserName() + " not authorized" + + " for post to .../replace-node-to-labels "; + throw new AuthorizationException(msg); + } + + Map> nodeIdToLabels = + new HashMap>(); + + for (Map.Entry nitle : + newNodeToLabels.getNodeToLabels().entrySet()) { + nodeIdToLabels.put(ConverterUtils.toNodeIdWithDefaultPort(nitle.getKey()), + new HashSet(nitle.getValue().getNodeLabels())); + } + + rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeIdToLabels); + + return Response.status(Status.OK).build(); + } + + @GET + @Path("/get-node-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr) + throws IOException { + init(); + + NodeLabelsInfo ret = + new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager() + .getClusterNodeLabels()); + + return ret; + } + + @POST + @Path("/add-node-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated for" + + " post to .../add-node-labels"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User " + callerUGI.getShortUserName() + " not authorized" + + " for post to .../add-node-labels "; + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager() + .addToCluserNodeLabels(new HashSet( + newNodeLabels.getNodeLabels())); + + return Response.status(Status.OK).build(); + + } + + @POST + @Path("/remove-node-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response removeFromCluserNodeLabels(final NodeLabelsInfo oldNodeLabels, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated for" + + " post to .../remove-node-labels"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User " + callerUGI.getShortUserName() + " not authorized" + + " for post to .../remove-node-labels "; + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager() + .removeFromClusterNodeLabels(new HashSet( + oldNodeLabels.getNodeLabels())); + + return Response.status(Status.OK).build(); + + } + + @GET + @Path("/nodes/{nodeId}/get-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr, + @PathParam("nodeId") String nodeId) + throws IOException { + init(); + + NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); + return new NodeLabelsInfo( + rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid)); + + } + + @POST + @Path("/nodes/{nodeId}/replace-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response replaceLabelsOnNode(NodeLabelsInfo newNodeLabelsInfo, + @Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated for" + + " post to .../nodes/nodeid/replace-labels"; + throw new AuthorizationException(msg); + } + + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User " + callerUGI.getShortUserName() + " not authorized" + + " for post to .../nodes/nodeid/replace-labels"; + throw new AuthorizationException(msg); + } + + NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); + + Map> newLabelsForNode = new HashMap>(); + + newLabelsForNode.put(nid, new HashSet(newNodeLabelsInfo.getNodeLabels())); + + rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(newLabelsForNode); + + return Response.status(Status.OK).build(); + + } protected Response killApp(RMApp app, UserGroupInformation callerUGI, HttpServletRequest hsr) throws IOException, InterruptedException { @@ -965,7 +1141,9 @@ protected ApplicationSubmissionContext createAppSubmissionContext( newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), - newApp.getKeepContainersAcrossApplicationAttempts()); + newApp.getKeepContainersAcrossApplicationAttempts(), + newApp.getAppNodeLabelExpression(), + newApp.getAMContainerNodeLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index f7233e6e3af..5278b3ed53e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -71,6 +71,12 @@ public class ApplicationSubmissionContextInfo { @XmlElementWrapper(name = "application-tags") @XmlElement(name = "tag") Set tags; + + @XmlElement(name = "app-node-label-expression") + String appNodeLabelExpression; + + @XmlElement(name = "am-container-node-label-expression") + String amContainerNodeLabelExpression; public ApplicationSubmissionContextInfo() { applicationId = ""; @@ -83,6 +89,8 @@ public ApplicationSubmissionContextInfo() { keepContainers = false; applicationType = ""; tags = new HashSet(); + appNodeLabelExpression = ""; + amContainerNodeLabelExpression = ""; } public String getApplicationId() { @@ -132,6 +140,14 @@ public boolean getKeepContainersAcrossApplicationAttempts() { public Set getApplicationTags() { return tags; } + + public String getAppNodeLabelExpression() { + return appNodeLabelExpression; + } + + public String getAMContainerNodeLabelExpression() { + return amContainerNodeLabelExpression; + } public void setApplicationId(String applicationId) { this.applicationId = applicationId; @@ -182,5 +198,12 @@ public void setApplicationType(String applicationType) { public void setApplicationTags(Set tags) { this.tags = tags; } + + public void setAppNodeLabelExpression(String appNodeLabelExpression) { + this.appNodeLabelExpression = appNodeLabelExpression; + } + public void setAMContainerNodeLabelExpression(String nodeLabelExpression) { + this.amContainerNodeLabelExpression = nodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java new file mode 100644 index 00000000000..1cb895aa6b0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "nodeLabelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodeLabelsInfo { + + protected ArrayList nodeLabels = new ArrayList(); + + public NodeLabelsInfo() { + } // JAXB needs this + + public NodeLabelsInfo(ArrayList nodeLabels) { + this.nodeLabels = nodeLabels; + } + + public NodeLabelsInfo(Set nodeLabelsSet) { + this.nodeLabels = new ArrayList(nodeLabelsSet); + } + + public ArrayList getNodeLabels() { + return nodeLabels; + } + + public void setNodeLabels(ArrayList nodeLabels) { + this.nodeLabels = nodeLabels; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java new file mode 100644 index 00000000000..f2e64418f1a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "nodeToLabelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodeToLabelsInfo { + + protected HashMap nodeToLabels = + new HashMap(); + + public NodeToLabelsInfo() { + } // JAXB needs this + + public HashMap getNodeToLabels() { + return nodeToLabels; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java new file mode 100644 index 00000000000..3c958f24c3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -0,0 +1,357 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.api.json.JSONUnmarshaller; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesNodeLabels extends JerseyTest { + + private static final Log LOG = LogFactory + .getLog(TestRMWebServicesNodeLabels.class); + + private static MockRM rm; + private YarnConfiguration conf; + + private String userName; + private String notUserName; + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + try { + userName = UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException ioe) { + throw new RuntimeException("Unable to get current user name " + + ioe.getMessage(), ioe); + } + notUserName = userName + "abc123"; + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + filter("/*").through( + TestRMWebServicesAppsModification.TestRMCustomAuthFilter.class); + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + public TestRMWebServicesNodeLabels() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testNodeLabels() throws JSONException, Exception { + WebResource r = resource(); + + ClientResponse response; + JSONObject json; + JSONArray jarr; + String responseString; + + // Add a label + response = + r.path("ws").path("v1").path("cluster") + .path("add-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("nodeLabels")); + + // Add another + response = + r.path("ws").path("v1").path("cluster") + .path("add-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + jarr = json.getJSONArray("nodeLabels"); + assertEquals(2, jarr.length()); + + // Add labels to a node + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("replace-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\": [\"a\", \"b\"]}", + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + LOG.info("posted node nodelabel"); + + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("get-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + jarr = json.getJSONArray("nodeLabels"); + assertEquals(2, jarr.length()); + + // Replace + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("replace-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + LOG.info("posted node nodelabel"); + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("get-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("nodeLabels")); + + // Replace labels using node-to-labels + NodeToLabelsInfo ntli = new NodeToLabelsInfo(); + NodeLabelsInfo nli = new NodeLabelsInfo(); + nli.getNodeLabels().add("a"); + nli.getNodeLabels().add("b"); + ntli.getNodeToLabels().put("nid:0", nli); + response = + r.path("ws").path("v1").path("cluster") + .path("replace-node-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(ntli, NodeToLabelsInfo.class), + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify, using node-to-labels + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-to-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ntli = response.getEntity(NodeToLabelsInfo.class); + nli = ntli.getNodeToLabels().get("nid:0"); + assertEquals(2, nli.getNodeLabels().size()); + assertTrue(nli.getNodeLabels().contains("a")); + assertTrue(nli.getNodeLabels().contains("b")); + + // Remove all + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("replace-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + LOG.info("posted node nodelabel"); + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("get-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("", json.getString("nodeLabels")); + + // Add a label back for auth tests + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("replace-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\": \"a\"}", + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + LOG.info("posted node nodelabel"); + + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("get-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("nodeLabels")); + + // Auth fail replace labels on node + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("replace-labels") + .queryParam("user.name", notUserName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\": [\"a\", \"b\"]}", + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nid:0") + .path("get-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("nodeLabels")); + + // Fail to add a label with post + response = + r.path("ws").path("v1").path("cluster") + .path("add-node-labels").queryParam("user.name", notUserName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"c\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + jarr = json.getJSONArray("nodeLabels"); + assertEquals(2, jarr.length()); + + // Remove cluster label (succeed, we no longer need it) + response = + r.path("ws").path("v1").path("cluster") + .path("remove-node-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("nodeLabels")); + + + // Remove cluster label with post + response = + r.path("ws").path("v1").path("cluster") + .path("remove-node-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + // Verify + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + String res = response.getEntity(String.class); + assertTrue(res.equals("null")); + } + + @SuppressWarnings("rawtypes") + private String toJson(Object nsli, Class klass) throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(nsli, sw); + return sw.toString(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Object fromJson(String json, Class klass) throws Exception { + StringReader sr = new StringReader(json); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONUnmarshaller jm = ctx.createJSONUnmarshaller(); + return jm.unmarshalFromJSON(sr, klass); + } + +}