YARN-2505. Supported get/add/remove/change labels in RM REST API. Contributed by Craig Welch.

hadoop-yarn-project/CHANGES.txt

@@ -198,6 +198,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar
     Vavilapalli via jlowe)
 
+    YARN-2505. Supported get/add/remove/change labels in RM REST API. (Craig Welch
+    via zjshen)
+
   IMPROVEMENTS
 
     YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java

@@ -151,6 +151,13 @@ public class ConverterUtils {
   public static String toString(ContainerId cId) {
     return cId == null ? null : cId.toString();
   }
+  
+  public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
+    if (nodeIdStr.indexOf(":") < 0) {
+      return toNodeId(nodeIdStr + ":0");
+    }
+    return toNodeId(nodeIdStr);
+  }
 
   public static NodeId toNodeId(String nodeIdStr) {
     String[] parts = nodeIdStr.split(":");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.TestContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.junit.Test;
 
 public class TestConverterUtils {
@@ -85,4 +86,17 @@ public class TestConverterUtils {
   public void testContainerIdNull() throws URISyntaxException {
     assertNull(ConverterUtils.toString((ContainerId)null));
   }  
+  
+  @Test
+  public void testNodeIdWithDefaultPort() throws URISyntaxException {
+    NodeId nid;
+    
+    nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
+    assertEquals(nid.getPort(), 10);
+    assertEquals(nid.getHost(), "node");
+    
+    nid = ConverterUtils.toNodeIdWithDefaultPort("node");
+    assertEquals(nid.getPort(), 0);
+    assertEquals(nid.getHost(), "node");
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -24,6 +24,7 @@ import java.security.AccessControlException;
 import java.nio.ByteBuffer;
 import java.security.Principal;
 import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
@@ -133,6 +134,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -715,6 +718,179 @@ public class RMWebServices {
 
     return Response.status(Status.OK).entity(ret).build();
   }
+  
+  @GET
+  @Path("/get-node-to-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr) 
+    throws IOException {
+    init();
+
+    NodeToLabelsInfo ntl = new NodeToLabelsInfo();
+    HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
+    Map<NodeId, Set<String>> nodeIdToLabels =   
+      rm.getRMContext().getNodeLabelManager().getNodeLabels();
+      
+    for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
+      ntlMap.put(nitle.getKey().toString(), 
+        new NodeLabelsInfo(nitle.getValue()));
+    }
+
+    return ntl;
+  }
+  
+  @POST
+  @Path("/replace-node-to-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response replaceLabelsOnNodes(
+    final NodeToLabelsInfo newNodeToLabels,
+    @Context HttpServletRequest hsr) 
+    throws IOException {
+    init();
+    
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated for"
+        + " post to .../replace-node-to-labels";
+      throw new AuthorizationException(msg);
+    }
+    if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+      String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+        + " for post to .../replace-node-to-labels ";
+      throw new AuthorizationException(msg);
+    }
+    
+    Map<NodeId, Set<String>> nodeIdToLabels = 
+      new HashMap<NodeId, Set<String>>();
+
+    for (Map.Entry<String, NodeLabelsInfo> nitle : 
+      newNodeToLabels.getNodeToLabels().entrySet()) {
+     nodeIdToLabels.put(ConverterUtils.toNodeIdWithDefaultPort(nitle.getKey()),
+       new HashSet<String>(nitle.getValue().getNodeLabels()));
+    }
+    
+    rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeIdToLabels);
+
+    return Response.status(Status.OK).build();
+  }
+  
+  @GET
+  @Path("/get-node-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr) 
+    throws IOException {
+    init();
+
+    NodeLabelsInfo ret = 
+      new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager()
+        .getClusterNodeLabels());
+
+    return ret;
+  }
+  
+  @POST
+  @Path("/add-node-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
+      @Context HttpServletRequest hsr)
+      throws Exception {
+    init();
+    
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated for"
+        + " post to .../add-node-labels";
+      throw new AuthorizationException(msg);
+    }
+    if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+      String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+        + " for post to .../add-node-labels ";
+      throw new AuthorizationException(msg);
+    }
+    
+    rm.getRMContext().getNodeLabelManager()
+        .addToCluserNodeLabels(new HashSet<String>(
+          newNodeLabels.getNodeLabels()));
+            
+    return Response.status(Status.OK).build();
+
+  }
+  
+  @POST
+  @Path("/remove-node-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response removeFromCluserNodeLabels(final NodeLabelsInfo oldNodeLabels,
+      @Context HttpServletRequest hsr)
+      throws Exception {
+    init();
+    
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated for"
+        + " post to .../remove-node-labels";
+      throw new AuthorizationException(msg);
+    }
+    if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+      String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+        + " for post to .../remove-node-labels ";
+      throw new AuthorizationException(msg);
+    }
+    
+    rm.getRMContext().getNodeLabelManager()
+        .removeFromClusterNodeLabels(new HashSet<String>(
+          oldNodeLabels.getNodeLabels()));
+            
+    return Response.status(Status.OK).build();
+
+  }
+  
+  @GET
+  @Path("/nodes/{nodeId}/get-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
+                                  @PathParam("nodeId") String nodeId) 
+    throws IOException {
+    init();
+
+    NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
+    return new NodeLabelsInfo(
+      rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));
+
+  }
+  
+  @POST
+  @Path("/nodes/{nodeId}/replace-labels")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response replaceLabelsOnNode(NodeLabelsInfo newNodeLabelsInfo,
+      @Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId)
+      throws Exception {
+    init();
+    
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated for"
+        + " post to .../nodes/nodeid/replace-labels";
+      throw new AuthorizationException(msg);
+    }
+
+    if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+      String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+        + " for post to .../nodes/nodeid/replace-labels";
+      throw new AuthorizationException(msg);
+    }
+    
+    NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
+    
+    Map<NodeId, Set<String>> newLabelsForNode = new HashMap<NodeId,
+      Set<String>>();
+    
+    newLabelsForNode.put(nid, new HashSet<String>(newNodeLabelsInfo.getNodeLabels()));
+    
+    rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(newLabelsForNode);
+    
+    return Response.status(Status.OK).build();
+
+  }
 
   protected Response killApp(RMApp app, UserGroupInformation callerUGI,
       HttpServletRequest hsr) throws IOException, InterruptedException {
@@ -965,7 +1141,9 @@ public class RMWebServices {
           newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
           createAppSubmissionContextResource(newApp),
           newApp.getApplicationType(),
-          newApp.getKeepContainersAcrossApplicationAttempts());
+          newApp.getKeepContainersAcrossApplicationAttempts(),
+          newApp.getAppNodeLabelExpression(),
+          newApp.getAMContainerNodeLabelExpression());
     appContext.setApplicationTags(newApp.getApplicationTags());
 
     return appContext;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java

@@ -71,6 +71,12 @@ public class ApplicationSubmissionContextInfo {
   @XmlElementWrapper(name = "application-tags")
   @XmlElement(name = "tag")
   Set<String> tags;
+  
+  @XmlElement(name = "app-node-label-expression")
+  String appNodeLabelExpression;
+  
+  @XmlElement(name = "am-container-node-label-expression")
+  String amContainerNodeLabelExpression;
 
   public ApplicationSubmissionContextInfo() {
     applicationId = "";
@@ -83,6 +89,8 @@ public class ApplicationSubmissionContextInfo {
     keepContainers = false;
     applicationType = "";
     tags = new HashSet<String>();
+    appNodeLabelExpression = "";
+    amContainerNodeLabelExpression = "";
   }
 
   public String getApplicationId() {
@@ -132,6 +140,14 @@ public class ApplicationSubmissionContextInfo {
   public Set<String> getApplicationTags() {
     return tags;
   }
+  
+  public String getAppNodeLabelExpression() {
+    return appNodeLabelExpression;
+  }
+  
+  public String getAMContainerNodeLabelExpression() {
+    return amContainerNodeLabelExpression;
+  }
 
   public void setApplicationId(String applicationId) {
     this.applicationId = applicationId;
@@ -182,5 +198,12 @@ public class ApplicationSubmissionContextInfo {
   public void setApplicationTags(Set<String> tags) {
     this.tags = tags;
   }
+  
+  public void setAppNodeLabelExpression(String appNodeLabelExpression) {
+    this.appNodeLabelExpression = appNodeLabelExpression;
+  }
 
+  public void setAMContainerNodeLabelExpression(String nodeLabelExpression) {
+    this.amContainerNodeLabelExpression = nodeLabelExpression;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.*;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "nodeLabelsInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeLabelsInfo {
+
+  protected ArrayList<String> nodeLabels = new ArrayList<String>();
+
+  public NodeLabelsInfo() {
+  } // JAXB needs this
+  
+  public NodeLabelsInfo(ArrayList<String> nodeLabels) {
+   this.nodeLabels = nodeLabels; 
+  }
+  
+  public NodeLabelsInfo(Set<String> nodeLabelsSet) {
+   this.nodeLabels = new ArrayList<String>(nodeLabelsSet); 
+  }
+  
+  public ArrayList<String> getNodeLabels() {
+    return nodeLabels;
+  }
+  
+  public void setNodeLabels(ArrayList<String> nodeLabels) {
+    this.nodeLabels = nodeLabels; 
+  }
+  
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.*;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "nodeToLabelsInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeToLabelsInfo {
+
+  protected HashMap<String, NodeLabelsInfo> nodeToLabels = 
+    new HashMap<String, NodeLabelsInfo>();
+
+  public NodeToLabelsInfo() {
+  } // JAXB needs this
+  
+  public HashMap<String, NodeLabelsInfo> getNodeToLabels() {
+   return nodeToLabels; 
+  }
+  
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java

@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+public class TestRMWebServicesNodeLabels extends JerseyTest {
+
+  private static final Log LOG = LogFactory
+      .getLog(TestRMWebServicesNodeLabels.class);
+
+  private static MockRM rm;
+  private YarnConfiguration conf;
+
+  private String userName;
+  private String notUserName;
+
+  private Injector injector = Guice.createInjector(new ServletModule() {
+    @Override
+    protected void configureServlets() {
+      bind(JAXBContextResolver.class);
+      bind(RMWebServices.class);
+      bind(GenericExceptionHandler.class);
+      try {
+        userName = UserGroupInformation.getCurrentUser().getShortUserName();
+      } catch (IOException ioe) {
+        throw new RuntimeException("Unable to get current user name "
+            + ioe.getMessage(), ioe);
+      }
+      notUserName = userName + "abc123";
+      conf = new YarnConfiguration();
+      conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+      rm = new MockRM(conf);
+      bind(ResourceManager.class).toInstance(rm);
+      bind(RMContext.class).toInstance(rm.getRMContext());
+      filter("/*").through(
+          TestRMWebServicesAppsModification.TestRMCustomAuthFilter.class);
+      serve("/*").with(GuiceContainer.class);
+    }
+  });
+
+  public class GuiceServletConfig extends GuiceServletContextListener {
+
+    @Override
+    protected Injector getInjector() {
+      return injector;
+    }
+  }
+
+  public TestRMWebServicesNodeLabels() {
+    super(new WebAppDescriptor.Builder(
+        "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+        .contextListenerClass(GuiceServletConfig.class)
+        .filterClass(com.google.inject.servlet.GuiceFilter.class)
+        .contextPath("jersey-guice-filter").servletPath("/").build());
+  }
+
+  @Test
+  public void testNodeLabels() throws JSONException, Exception {
+    WebResource r = resource();
+
+    ClientResponse response;
+    JSONObject json;
+    JSONArray jarr;
+    String responseString;
+
+    // Add a label
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("add-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("a", json.getString("nodeLabels"));
+    
+    // Add another
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("add-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    jarr = json.getJSONArray("nodeLabels");
+    assertEquals(2, jarr.length());
+    
+    // Add labels to a node
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("replace-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+              MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    LOG.info("posted node nodelabel");
+
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("get-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    jarr = json.getJSONArray("nodeLabels");
+    assertEquals(2, jarr.length());
+    
+    // Replace
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("replace-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    LOG.info("posted node nodelabel");
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("get-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("a", json.getString("nodeLabels"));
+            
+    // Replace labels using node-to-labels
+    NodeToLabelsInfo ntli = new NodeToLabelsInfo();
+    NodeLabelsInfo nli = new NodeLabelsInfo();
+    nli.getNodeLabels().add("a");
+    nli.getNodeLabels().add("b");
+    ntli.getNodeToLabels().put("nid:0", nli);
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("replace-node-to-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity(toJson(ntli, NodeToLabelsInfo.class),
+              MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+        
+    // Verify, using node-to-labels
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-to-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    ntli = response.getEntity(NodeToLabelsInfo.class);
+    nli = ntli.getNodeToLabels().get("nid:0");
+    assertEquals(2, nli.getNodeLabels().size());
+    assertTrue(nli.getNodeLabels().contains("a"));
+    assertTrue(nli.getNodeLabels().contains("b"));
+    
+    // Remove all
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("replace-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    LOG.info("posted node nodelabel");
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("get-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("", json.getString("nodeLabels"));
+    
+    // Add a label back for auth tests
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("replace-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\": \"a\"}",
+              MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    LOG.info("posted node nodelabel");
+
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("get-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("a", json.getString("nodeLabels"));
+    
+    // Auth fail replace labels on node
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("replace-labels")
+            .queryParam("user.name", notUserName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+              MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("nodes").path("nid:0")
+            .path("get-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("a", json.getString("nodeLabels"));
+    
+    // Fail to add a label with post
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("add-node-labels").queryParam("user.name", notUserName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"c\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    jarr = json.getJSONArray("nodeLabels");
+    assertEquals(2, jarr.length());
+    
+    // Remove cluster label (succeed, we no longer need it)
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("remove-node-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("a", json.getString("nodeLabels"));
+    
+    
+    // Remove cluster label with post
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("remove-node-labels")
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+    // Verify
+    response =
+        r.path("ws").path("v1").path("cluster")
+            .path("get-node-labels").queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    String res = response.getEntity(String.class);
+    assertTrue(res.equals("null"));
+  }
+
+  @SuppressWarnings("rawtypes")
+  private String toJson(Object nsli, Class klass) throws Exception {
+    StringWriter sw = new StringWriter();
+    JSONJAXBContext ctx = new JSONJAXBContext(klass);
+    JSONMarshaller jm = ctx.createJSONMarshaller();
+    jm.marshallToJSON(nsli, sw);
+    return sw.toString();
+  }
+
+  @SuppressWarnings({ "rawtypes", "unchecked" })
+  private Object fromJson(String json, Class klass) throws Exception {
+    StringReader sr = new StringReader(json);
+    JSONJAXBContext ctx = new JSONJAXBContext(klass);
+    JSONUnmarshaller jm = ctx.createJSONUnmarshaller();
+    return jm.unmarshalFromJSON(sr, klass);
+  }
+
+}