YARN-9146. Added REST API to configure auxiliary service.

Contributed by Billie Rinaldi

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md

@@ -64,7 +64,10 @@ The collector class configuration may specify a comma-separated list of collecto
 ### NodeManager Configuration properties, `yarn-site.xml` in all nodes:
 
 There are two ways to configure auxiliary services, through a manifest file or through the Configuration (the old way). If a manifest file is used, auxiliary service configurations are not read from the Configuration.
-If using a manifest file, the file name should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`. Otherwise, set the following properties to configure aux services through the Configuration.
+
+If using a manifest file, the file name can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`, or the file may be sent to the NM via a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. If the file name is set in the Configuration, NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically).
+
+Otherwise, set the following properties to configure aux services through the Configuration.
 
 | **Property** | **Default Value** | **Explanation** |
 |:---- |:---- |:---- |

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java

@@ -484,6 +484,22 @@ public class AuxServices extends AbstractService
     loadManifest(getConfig(), true);
   }
 
+  /**
+   * Reloads auxiliary services. Must be called after service init.
+   *
+   * @param services a list of auxiliary services
+   * @throws IOException if aux services have not been started yet
+   */
+  public void reload(AuxServiceRecords services) throws IOException {
+    if (getServiceState() != Service.STATE.STARTED) {
+      throw new IOException("Auxiliary services have not been started yet, " +
+          "please retry later");
+    }
+    LOG.info("Received list of auxiliary services: " + mapper
+        .writeValueAsString(services));
+    loadServices(services, getConfig(), true);
+  }
+
   private boolean checkManifestPermissions(FileStatus status) throws
       IOException {
     if ((status.getPermission().toShort() & 0022) != 0) {
@@ -578,6 +594,19 @@ public class AuxServices extends AbstractService
       return;
     }
     AuxServiceRecords services = maybeReadManifestFile();
+    loadServices(services, conf, startServices);
+  }
+
+  /**
+   * Updates current aux services based on changes found in the service list.
+   *
+   * @param services list of auxiliary services
+   * @param conf configuration
+   * @param startServices if true starts services, otherwise only inits services
+   * @throws IOException
+   */
+  private synchronized void loadServices(AuxServiceRecords services,
+      Configuration conf, boolean startServices) throws IOException {
     if (services == null) {
       // read did not occur or no changes detected
       return;
@@ -613,7 +642,7 @@ public class AuxServices extends AbstractService
       }
     }
 
-    // remove aux services that do not appear in the manifest
+    // remove aux services that do not appear in the new list
     Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
     servicesToRemove.removeAll(loadedAuxServices);
     for (String sName : servicesToRemove) {
@@ -622,7 +651,7 @@ public class AuxServices extends AbstractService
     }
 
     if (!foundChanges) {
-      LOG.info("No auxiliary services changes detected in manifest");
+      LOG.info("No auxiliary services changes detected");
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -31,6 +31,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo;
@@ -575,6 +576,27 @@ public class NMWebServices {
     return auxiliaryServices;
   }
 
+  @PUT
+  @Path("/auxiliaryservices")
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public Response putAuxiliaryServices(@javax.ws.rs.core.Context
+      HttpServletRequest req, AuxServiceRecords services) {
+    if (!hasAdminAccess(req)) {
+      return Response.status(Status.FORBIDDEN).build();
+    }
+    if (services == null) {
+      return Response.status(Status.BAD_REQUEST).build();
+    }
+    try {
+      nmContext.getAuxServices().reload(services);
+    } catch (Exception e) {
+      LOG.error("Fail to reload auxiliary services, reason: ", e);
+      return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e).build();
+    }
+    return Response.ok().build();
+  }
+
   @PUT
   @Path("/yarn/sysfs/{user}/{appId}")
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java

@@ -898,4 +898,27 @@ public class TestAuxServices {
     aux.loadManifest(conf, false);
     assertEquals(0, aux.getServices().size());
   }
+
+  @Test
+  public void testManualReload() throws IOException {
+    Configuration conf = getABConf();
+    final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
+        MOCK_CONTEXT, MOCK_DEL_SERVICE);
+    aux.init(conf);
+    try {
+      aux.reload(null);
+      Assert.fail("Should receive the exception.");
+    } catch (IOException e) {
+      assertTrue("Wrong message: " + e.getMessage(),
+          e.getMessage().equals("Auxiliary services have not been started " +
+              "yet, please retry later"));
+    }
+    aux.start();
+    assertEquals(2, aux.getServices().size());
+    aux.reload(null);
+    assertEquals(2, aux.getServices().size());
+    aux.reload(new AuxServiceRecords());
+    assertEquals(0, aux.getServices().size());
+    aux.stop();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md

@@ -106,7 +106,8 @@ To launch auxiliary services on a NodeManager, users have to add their jar to No
 ### Manifest
 This section describes the auxiliary service manifest for aux-service classpath isolation.
 
-The manifest file should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`.
+The manifest file can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically).
+Alternatively, the manifest file may be sent to the NM via REST API by making a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`.
 
 An example manifest that configures classpath isolation for a CustomAuxService follows. One or more files may be specified to make up the classpath of a service, with jar or archive formats being supported.
 ```

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md

@@ -591,6 +591,8 @@ Auxiliary Services API
 
 With the auxiliary services API, you can obtain a collection of resources, each of which represents an auxiliary service. When you run a GET operation on this resource, you obtain a collection of auxiliary service information objects.
 
+A YARN admin can use a PUT operation to update the auxiliary services running on the NodeManager. The body of the request should be of the same format as an auxiliary services manifest file.
+
 ### URI
 
       * http://nm-http-address:port/ws/v1/node/auxiliaryservices
@@ -598,6 +600,7 @@ With the auxiliary services API, you can obtain a collection of resources, each
 ### HTTP Operations Supported
 
  * GET
+ * PUT
 
 ### Query Parameters Supported
 
@@ -611,7 +614,7 @@ When you make a request for the list of auxiliary services, the information will
 |:---- |:---- |:---- |
 | services | array of service information objects(JSON)/zero or more service information objects (XML) | A collection of service information objects |
 
-### Response Examples
+### GET Response Examples
 
 **JSON response**