YARN-9146. Added REST API to configure auxiliary service.
Contributed by Billie Rinaldi
This commit is contained in:
Eric Yang
parent
0ef54f72a6
commit
2fa9389c2e
|
|
@ -64,7 +64,10 @@ The collector class configuration may specify a comma-separated list of collecto
|
|||
### NodeManager Configuration properties, `yarn-site.xml` in all nodes:
|
||||
|
||||
There are two ways to configure auxiliary services, through a manifest file or through the Configuration (the old way). If a manifest file is used, auxiliary service configurations are not read from the Configuration.
|
||||
If using a manifest file, the file name should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`. Otherwise, set the following properties to configure aux services through the Configuration.
|
||||
|
||||
If using a manifest file, the file name can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`, or the file may be sent to the NM via a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. If the file name is set in the Configuration, NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically).
|
||||
|
||||
Otherwise, set the following properties to configure aux services through the Configuration.
|
||||
|
||||
| **Property** | **Default Value** | **Explanation** |
|
||||
|:---- |:---- |:---- |
|
||||
|
|
|
|||
|
|
@ -484,6 +484,22 @@ public class AuxServices extends AbstractService
|
|||
loadManifest(getConfig(), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reloads auxiliary services. Must be called after service init.
|
||||
*
|
||||
* @param services a list of auxiliary services
|
||||
* @throws IOException if aux services have not been started yet
|
||||
*/
|
||||
public void reload(AuxServiceRecords services) throws IOException {
|
||||
if (getServiceState() != Service.STATE.STARTED) {
|
||||
throw new IOException("Auxiliary services have not been started yet, " +
|
||||
"please retry later");
|
||||
}
|
||||
LOG.info("Received list of auxiliary services: " + mapper
|
||||
.writeValueAsString(services));
|
||||
loadServices(services, getConfig(), true);
|
||||
}
|
||||
|
||||
private boolean checkManifestPermissions(FileStatus status) throws
|
||||
IOException {
|
||||
if ((status.getPermission().toShort() & 0022) != 0) {
|
||||
|
|
@ -578,6 +594,19 @@ public class AuxServices extends AbstractService
|
|||
return;
|
||||
}
|
||||
AuxServiceRecords services = maybeReadManifestFile();
|
||||
loadServices(services, conf, startServices);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates current aux services based on changes found in the service list.
|
||||
*
|
||||
* @param services list of auxiliary services
|
||||
* @param conf configuration
|
||||
* @param startServices if true starts services, otherwise only inits services
|
||||
* @throws IOException
|
||||
*/
|
||||
private synchronized void loadServices(AuxServiceRecords services,
|
||||
Configuration conf, boolean startServices) throws IOException {
|
||||
if (services == null) {
|
||||
// read did not occur or no changes detected
|
||||
return;
|
||||
|
|
@ -613,7 +642,7 @@ public class AuxServices extends AbstractService
|
|||
}
|
||||
}
|
||||
|
||||
// remove aux services that do not appear in the manifest
|
||||
// remove aux services that do not appear in the new list
|
||||
Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
|
||||
servicesToRemove.removeAll(loadedAuxServices);
|
||||
for (String sName : servicesToRemove) {
|
||||
|
|
@ -622,7 +651,7 @@ public class AuxServices extends AbstractService
|
|||
}
|
||||
|
||||
if (!foundChanges) {
|
||||
LOG.info("No auxiliary services changes detected in manifest");
|
||||
LOG.info("No auxiliary services changes detected");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import java.util.Map.Entry;
|
|||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo;
|
||||
|
|
@ -575,6 +576,27 @@ public class NMWebServices {
|
|||
return auxiliaryServices;
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Path("/auxiliaryservices")
|
||||
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
|
||||
public Response putAuxiliaryServices(@javax.ws.rs.core.Context
|
||||
HttpServletRequest req, AuxServiceRecords services) {
|
||||
if (!hasAdminAccess(req)) {
|
||||
return Response.status(Status.FORBIDDEN).build();
|
||||
}
|
||||
if (services == null) {
|
||||
return Response.status(Status.BAD_REQUEST).build();
|
||||
}
|
||||
try {
|
||||
nmContext.getAuxServices().reload(services);
|
||||
} catch (Exception e) {
|
||||
LOG.error("Fail to reload auxiliary services, reason: ", e);
|
||||
return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e).build();
|
||||
}
|
||||
return Response.ok().build();
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Path("/yarn/sysfs/{user}/{appId}")
|
||||
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
|
|
|
|||
|
|
@ -898,4 +898,27 @@ public class TestAuxServices {
|
|||
aux.loadManifest(conf, false);
|
||||
assertEquals(0, aux.getServices().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualReload() throws IOException {
|
||||
Configuration conf = getABConf();
|
||||
final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
|
||||
MOCK_CONTEXT, MOCK_DEL_SERVICE);
|
||||
aux.init(conf);
|
||||
try {
|
||||
aux.reload(null);
|
||||
Assert.fail("Should receive the exception.");
|
||||
} catch (IOException e) {
|
||||
assertTrue("Wrong message: " + e.getMessage(),
|
||||
e.getMessage().equals("Auxiliary services have not been started " +
|
||||
"yet, please retry later"));
|
||||
}
|
||||
aux.start();
|
||||
assertEquals(2, aux.getServices().size());
|
||||
aux.reload(null);
|
||||
assertEquals(2, aux.getServices().size());
|
||||
aux.reload(new AuxServiceRecords());
|
||||
assertEquals(0, aux.getServices().size());
|
||||
aux.stop();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,7 +106,8 @@ To launch auxiliary services on a NodeManager, users have to add their jar to No
|
|||
### Manifest
|
||||
This section describes the auxiliary service manifest for aux-service classpath isolation.
|
||||
|
||||
The manifest file should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`.
|
||||
The manifest file can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically).
|
||||
Alternatively, the manifest file may be sent to the NM via REST API by making a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`.
|
||||
|
||||
An example manifest that configures classpath isolation for a CustomAuxService follows. One or more files may be specified to make up the classpath of a service, with jar or archive formats being supported.
|
||||
```
|
||||
|
|
|
|||
|
|
@ -591,13 +591,16 @@ Auxiliary Services API
|
|||
|
||||
With the auxiliary services API, you can obtain a collection of resources, each of which represents an auxiliary service. When you run a GET operation on this resource, you obtain a collection of auxiliary service information objects.
|
||||
|
||||
A YARN admin can use a PUT operation to update the auxiliary services running on the NodeManager. The body of the request should be of the same format as an auxiliary services manifest file.
|
||||
|
||||
### URI
|
||||
|
||||
* http://nm-http-address:port/ws/v1/node/auxiliaryservices
|
||||
|
||||
### HTTP Operations Supported
|
||||
|
||||
* GET
|
||||
* GET
|
||||
* PUT
|
||||
|
||||
### Query Parameters Supported
|
||||
|
||||
|
|
@ -611,7 +614,7 @@ When you make a request for the list of auxiliary services, the information will
|
|||
|:---- |:---- |:---- |
|
||||
| services | array of service information objects(JSON)/zero or more service information objects (XML) | A collection of service information objects |
|
||||
|
||||
### Response Examples
|
||||
### GET Response Examples
|
||||
|
||||
**JSON response**
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue