YARN-8559. Expose mutable-conf scheduler's configuration in RM /scheduler-conf endpoint. Contributed by Weiwei Yang.
This commit is contained in:
Jonathan Hung
parent
2024260af6
commit
7abffe4529
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
|
||||
|
||||
|
|
@ -58,6 +59,12 @@ public interface MutableConfigurationProvider {
|
|||
*/
|
||||
void confirmPendingMutation(boolean isValid) throws Exception;
|
||||
|
||||
/**
|
||||
* Returns scheduler configuration cached in this provider.
|
||||
* @return scheduler configuration.
|
||||
*/
|
||||
Configuration getConfiguration();
|
||||
|
||||
/**
|
||||
* Closes the configuration provider, releasing any required resources.
|
||||
* @throws IOException on failure to close
|
||||
|
|
|
|||
|
|
@ -122,6 +122,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
|
|||
return new CapacitySchedulerConfiguration(loadedConf, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConfiguration() {
|
||||
return new Configuration(schedConf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConfigurationMutationACLPolicy getAclMutationPolicy() {
|
||||
return aclMutationPolicy;
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
|
||||
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.server.webapp.WebServices;
|
||||
|
|
@ -2424,4 +2425,36 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/scheduler-conf")
|
||||
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
|
||||
public Response getSchedulerConfiguration(@Context HttpServletRequest hsr)
|
||||
throws AuthorizationException {
|
||||
// Only admin user is allowed to read scheduler conf,
|
||||
// in order to avoid leaking sensitive info, such as ACLs
|
||||
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
|
||||
initForWritableEndpoints(callerUGI, true);
|
||||
|
||||
ResourceScheduler scheduler = rm.getResourceScheduler();
|
||||
if (scheduler instanceof MutableConfScheduler
|
||||
&& ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
|
||||
MutableConfigurationProvider mutableConfigurationProvider =
|
||||
((MutableConfScheduler) scheduler).getMutableConfProvider();
|
||||
// We load the cached configuration from configuration store,
|
||||
// this should be the conf properties used by the scheduler.
|
||||
Configuration schedulerConf = mutableConfigurationProvider
|
||||
.getConfiguration();
|
||||
return Response.status(Status.OK)
|
||||
.entity(new ConfInfo(schedulerConf))
|
||||
.build();
|
||||
} else {
|
||||
return Response.status(Status.BAD_REQUEST).entity(
|
||||
"This API only supports to retrieve scheduler configuration"
|
||||
+ " from a mutable-conf scheduler, underneath scheduler "
|
||||
+ scheduler.getClass().getSimpleName()
|
||||
+ " is not an instance of MutableConfScheduler")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
|
||||
@XmlRootElement(name = "configuration")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class ConfInfo {
|
||||
|
||||
protected ArrayList<ConfItem> property = new ArrayList<>();
|
||||
|
||||
public ConfInfo() {
|
||||
} // JAXB needs this
|
||||
|
||||
public ConfInfo(Configuration conf) {
|
||||
for (Map.Entry<String, String> entry : conf) {
|
||||
add(new ConfItem(entry.getKey(), entry.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
public void add(ConfItem confItem) {
|
||||
property.add(confItem);
|
||||
}
|
||||
|
||||
public ArrayList<ConfItem> getItems() {
|
||||
return property;
|
||||
}
|
||||
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public static class ConfItem {
|
||||
|
||||
private String name;
|
||||
private String value;
|
||||
|
||||
public ConfItem() {
|
||||
// JAXB needs this
|
||||
}
|
||||
|
||||
public ConfItem(String name, String value){
|
||||
this.name = name;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -40,6 +40,9 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
|||
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
|
||||
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
|
||||
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
|
||||
import org.codehaus.jettison.json.JSONArray;
|
||||
import org.codehaus.jettison.json.JSONException;
|
||||
import org.codehaus.jettison.json.JSONObject;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
@ -55,6 +58,7 @@ import java.util.Map;
|
|||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
/**
|
||||
* Test scheduler configuration mutation via REST API.
|
||||
|
|
@ -159,8 +163,40 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
|
|||
config.setCapacity(c1, 0f);
|
||||
}
|
||||
|
||||
private CapacitySchedulerConfiguration getSchedulerConf()
|
||||
throws JSONException {
|
||||
WebResource r = resource();
|
||||
ClientResponse response =
|
||||
r.path("ws").path("v1").path("cluster")
|
||||
.queryParam("user.name", userName).path("scheduler-conf")
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(Status.OK.getStatusCode(), response.getStatus());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
JSONArray items = (JSONArray) json.get("property");
|
||||
CapacitySchedulerConfiguration parsedConf =
|
||||
new CapacitySchedulerConfiguration();
|
||||
for (int i=0; i<items.length(); i++) {
|
||||
JSONObject obj = (JSONObject) items.get(i);
|
||||
parsedConf.set(obj.get("name").toString(),
|
||||
obj.get("value").toString());
|
||||
}
|
||||
return parsedConf;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSchedulerConf() throws Exception {
|
||||
CapacitySchedulerConfiguration orgConf = getSchedulerConf();
|
||||
assertNotNull(orgConf);
|
||||
assertEquals(3, orgConf.getQueues("root").length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddNestedQueue() throws Exception {
|
||||
CapacitySchedulerConfiguration orgConf = getSchedulerConf();
|
||||
assertNotNull(orgConf);
|
||||
assertEquals(3, orgConf.getQueues("root").length);
|
||||
|
||||
WebResource r = resource();
|
||||
|
||||
ClientResponse response;
|
||||
|
|
@ -200,6 +236,10 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
|
|||
0.01f);
|
||||
assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
|
||||
0.01f);
|
||||
|
||||
CapacitySchedulerConfiguration newConf = getSchedulerConf();
|
||||
assertNotNull(newConf);
|
||||
assertEquals(4, newConf.getQueues("root").length);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -4445,6 +4445,7 @@ Please note that this feature is currently in the alpha stage and is subject to
|
|||
|
||||
### HTTP Operations Supported
|
||||
|
||||
* GET
|
||||
* PUT
|
||||
|
||||
### Elements of the *sched-conf* object
|
||||
|
|
@ -4456,6 +4457,45 @@ Please note that this feature is currently in the alpha stage and is subject to
|
|||
| remove-queue | string | Full path name of a queue to remove |
|
||||
| global-updates | map | Map of key value pairs to update scheduler's global configuration |
|
||||
|
||||
### GET Request Examples
|
||||
|
||||
Get requests are used to retrieve the scheduler's configuration that is currently loaded into scheduler's context.
|
||||
|
||||
**XML response**
|
||||
|
||||
HTTP Request:
|
||||
|
||||
Accept: application/xml
|
||||
Content-Type: application/xml
|
||||
GET http://rm-http-address:port/ws/v1/cluster/scheduler-conf
|
||||
|
||||
Response Header:
|
||||
|
||||
TTP/1.1 200 OK
|
||||
Content-Type: application/xml; charset=utf-8
|
||||
Transfer-Encoding: chunked
|
||||
|
||||
Response Body:
|
||||
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>yarn.scheduler.capacity.root.queues</name>
|
||||
<value>default</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.scheduler.capacity.maximum-applications</name>
|
||||
<value>10000</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.scheduler.capacity.root.default.capacity</name>
|
||||
<value>100</value>
|
||||
</property>
|
||||
</configuration>
|
||||
```
|
||||
|
||||
### PUT Request Examples
|
||||
|
||||
Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response.
|
||||
|
|
|
|||
Loading…
Reference in New Issue