Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9781. SchedConfCli to get current stored scheduler configuration. Contributed by Prabhu Joseph

This commit is contained in:
Szilard Nemeth 2019-12-05 20:31:24 +01:00
parent 520fe2c99b
commit c71befaf8f
4 changed files with 151 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
View File

@ -38,13 +38,24 @@ import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@ -65,12 +76,16 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String REMOVE_QUEUES_OPTION = "removeQueues";
private static final String UPDATE_QUEUES_OPTION = "updateQueues";
private static final String GLOBAL_OPTIONS = "globalUpdates";
private static final String GET_SCHEDULER_CONF = "getConf";
private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String CONF_ERR_MSG = "Specify configuration key " +
"value as confKey=confVal.";
private SSLFactory sslFactory;
private Client client;
public SchedConfCLI() {
super(new YarnConfiguration());
}
@ -93,6 +108,8 @@ public class SchedConfCLI extends Configured implements Tool {
"Update queue configurations");
opts.addOption("global", GLOBAL_OPTIONS, true,
"Update global scheduler configurations");
opts.addOption("getconf", GET_SCHEDULER_CONF, false,
"Get current scheduler configurations");
opts.addOption("format", FORMAT_CONF, false,
"Format Scheduler Configuration and reload from" +
" capacity-scheduler.xml");
@ -115,6 +132,7 @@ public class SchedConfCLI extends Configured implements Tool {
boolean hasOption = false;
boolean format = false;
boolean getConf = false;
SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
try {
if (parsedCli.hasOption(ADD_QUEUES_OPTION)) {
@ -139,6 +157,10 @@ public class SchedConfCLI extends Configured implements Tool {
hasOption = true;
format = true;
}
if (parsedCli.hasOption(GET_SCHEDULER_CONF)) {
hasOption = true;
getConf = true;
}
} catch (IllegalArgumentException e) {
System.err.println(e.getMessage());
@ -154,27 +176,98 @@ public class SchedConfCLI extends Configured implements Tool {
Configuration conf = getConf();
if (format) {
return WebAppUtils.execOnActiveRM(conf, this::formatSchedulerConf, null);
} else if (getConf) {
return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf, null);
} else {
return WebAppUtils.execOnActiveRM(conf,
this::updateSchedulerConfOnRMNode, updateInfo);
}
}
private static void prettyFormatWithIndent(String input, int indent)
throws Exception {
Source xmlInput = new StreamSource(new StringReader(input));
StringWriter sw = new StringWriter();
StreamResult xmlOutput = new StreamResult(sw);
TransformerFactory transformerFactory = TransformerFactory.newInstance();
transformerFactory.setAttribute("indent-number", indent);
Transformer transformer = transformerFactory.newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.transform(xmlInput, xmlOutput);
System.out.println(xmlOutput.getWriter().toString());
}
private WebResource initializeWebResource(String webAppAddress) {
Configuration conf = getConf();
if (YarnConfiguration.useHttps(conf)) {
sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
}
client = createWebServiceClient(sslFactory);
return client.resource(webAppAddress);
}
private void destroyClient() {
if (client != null) {
client.destroy();
}
if (sslFactory != null) {
sslFactory.destroy();
}
}
@VisibleForTesting
int getSchedulerConf(String webAppAddress, WebResource resource)
throws Exception {
ClientResponse response = null;
resource = (resource != null) ? resource :
initializeWebResource(webAppAddress);
try {
Builder builder;
if (UserGroupInformation.isSecurityEnabled()) {
builder = resource
.path("ws").path("v1").path("cluster")
.path("scheduler-conf").accept(MediaType.APPLICATION_XML);
} else {
builder = resource
.path("ws").path("v1").path("cluster").path("scheduler-conf")
.queryParam("user.name", UserGroupInformation.getCurrentUser()
.getShortUserName()).accept(MediaType.APPLICATION_XML);
}
response = builder.get(ClientResponse.class);
if (response != null) {
if (response.getStatus() == Status.OK.getStatusCode()) {
ConfInfo schedulerConf = response.getEntity(ConfInfo.class);
JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class);
Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
StringWriter sw = new StringWriter();
jaxbMarshaller.marshal(schedulerConf, sw);
prettyFormatWithIndent(sw.toString(), 2);
return 0;
} else {
System.err.println("Failed to get scheduler configuration: "
+ response.getEntity(String.class));
}
} else {
System.err.println("Failed to get scheduler configuration: " +
"null response");
}
return -1;
} finally {
if (response != null) {
response.close();
}
destroyClient();
}
}
@VisibleForTesting
int formatSchedulerConf(String webAppAddress, WebResource resource)
throws Exception {
Configuration conf = getConf();
SSLFactory clientSslFactory = null;
if (YarnConfiguration.useHttps(conf)) {
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
}
Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
resource = (resource != null) ? resource :
webServiceClient.resource(webAppAddress);
initializeWebResource(webAppAddress);
try {
Builder builder = null;
Builder builder;
if (UserGroupInformation.isSecurityEnabled()) {
builder = resource
.path("ws").path("v1").path("cluster")
@ -206,27 +299,15 @@ public class SchedConfCLI extends Configured implements Tool {
if (response != null) {
response.close();
}
if (webServiceClient != null) {
webServiceClient.destroy();
}
if (clientSslFactory != null) {
clientSslFactory.destroy();
}
destroyClient();
}
}
@VisibleForTesting
int updateSchedulerConfOnRMNode(String webAppAddress,
SchedConfUpdateInfo updateInfo) throws Exception {
Configuration conf = getConf();
SSLFactory clientSslFactory = null;
if (YarnConfiguration.useHttps(conf)) {
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
}
Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
WebResource resource = webServiceClient.resource(webAppAddress);
WebResource resource = initializeWebResource(webAppAddress);
try {
Builder builder = null;
if (UserGroupInformation.isSecurityEnabled()) {
@ -258,12 +339,7 @@ public class SchedConfCLI extends Configured implements Tool {
if (response != null) {
response.close();
}
if (webServiceClient != null) {
webServiceClient.destroy();
}
if (clientSslFactory != null) {
clientSslFactory.destroy();
}
destroyClient();
}
}
@ -380,7 +456,8 @@ public class SchedConfCLI extends Configured implements Tool {
+ "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
+ "[-global globalConfKey1=globalConfVal1,"
+ "globalConfKey2=globalConfVal2] "
+ "[-format]\n"
+ "[-format] "
+ "[-getconf]\n"
+ "Example (adding queues): yarn schedulerconf -add "
+ "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ "maximum-capacity=0\"\n"
@ -393,6 +470,8 @@ public class SchedConfCLI extends Configured implements Tool {
+ "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ "Example (format scheduler configuration): yarn schedulerconf "
+ "-format\n"
+ "Example (get scheduler configuration): yarn schedulerconf "
+ "-getconf\n"
+ "Note: This is an alpha feature, the syntax/options are subject to "
+ "change, please run at your own risk.");
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
View File

@ -200,6 +200,38 @@ public class TestSchedConfCLI extends JerseyTestBase {
config.setMaximumCapacity(a, 100f);
}
private void cleanUp() throws Exception {
if (rm != null) {
rm.stop();
}
CONF_FILE.delete();
if (CONF_FILE.exists()) {
throw new RuntimeException("Failed to delete configuration file");
}
if (OLD_CONF_FILE.exists()) {
if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
throw new RuntimeException("Failed to re-copy old" +
" configuration file");
}
}
super.tearDown();
}
@Test(timeout = 10000)
public void testGetSchedulerConf() throws Exception {
try {
super.setUp();
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
int exitCode = cli.getSchedulerConf("", resource());
assertEquals("SchedConfCLI failed to run", 0, exitCode);
assertTrue("Failed to get scheduler configuration",
sysOutStream.toString().contains("testqueue"));
} finally {
cleanUp();
}
}
@Test(timeout = 10000)
public void testFormatSchedulerConf() throws Exception {
try {
@ -229,17 +261,7 @@ public class TestSchedConfCLI extends JerseyTestBase {
schedulerConf = provider.getConfiguration();
assertNull(schedulerConf.get("schedKey1"));
} finally {
if (rm != null) {
rm.stop();
}
CONF_FILE.delete();
if (OLD_CONF_FILE.exists()) {
if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
throw new RuntimeException("Failed to re-copy old" +
" configuration file");
}
}
super.tearDown();
cleanUp();
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
package org.apache.hadoop.yarn.webapp.dao;
import org.apache.hadoop.conf.Configuration;
@ -24,11 +24,14 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
/**
* This class stores the Scheduler Configuration.
*/
@XmlRootElement(name = "configuration")
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfInfo {
protected ArrayList<ConfItem> property = new ArrayList<>();
private ArrayList<ConfItem> property = new ArrayList<>();
public ConfInfo() {
} // JAXB needs this
@ -46,6 +49,9 @@ public class ConfInfo {
return property;
}
/**
* This class stores the Configuration Property.
*/
@XmlAccessorType(XmlAccessType.FIELD)
public static class ConfItem {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -197,7 +197,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices;
@ -210,6 +209,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;