YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1528201 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza
parent
cc8a0cab59
commit
aa2745abe5
|
|
@ -85,6 +85,8 @@ Release 2.1.2 - UNRELEASED
|
|||
YARN-899. Added back queue level administrator-acls so that there is no
|
||||
regression w.r.t 1.x. (Xuan Gong via vinodkv)
|
||||
|
||||
YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
|||
|
|
@ -18,9 +18,12 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
|
@ -33,6 +36,9 @@ import org.apache.hadoop.yarn.util.resource.Resources;
|
|||
@Evolving
|
||||
public class FairSchedulerConfiguration extends Configuration {
|
||||
|
||||
public static final Log LOG = LogFactory.getLog(
|
||||
FairSchedulerConfiguration.class.getName());
|
||||
|
||||
/** Increment request grant-able by the RM scheduler.
|
||||
* These properties are looked up in the yarn-site.xml */
|
||||
public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB =
|
||||
|
|
@ -42,11 +48,10 @@ public class FairSchedulerConfiguration extends Configuration {
|
|||
YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores";
|
||||
public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1;
|
||||
|
||||
public static final String FS_CONFIGURATION_FILE = "fair-scheduler.xml";
|
||||
|
||||
private static final String CONF_PREFIX = "yarn.scheduler.fair.";
|
||||
|
||||
protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
|
||||
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
|
||||
protected static final String EVENT_LOG_DIR = "eventlog.dir";
|
||||
|
||||
/** Whether to use the user name as the queue name (instead of "default") if
|
||||
|
|
@ -105,7 +110,6 @@ public class FairSchedulerConfiguration extends Configuration {
|
|||
|
||||
public FairSchedulerConfiguration(Configuration conf) {
|
||||
super(conf);
|
||||
addResource(FS_CONFIGURATION_FILE);
|
||||
}
|
||||
|
||||
public Resource getMinimumAllocation() {
|
||||
|
|
@ -182,8 +186,28 @@ public class FairSchedulerConfiguration extends Configuration {
|
|||
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
|
||||
}
|
||||
|
||||
public String getAllocationFile() {
|
||||
return get(ALLOCATION_FILE);
|
||||
/**
|
||||
* Path to XML file containing allocations. If the
|
||||
* path is relative, it is searched for in the
|
||||
* classpath, but loaded like a regular File.
|
||||
*/
|
||||
public File getAllocationFile() {
|
||||
String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE);
|
||||
File allocFile = new File(allocFilePath);
|
||||
if (!allocFile.isAbsolute()) {
|
||||
URL url = Thread.currentThread().getContextClassLoader()
|
||||
.getResource(allocFilePath);
|
||||
if (url == null) {
|
||||
LOG.warn(allocFilePath + " not found on the classpath.");
|
||||
allocFile = null;
|
||||
} else if (!url.getProtocol().equalsIgnoreCase("file")) {
|
||||
throw new RuntimeException("Allocation file " + url
|
||||
+ " found on the classpath is not on the local filesystem.");
|
||||
} else {
|
||||
allocFile = new File(url.getPath());
|
||||
}
|
||||
}
|
||||
return allocFile;
|
||||
}
|
||||
|
||||
public String getEventlogDir() {
|
||||
|
|
|
|||
|
|
@ -75,11 +75,8 @@ public class QueueManager {
|
|||
|
||||
private final FairScheduler scheduler;
|
||||
|
||||
private Object allocFile; // Path to XML file containing allocations. This
|
||||
// is either a URL to specify a classpath resource
|
||||
// (if the fair-scheduler.xml on the classpath is
|
||||
// used) or a String to specify an absolute path (if
|
||||
// mapred.fairscheduler.allocation.file is used).
|
||||
// Path to XML file containing allocations.
|
||||
private File allocFile;
|
||||
|
||||
private final Collection<FSLeafQueue> leafQueues =
|
||||
new CopyOnWriteArrayList<FSLeafQueue>();
|
||||
|
|
@ -107,16 +104,7 @@ public class QueueManager {
|
|||
queues.put(rootQueue.getName(), rootQueue);
|
||||
|
||||
this.allocFile = conf.getAllocationFile();
|
||||
if (allocFile == null) {
|
||||
// No allocation file specified in jobconf. Use the default allocation
|
||||
// file, fair-scheduler.xml, looking for it on the classpath.
|
||||
allocFile = new Configuration().getResource("fair-scheduler.xml");
|
||||
if (allocFile == null) {
|
||||
LOG.error("The fair scheduler allocation file fair-scheduler.xml was "
|
||||
+ "not found on the classpath, and no other config file is given "
|
||||
+ "through mapred.fairscheduler.allocation.file.");
|
||||
}
|
||||
}
|
||||
|
||||
reloadAllocs();
|
||||
lastSuccessfulReload = scheduler.getClock().getTime();
|
||||
lastReloadAttempt = scheduler.getClock().getTime();
|
||||
|
|
@ -255,14 +243,7 @@ public class QueueManager {
|
|||
try {
|
||||
// Get last modified time of alloc file depending whether it's a String
|
||||
// (for a path name) or an URL (for a classloader resource)
|
||||
long lastModified;
|
||||
if (allocFile instanceof String) {
|
||||
File file = new File((String) allocFile);
|
||||
lastModified = file.lastModified();
|
||||
} else { // allocFile is an URL
|
||||
URLConnection conn = ((URL) allocFile).openConnection();
|
||||
lastModified = conn.getLastModified();
|
||||
}
|
||||
long lastModified = allocFile.lastModified();
|
||||
if (lastModified > lastSuccessfulReload &&
|
||||
time > lastModified + ALLOC_RELOAD_WAIT) {
|
||||
reloadAllocs();
|
||||
|
|
@ -321,12 +302,7 @@ public class QueueManager {
|
|||
DocumentBuilderFactory.newInstance();
|
||||
docBuilderFactory.setIgnoringComments(true);
|
||||
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
|
||||
Document doc;
|
||||
if (allocFile instanceof String) {
|
||||
doc = builder.parse(new File((String) allocFile));
|
||||
} else {
|
||||
doc = builder.parse(allocFile.toString());
|
||||
}
|
||||
Document doc = builder.parse(allocFile);
|
||||
Element root = doc.getDocumentElement();
|
||||
if (!"allocations".equals(root.getTagName()))
|
||||
throw new AllocationConfigurationException("Bad fair scheduler config " +
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
|
|||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import junit.framework.Assert;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
|
|
@ -55,4 +60,15 @@ public class TestFairSchedulerConfiguration {
|
|||
public void testGibberish() throws Exception {
|
||||
parseResourceConfigValue("1o24vc0res");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAllocationFileFromClasspath() {
|
||||
FairSchedulerConfiguration conf = new FairSchedulerConfiguration(
|
||||
new Configuration());
|
||||
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||
"test-fair-scheduler.xml");
|
||||
File allocationFile = conf.getAllocationFile();
|
||||
Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName());
|
||||
Assert.assertTrue(allocationFile.exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
<?xml version="1.0"?>
|
||||
<allocations>
|
||||
<queue name="queue1"></queue>
|
||||
</allocations>
|
||||
|
||||
|
|
@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair Scheduler
|
|||
distribute the resources assigned to them to their children in the same
|
||||
fashion. Applications may only be scheduled on leaf queues. Queues can be
|
||||
specified as children of other queues by placing them as sub-elements of
|
||||
their parents in the fair scheduler configuration file.
|
||||
their parents in the fair scheduler allocation file.
|
||||
|
||||
A queue's name starts with the names of its parents, with periods as
|
||||
separators. So a queue named "queue1" under the root queue, would be referred
|
||||
|
|
@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair Scheduler
|
|||
Customizing the Fair Scheduler typically involves altering two files. First,
|
||||
scheduler-wide options can be set by adding configuration properties in the
|
||||
yarn-site.xml file in your existing configuration directory. Second, in
|
||||
most cases users will want to create a manifest file listing which queues
|
||||
exist and their respective weights and capacities. The location of this file
|
||||
is flexible - but it must be declared in yarn-site.xml.
|
||||
most cases users will want to create an allocation file listing which queues
|
||||
exist and their respective weights and capacities. The allocation file
|
||||
is reloaded every 10 seconds, allowing changes to be made on the fly.
|
||||
|
||||
Properties that can be placed in yarn-site.xml
|
||||
|
||||
* <<<yarn.scheduler.fair.allocation.file>>>
|
||||
|
||||
* Path to allocation file. An allocation file is an XML manifest describing
|
||||
queues and their properties, in addition to certain policy defaults. This file
|
||||
must be in XML format as described in the next section.
|
||||
Defaults to fair-scheduler.xml in configuration directory.
|
||||
must be in the XML format described in the next section. If a relative path is
|
||||
given, the file is searched for on the classpath (which typically includes
|
||||
the Hadoop conf directory).
|
||||
Defaults to fair-scheduler.xml.
|
||||
|
||||
* <<<yarn.scheduler.fair.user-as-default-queue>>>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue