Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge -r 1407432:1407433 from trunk to branch. Fixes: YARN-183. Clean up fair scheduler code. Contributed by Sandy Ryza. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1407434 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White 2012-11-09 12:39:26 +00:00
parent 3d73261636
commit c6dd3c0263
10 changed files with 132 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -52,6 +52,8 @@ Release 2.0.3-alpha - Unreleased
YARN-136. Make ClientToAMTokenSecretManager part of RMContext (Vinod Kumar
Vavilapalli via sseth)
YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite)
OPTIMIZATIONS
BUG FIXES

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
View File

@ -101,7 +101,7 @@ public class AppSchedulable extends Schedulable {
@Override
public Resource getResourceUsage() {
return this.app.getCurrentConsumption();
return app.getCurrentConsumption();
}
@ -114,7 +114,7 @@ public class AppSchedulable extends Schedulable {
* Get metrics reference from containing queue.
*/
public QueueMetrics getMetrics() {
return this.queue.getQueueSchedulable().getMetrics();
return queue.getQueueSchedulable().getMetrics();
}
@Override
@ -190,9 +190,9 @@ public class AppSchedulable extends Schedulable {
RMContainer rmContainer = application.reserve(node, priority, null,
container);
node.reserveResource(application, priority, rmContainer);
getMetrics().reserveResource(this.app.getUser(),
getMetrics().reserveResource(app.getUser(),
container.getResource());
scheduler.getRootQueueMetrics().reserveResource(this.app.getUser(),
scheduler.getRootQueueMetrics().reserveResource(app.getUser(),
container.getResource());
}
@ -257,13 +257,13 @@ public class AppSchedulable extends Schedulable {
// TODO this should subtract resource just assigned
// TEMPROARY
getMetrics().setAvailableResourcesToQueue(
this.scheduler.getClusterCapacity());
scheduler.getClusterCapacity());
}
// If we had previously made a reservation, delete it
if (reserved) {
this.unreserve(application, priority, node);
unreserve(application, priority, node);
}
// Inform the node
@ -290,7 +290,7 @@ public class AppSchedulable extends Schedulable {
// Make sure the application still needs requests at this priority
if (app.getTotalRequiredResources(priority) == 0) {
this.unreserve(app, priority, node);
unreserve(app, priority, node);
return Resources.none();
}
} else {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
View File

@ -61,7 +61,7 @@ public class FSQueue {
queueSchedulable.addApp(appSchedulable);
}
public void removeJob(FSSchedulerApp app) {
public void removeApp(FSSchedulerApp app) {
applications.remove(app);
queueSchedulable.removeApp(app);
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java
View File

@ -80,7 +80,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
this.scheduler = scheduler;
this.queue = queue;
this.queueMgr = scheduler.getQueueManager();
this.metrics = QueueMetrics.forQueue(this.getName(), null, true, scheduler.getConf());
this.metrics = QueueMetrics.forQueue(getName(), null, true, scheduler.getConf());
this.lastTimeAtMinShare = scheduler.getClock().getTime();
this.lastTimeAtHalfFairShare = scheduler.getClock().getTime();
}
@ -113,7 +113,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
Resource toAdd = sched.getDemand();
if (LOG.isDebugEnabled()) {
LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
+ "; Total resource consumption for " + this.getName() + " now "
+ "; Total resource consumption for " + getName() + " now "
+ demand);
}
demand = Resources.add(demand, toAdd);
@ -123,7 +123,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + this.getName() + " is " + demand
LOG.debug("The updated demand for " + getName() + " is " + demand
+ "; the max is " + maxRes);
}
}
@ -164,9 +164,9 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
@Override
public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
LOG.debug("Node offered to queue: " + this.getName() + " reserved: " + reserved);
LOG.debug("Node offered to queue: " + getName() + " reserved: " + reserved);
// If this queue is over its limit, reject
if (Resources.greaterThan(this.getResourceUsage(),
if (Resources.greaterThan(getResourceUsage(),
queueMgr.getMaxResources(queue.getName()))) {
return Resources.none();
}
@ -258,7 +258,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
@Override
public Map<QueueACL, AccessControlList> getQueueAcls() {
Map<QueueACL, AccessControlList> acls = this.queueMgr.getQueueAcls(this.getName());
Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
return new HashMap<QueueACL, AccessControlList>(acls);
}
@ -284,7 +284,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>();
for (QueueACL operation : QueueACL.values()) {
Map<QueueACL, AccessControlList> acls = this.queueMgr.getQueueAcls(this.getName());
Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
if (acls.get(operation).isUserAllowed(user)) {
operations.add(operation);
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
View File

@ -112,12 +112,12 @@ public class FSSchedulerApp extends SchedulerApplication {
}
public ApplicationId getApplicationId() {
return this.appSchedulingInfo.getApplicationId();
return appSchedulingInfo.getApplicationId();
}
@Override
public ApplicationAttemptId getApplicationAttemptId() {
return this.appSchedulingInfo.getApplicationAttemptId();
return appSchedulingInfo.getApplicationAttemptId();
}
public void setAppSchedulable(AppSchedulable appSchedulable) {
@ -129,7 +129,7 @@ public class FSSchedulerApp extends SchedulerApplication {
}
public String getUser() {
return this.appSchedulingInfo.getUser();
return appSchedulingInfo.getUser();
}
public synchronized void updateResourceRequests(
@ -138,19 +138,19 @@ public class FSSchedulerApp extends SchedulerApplication {
}
public Map<String, ResourceRequest> getResourceRequests(Priority priority) {
return this.appSchedulingInfo.getResourceRequests(priority);
return appSchedulingInfo.getResourceRequests(priority);
}
public int getNewContainerId() {
return this.appSchedulingInfo.getNewContainerId();
return appSchedulingInfo.getNewContainerId();
}
public Collection<Priority> getPriorities() {
return this.appSchedulingInfo.getPriorities();
return appSchedulingInfo.getPriorities();
}
public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) {
return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress);
return appSchedulingInfo.getResourceRequest(priority, nodeAddress);
}
public synchronized int getTotalRequiredResources(Priority priority) {
@ -158,7 +158,7 @@ public class FSSchedulerApp extends SchedulerApplication {
}
public Resource getResource(Priority priority) {
return this.appSchedulingInfo.getResource(priority);
return appSchedulingInfo.getResource(priority);
}
/**
@ -167,11 +167,11 @@ public class FSSchedulerApp extends SchedulerApplication {
*/
@Override
public boolean isPending() {
return this.appSchedulingInfo.isPending();
return appSchedulingInfo.isPending();
}
public String getQueueName() {
return this.appSchedulingInfo.getQueueName();
return appSchedulingInfo.getQueueName();
}
/**
@ -185,7 +185,7 @@ public class FSSchedulerApp extends SchedulerApplication {
public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) {
// Cleanup all scheduling information
this.appSchedulingInfo.stop(rmAppAttemptFinalState);
appSchedulingInfo.stop(rmAppAttemptFinalState);
}
@SuppressWarnings("unchecked")
@ -196,7 +196,7 @@ public class FSSchedulerApp extends SchedulerApplication {
getRMContainer(containerId);
if (rmContainer == null) {
// Some unknown container sneaked into the system. Kill it.
this.rmContext.getDispatcher().getEventHandler()
rmContext.getDispatcher().getEventHandler()
.handle(new RMNodeCleanContainerEvent(nodeId, containerId));
return;
}
@ -272,7 +272,7 @@ public class FSSchedulerApp extends SchedulerApplication {
}
synchronized public void addSchedulingOpportunity(Priority priority) {
this.schedulingOpportunities.setCount(priority,
schedulingOpportunities.setCount(priority,
schedulingOpportunities.count(priority) + 1);
}
@ -282,19 +282,19 @@ public class FSSchedulerApp extends SchedulerApplication {
* successfully did so.
*/
synchronized public int getSchedulingOpportunities(Priority priority) {
return this.schedulingOpportunities.count(priority);
return schedulingOpportunities.count(priority);
}
synchronized void resetReReservations(Priority priority) {
this.reReservations.setCount(priority, 0);
reReservations.setCount(priority, 0);
}
synchronized void addReReservation(Priority priority) {
this.reReservations.add(priority);
reReservations.add(priority);
}
synchronized public int getReReservations(Priority priority) {
return this.reReservations.count(priority);
return reReservations.count(priority);
}
public synchronized int getNumReservedContainers(Priority priority) {
@ -458,8 +458,8 @@ public class FSSchedulerApp extends SchedulerApplication {
* @param priority The priority of the container scheduled.
*/
synchronized public void resetSchedulingOpportunities(Priority priority) {
this.lastScheduledContainer.put(priority, System.currentTimeMillis());
this.schedulingOpportunities.setCount(priority, 0);
lastScheduledContainer.put(priority, System.currentTimeMillis());
schedulingOpportunities.setCount(priority, 0);
}
/**
@ -494,14 +494,14 @@ public class FSSchedulerApp extends SchedulerApplication {
rackLocalityThreshold;
// Relax locality constraints once we've surpassed threshold.
if (this.getSchedulingOpportunities(priority) > (numNodes * threshold)) {
if (getSchedulingOpportunities(priority) > (numNodes * threshold)) {
if (allowed.equals(NodeType.NODE_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL);
this.resetSchedulingOpportunities(priority);
resetSchedulingOpportunities(priority);
}
else if (allowed.equals(NodeType.RACK_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH);
this.resetSchedulingOpportunities(priority);
resetSchedulingOpportunities(priority);
}
}
return allowedLocalityLevel.get(priority);
@ -512,7 +512,7 @@ public class FSSchedulerApp extends SchedulerApplication {
Priority priority, ResourceRequest request,
Container container) {
// Update allowed locality level
NodeType allowed = this.allowedLocalityLevel.get(priority);
NodeType allowed = allowedLocalityLevel.get(priority);
if (allowed != null) {
if (allowed.equals(NodeType.OFF_SWITCH) &&
(type.equals(NodeType.NODE_LOCAL) ||
@ -532,9 +532,9 @@ public class FSSchedulerApp extends SchedulerApplication {
}
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container, this
.getApplicationAttemptId(), node.getNodeID(), this.rmContext
.getDispatcher().getEventHandler(), this.rmContext
RMContainer rmContainer = new RMContainerImpl(container,
getApplicationAttemptId(), node.getNodeID(), rmContext
.getDispatcher().getEventHandler(), rmContext
.getContainerAllocationExpirer());
// Add it to allContainers list.

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
View File

@ -67,25 +67,25 @@ public class FSSchedulerNode extends SchedulerNode {
}
public RMNode getRMNode() {
return this.rmNode;
return rmNode;
}
public NodeId getNodeID() {
return this.rmNode.getNodeID();
return rmNode.getNodeID();
}
public String getHttpAddress() {
return this.rmNode.getHttpAddress();
return rmNode.getHttpAddress();
}
@Override
public String getHostName() {
return this.rmNode.getHostName();
return rmNode.getHostName();
}
@Override
public String getRackName() {
return this.rmNode.getRackName();
return rmNode.getRackName();
}
/**
@ -112,17 +112,18 @@ public class FSSchedulerNode extends SchedulerNode {
@Override
public synchronized Resource getAvailableResource() {
return this.availableResource;
return availableResource;
}
@Override
public synchronized Resource getUsedResource() {
return this.usedResource;
return usedResource;
}
private synchronized boolean isValidContainer(Container c) {
if (launchedContainers.containsKey(c.getId()))
if (launchedContainers.containsKey(c.getId())) {
return true;
}
return false;
}

82
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -139,11 +139,11 @@ public class FairScheduler implements ResourceScheduler {
public FairSchedulerConfiguration getConf() {
return this.conf;
return conf;
}
public QueueManager getQueueManager() {
return this.queueMgr;
return queueMgr;
}
public List<FSQueueSchedulable> getQueueSchedulables() {
@ -183,8 +183,7 @@ public class FairScheduler implements ResourceScheduler {
* fair shares, deficits, minimum slot allocations, and amount of used and
* required resources per job.
*/
protected void update() {
synchronized (this) {
protected synchronized void update() {
queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file
updateRunnability(); // Set job runnability based on user/queue limits
updatePreemptionVariables(); // Determine if any queues merit preemption
@ -195,7 +194,7 @@ public class FairScheduler implements ResourceScheduler {
}
// Compute fair shares based on updated demands
List<FSQueueSchedulable> queueScheds = this.getQueueSchedulables();
List<FSQueueSchedulable> queueScheds = getQueueSchedulables();
SchedulingAlgorithms.computeFairShares(
queueScheds, clusterCapacity);
@ -213,7 +212,6 @@ public class FairScheduler implements ResourceScheduler {
// when fair share is calculated).
rootMetrics.setAvailableResourcesToQueue(clusterCapacity);
}
}
/**
* Update the preemption fields for all QueueScheduables, i.e. the times since
@ -257,17 +255,16 @@ public class FairScheduler implements ResourceScheduler {
* have been below half their fair share for the fairSharePreemptionTimeout.
* If such queues exist, compute how many tasks of each type need to be
* preempted and then select the right ones using preemptTasks.
*
* This method computes and logs the number of tasks we want to preempt even
* if preemption is disabled, for debugging purposes.
*/
protected void preemptTasksIfNecessary() {
if (!preemptionEnabled)
if (!preemptionEnabled) {
return;
}
long curTime = clock.getTime();
if (curTime - lastPreemptCheckTime < preemptionInterval)
if (curTime - lastPreemptCheckTime < preemptionInterval) {
return;
}
lastPreemptCheckTime = curTime;
Resource resToPreempt = Resources.none();
@ -288,8 +285,9 @@ public class FairScheduler implements ResourceScheduler {
* lowest priority to preempt.
*/
protected void preemptResources(List<FSQueueSchedulable> scheds, Resource toPreempt) {
if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none()))
if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) {
return;
}
Map<RMContainer, FSSchedulerApp> apps =
new HashMap<RMContainer, FSSchedulerApp>();
@ -330,7 +328,7 @@ public class FairScheduler implements ResourceScheduler {
// TODO: Not sure if this ever actually adds this to the list of cleanup
// containers on the RMNode (see SchedulerNode.releaseContainer()).
this.completedContainer(container, status, RMContainerEventType.KILL);
completedContainer(container, status, RMContainerEventType.KILL);
toPreempt = Resources.subtract(toPreempt,
container.getContainer().getResource());
@ -413,7 +411,7 @@ public class FairScheduler implements ResourceScheduler {
}
public RMContainerTokenSecretManager getContainerTokenSecretManager() {
return this.rmContext.getContainerTokenSecretManager();
return rmContext.getContainerTokenSecretManager();
}
public double getAppWeight(AppSchedulable app) {
@ -437,28 +435,28 @@ public class FairScheduler implements ResourceScheduler {
@Override
public Resource getMinimumResourceCapability() {
return this.minimumAllocation;
return minimumAllocation;
}
@Override
public Resource getMaximumResourceCapability() {
return this.maximumAllocation;
return maximumAllocation;
}
public double getNodeLocalityThreshold() {
return this.nodeLocalityThreshold;
return nodeLocalityThreshold;
}
public double getRackLocalityThreshold() {
return this.rackLocalityThreshold;
return rackLocalityThreshold;
}
public Resource getClusterCapacity() {
return this.clusterCapacity;
return clusterCapacity;
}
public Clock getClock() {
return this.clock;
return clock;
}
protected void setClock(Clock clock) {
@ -478,11 +476,11 @@ public class FairScheduler implements ResourceScheduler {
addApplication(ApplicationAttemptId applicationAttemptId,
String queueName, String user) {
FSQueue queue = this.queueMgr.getQueue(queueName);
FSQueue queue = queueMgr.getQueue(queueName);
FSSchedulerApp schedulerApp =
new FSSchedulerApp(applicationAttemptId, user,
queue.getQueueSchedulable(), new ActiveUsersManager(this.getRootQueueMetrics()),
queue.getQueueSchedulable(), new ActiveUsersManager(getRootQueueMetrics()),
rmContext, null);
// Inforce ACLs
@ -553,8 +551,8 @@ public class FairScheduler implements ResourceScheduler {
application.stop(rmAppAttemptFinalState);
// Inform the queue
FSQueue queue = this.queueMgr.getQueue(application.getQueue().getQueueName());
queue.removeJob(application);
FSQueue queue = queueMgr.getQueue(application.getQueue().getQueueName());
queue.removeApp(application);
// Remove from our data-structure
applications.remove(applicationAttemptId);
@ -600,7 +598,7 @@ public class FairScheduler implements ResourceScheduler {
}
private synchronized void addNode(RMNode node) {
this.nodes.put(node.getNodeID(), new FSSchedulerNode(node));
nodes.put(node.getNodeID(), new FSSchedulerNode(node));
Resources.addTo(clusterCapacity, node.getTotalCapability());
LOG.info("Added node " + node.getNodeAddress() +
@ -608,7 +606,7 @@ public class FairScheduler implements ResourceScheduler {
}
private synchronized void removeNode(RMNode rmNode) {
FSSchedulerNode node = this.nodes.get(rmNode.getNodeID());
FSSchedulerNode node = nodes.get(rmNode.getNodeID());
Resources.subtractFrom(clusterCapacity, rmNode.getTotalCapability());
// Remove running containers
@ -631,7 +629,7 @@ public class FairScheduler implements ResourceScheduler {
RMContainerEventType.KILL);
}
this.nodes.remove(rmNode.getNodeID());
nodes.remove(rmNode.getNodeID());
LOG.info("Removed node " + rmNode.getNodeAddress() +
" cluster capacity: " + clusterCapacity);
}
@ -669,9 +667,7 @@ public class FairScheduler implements ResourceScheduler {
}
synchronized (application) {
if (!ask.isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("allocate: pre-update" +
" applicationAttemptId=" + appAttemptId +
@ -764,7 +760,7 @@ public class FairScheduler implements ResourceScheduler {
int assignedContainers = 0;
while (true) {
// At most one task is scheduled each iteration of this loop
List<FSQueueSchedulable> scheds = this.getQueueSchedulables();
List<FSQueueSchedulable> scheds = getQueueSchedulables();
Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator());
boolean assignedContainer = false;
for (FSQueueSchedulable sched : scheds) {
@ -796,11 +792,11 @@ public class FairScheduler implements ResourceScheduler {
@Override
public SchedulerAppReport getSchedulerAppInfo(
ApplicationAttemptId appAttemptId) {
if (!this.applications.containsKey(appAttemptId)) {
if (!applications.containsKey(appAttemptId)) {
LOG.error("Request for appInfo of unknown attempt" + appAttemptId);
return null;
}
return new SchedulerAppReport(this.applications.get(appAttemptId));
return new SchedulerAppReport(applications.get(appAttemptId));
}
@Override
@ -812,37 +808,30 @@ public class FairScheduler implements ResourceScheduler {
public void handle(SchedulerEvent event) {
switch(event.getType()) {
case NODE_ADDED:
{
if (!(event instanceof NodeAddedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
addNode(nodeAddedEvent.getAddedRMNode());
}
break;
case NODE_REMOVED:
{
if (!(event instanceof NodeRemovedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
removeNode(nodeRemovedEvent.getRemovedRMNode());
}
break;
case NODE_UPDATE:
{
if (!(event instanceof NodeUpdateSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeUpdateSchedulerEvent nodeUpdatedEvent =
(NodeUpdateSchedulerEvent)event;
this.nodeUpdate(nodeUpdatedEvent.getRMNode(),
nodeUpdate(nodeUpdatedEvent.getRMNode(),
nodeUpdatedEvent.getNewlyLaunchedContainers(),
nodeUpdatedEvent.getCompletedContainers());
}
break;
case APP_ADDED:
{
if (!(event instanceof AppAddedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
@ -857,20 +846,16 @@ public class FairScheduler implements ResourceScheduler {
addApplication(appAddedEvent.getApplicationAttemptId(), queue,
appAddedEvent.getUser());
}
break;
case APP_REMOVED:
{
if (!(event instanceof AppRemovedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
this.removeApplication(appRemovedEvent.getApplicationAttemptID(),
removeApplication(appRemovedEvent.getApplicationAttemptID(),
appRemovedEvent.getFinalAttemptState());
}
break;
case CONTAINER_EXPIRED:
{
if (!(event instanceof ContainerExpiredSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
@ -882,7 +867,6 @@ public class FairScheduler implements ResourceScheduler {
containerId,
SchedulerUtils.EXPIRED_CONTAINER),
RMContainerEventType.EXPIRE);
}
break;
default:
LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
@ -897,9 +881,9 @@ public class FairScheduler implements ResourceScheduler {
@Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
if (!this.initialized) {
if (!initialized) {
this.conf = new FairSchedulerConfiguration(conf);
this.rootMetrics = QueueMetrics.forQueue("root", null, true, conf);
rootMetrics = QueueMetrics.forQueue("root", null, true, conf);
this.rmContext = rmContext;
this.clock = new SystemClock();
this.eventLog = new FairSchedulerEventLog();
@ -973,7 +957,7 @@ public class FairScheduler implements ResourceScheduler {
@Override
public int getNumClusterNodes() {
return this.nodes.size();
return nodes.size();
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewJobWeightBooster.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configured;
*/
@Private
@Unstable
public class NewJobWeightBooster extends Configured implements WeightAdjuster {
public class NewAppWeightBooster extends Configured implements WeightAdjuster {
private static final float DEFAULT_FACTOR = 3;
private static final long DEFAULT_DURATION = 5 * 60 * 1000;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
View File

@ -202,7 +202,7 @@ public class QueueManager {
* Get the queue for a given AppSchedulable.
*/
public FSQueue getQueueForApp(AppSchedulable app) {
return this.getQueue(app.getApp().getQueueName());
return getQueue(app.getApp().getQueueName());
}
/**
@ -472,8 +472,8 @@ public class QueueManager {
/**
* Remove an app
*/
public synchronized void removeJob(FSSchedulerApp app) {
getQueue(app.getQueueName()).removeJob(app);
public synchronized void removeApp(FSSchedulerApp app) {
getQueue(app.getQueueName()).removeApp(app);
}
/**
@ -575,6 +575,7 @@ public class QueueManager {
queueWeights = weights;
}
}
/**
* Get a queue's min share preemption timeout, in milliseconds. This is the
* time after which jobs in the queue may kill other queues' tasks if they

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
View File

@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configurable;
/**
* A pluggable object for altering the weights of apps in the fair scheduler,
* which is used for example by {@link NewJobWeightBooster} to give higher
* which is used for example by {@link NewAppWeightBooster} to give higher
* weight to new jobs so that short jobs finish faster.
*
* May implement {@link Configurable} to access configuration parameters.