From 89a7c9843605efcc9f6d6ee6df47260b035520dd Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 15 Apr 2015 09:56:32 -0700 Subject: [PATCH] YARN-3318. Create Initial OrderingPolicy Framework and FifoOrderingPolicy. (Craig Welch via wangda) (cherry picked from commit 5004e753322084e42dfda4be1d2db66677f86a1e) --- hadoop-yarn-project/CHANGES.txt | 3 + .../dev-support/findbugs-exclude.xml | 4 + .../scheduler/ResourceUsage.java | 55 +++++++- .../AbstractComparatorOrderingPolicy.java | 119 ++++++++++++++++++ .../scheduler/policy/FifoComparator.java | 37 ++++++ .../scheduler/policy/FifoOrderingPolicy.java | 54 ++++++++ .../scheduler/policy/OrderingPolicy.java | 109 ++++++++++++++++ .../scheduler/policy/SchedulableEntity.java | 51 ++++++++ .../policy/MockSchedulableEntity.java | 78 ++++++++++++ .../policy/TestFifoOrderingPolicy.java | 83 ++++++++++++ 10 files changed, 592 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b5e850e4637..bcbf445787f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -27,6 +27,9 @@ Release 2.8.0 - UNRELEASED YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. (Wangda Tan via jianhe) + YARN-3318. Create Initial OrderingPolicy Framework and FifoOrderingPolicy. + (Craig Welch via wangda) + IMPROVEMENTS YARN-1880. Cleanup TestApplicationClientProtocolOnHA diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 375d19c1c2d..4b01a4d5448 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -140,6 +140,10 @@ + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index 5169b78dd58..2f7e19dd2cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -57,7 +57,10 @@ public class ResourceUsage { // Usage enum here to make implement cleaner private enum ResourceType { - USED(0), PENDING(1), AMUSED(2), RESERVED(3); + //CACHED_USED and CACHED_PENDING may be read by anyone, but must only + //be written by ordering policies + USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), + CACHED_PENDING(5); private int idx; @@ -102,6 +105,14 @@ public class ResourceUsage { public Resource getUsed(String label) { return _get(label, ResourceType.USED); } + + public Resource getCachedUsed(String label) { + return _get(label, ResourceType.CACHED_USED); + } + + public Resource getCachedPending(String label) { + return _get(label, ResourceType.CACHED_PENDING); + } public void incUsed(String label, Resource res) { _inc(label, ResourceType.USED, res); @@ -137,6 +148,14 @@ public class ResourceUsage { public void setUsed(String label, Resource res) { _set(label, ResourceType.USED, res); } + + public void setCachedUsed(String label, Resource res) { + _set(label, ResourceType.CACHED_USED, res); + } + + public void setCachedPending(String label, Resource res) { + _set(label, ResourceType.CACHED_PENDING, res); + } /* * Pending @@ -266,6 +285,28 @@ public class ResourceUsage { readLock.unlock(); } } + + private Resource _getAll(ResourceType type) { + try { + readLock.lock(); + Resource allOfType = Resources.createResource(0); + for (Map.Entry usageEntry : usages.entrySet()) { + //all usages types are initialized + Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]); + } + return allOfType; + } finally { + readLock.unlock(); + } + } + + public Resource getAllPending() { + return _getAll(ResourceType.PENDING); + } + + public Resource getAllUsed() { + return _getAll(ResourceType.USED); + } private UsageByLabel getAndAddIfMissing(String label) { if (label == null) { @@ -309,6 +350,18 @@ public class ResourceUsage { writeLock.unlock(); } } + + public Resource getCachedDemand(String label) { + try { + readLock.lock(); + Resource demand = Resources.createResource(0); + Resources.addTo(demand, getCachedUsed(label)); + Resources.addTo(demand, getCachedPending(label)); + return demand; + } finally { + readLock.unlock(); + } + } @Override public String toString() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java new file mode 100644 index 00000000000..5b32b3dee8d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import com.google.common.annotations.VisibleForTesting; + + +/** + * An OrderingPolicy which can serve as a baseclass for policies which can be + * expressed in terms of comparators + */ +public abstract class AbstractComparatorOrderingPolicy implements OrderingPolicy { + + private static final Log LOG = LogFactory.getLog(OrderingPolicy.class); + + protected TreeSet schedulableEntities; + protected Comparator comparator; + + public AbstractComparatorOrderingPolicy() { } + + @Override + public Collection getSchedulableEntities() { + return schedulableEntities; + } + + @Override + public Iterator getAssignmentIterator() { + return schedulableEntities.iterator(); + } + + @Override + public Iterator getPreemptionIterator() { + return schedulableEntities.descendingIterator(); + } + + public static void updateSchedulingResourceUsage(ResourceUsage ru) { + ru.setCachedUsed(CommonNodeLabelsManager.ANY, ru.getAllUsed()); + ru.setCachedPending(CommonNodeLabelsManager.ANY, ru.getAllPending()); + } + + protected void reorderSchedulableEntity(S schedulableEntity) { + //remove, update comparable data, and reinsert to update position in order + schedulableEntities.remove(schedulableEntity); + updateSchedulingResourceUsage( + schedulableEntity.getSchedulingResourceUsage()); + schedulableEntities.add(schedulableEntity); + } + + public void setComparator(Comparator comparator) { + this.comparator = comparator; + TreeSet schedulableEntities = new TreeSet(comparator); + if (this.schedulableEntities != null) { + schedulableEntities.addAll(this.schedulableEntities); + } + this.schedulableEntities = schedulableEntities; + } + + @VisibleForTesting + public Comparator getComparator() { + return comparator; + } + + @Override + public void addSchedulableEntity(S s) { + schedulableEntities.add(s); + } + + @Override + public boolean removeSchedulableEntity(S s) { + return schedulableEntities.remove(s); + } + + @Override + public void addAllSchedulableEntities(Collection sc) { + schedulableEntities.addAll(sc); + } + + @Override + public int getNumSchedulableEntities() { + return schedulableEntities.size(); + } + + @Override + public abstract void configure(String conf); + + @Override + public abstract void containerAllocated(S schedulableEntity, + RMContainer r); + + @Override + public abstract void containerReleased(S schedulableEntity, + RMContainer r); + + @Override + public abstract String getStatusMessage(); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java new file mode 100644 index 00000000000..b92b2644c28 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; + +/** + * A Comparator which orders SchedulableEntities by input order + */ +public class FifoComparator + implements Comparator { + + @Override + public int compare(SchedulableEntity r1, SchedulableEntity r2) { + int res = r1.compareInputOrderTo(r2); + return res; + } +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java new file mode 100644 index 00000000000..8a2f5ba11ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; + +/** + * An OrderingPolicy which orders SchedulableEntities by input order + */ +public class FifoOrderingPolicy extends AbstractComparatorOrderingPolicy { + + public FifoOrderingPolicy() { + setComparator(new FifoComparator()); + } + + @Override + public void configure(String conf) { + + } + + @Override + public void containerAllocated(S schedulableEntity, + RMContainer r) { + } + + @Override + public void containerReleased(S schedulableEntity, + RMContainer r) { + } + + @Override + public String getStatusMessage() { + return "FifoOrderingPolicy"; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java new file mode 100644 index 00000000000..f907cea2001 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; + + +/** + * OrderingPolicy is used by the scheduler to order SchedulableEntities for + * container assignment and preemption + */ +public interface OrderingPolicy { + /* + * Note: OrderingPolicy depends upon external + * synchronization of all use of the SchedulableEntity Collection and + * Iterators for correctness and to avoid concurrent modification issues + */ + + /** + * Get the collection of SchedulableEntities which are managed by this + * OrderingPolicy - should include processes returned by the Assignment and + * Preemption iterator with no guarantees regarding order + */ + public Collection getSchedulableEntities(); + + /** + * Return an iterator over the collection of SchedulableEntities which orders + * them for container assignment + */ + public Iterator getAssignmentIterator(); + + /** + * Return an iterator over the collection of SchedulableEntities which orders + * them for preemption + */ + public Iterator getPreemptionIterator(); + + /** + * Add a SchedulableEntity to be managed for allocation and preemption + * ordering + */ + public void addSchedulableEntity(S s); + + /** + * Remove a SchedulableEntity from management for allocation and preemption + * ordering + */ + public boolean removeSchedulableEntity(S s); + + /** + * Add a collection of SchedulableEntities to be managed for allocation + * and preemption ordering + */ + public void addAllSchedulableEntities(Collection sc); + + /** + * Get the number of SchedulableEntities managed for allocation and + * preemption ordering + */ + public int getNumSchedulableEntities(); + + /** + * Provides configuration information for the policy from the scheduler + * configuration + */ + public void configure(String conf); + + /** + * The passed SchedulableEntity has been allocated the passed Container, + * take appropriate action (depending on comparator, a reordering of the + * SchedulableEntity may be required) + */ + public void containerAllocated(S schedulableEntity, + RMContainer r); + + /** + * The passed SchedulableEntity has released the passed Container, + * take appropriate action (depending on comparator, a reordering of the + * SchedulableEntity may be required) + */ + public void containerReleased(S schedulableEntity, + RMContainer r); + + /** + * Display information regarding configuration & status + */ + public String getStatusMessage(); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java new file mode 100644 index 00000000000..9b9d73d920c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; + + +/** + * A SchedulableEntity is a process to be scheduled, + * for example, an application / application attempt + */ +public interface SchedulableEntity { + + /** + * Id - each entity must have a unique id + */ + public String getId(); + + /** + * Compare the passed SchedulableEntity to this one for input order. + * Input order is implementation defined and should reflect the + * correct ordering for first-in first-out processing + */ + public int compareInputOrderTo(SchedulableEntity other); + + /** + * View of Resources wanted and consumed by the entity + */ + public ResourceUsage getSchedulingResourceUsage(); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java new file mode 100644 index 00000000000..fe8c455cdf8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + + +public class MockSchedulableEntity implements SchedulableEntity { + + private String id; + private long serial = 0; + + public MockSchedulableEntity() { } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setSerial(long serial) { + this.serial = serial; + } + + public long getSerial() { + return serial; + } + + public void setUsed(Resource value) { + schedulingResourceUsage.setUsed(CommonNodeLabelsManager.ANY, value); + } + + public void setPending(Resource value) { + schedulingResourceUsage.setPending(CommonNodeLabelsManager.ANY, value); + } + + private ResourceUsage schedulingResourceUsage = new ResourceUsage(); + + @Override + public ResourceUsage getSchedulingResourceUsage() { + return schedulingResourceUsage; + } + + @Override + public int compareInputOrderTo(SchedulableEntity other) { + if (other instanceof MockSchedulableEntity) { + MockSchedulableEntity r2 = (MockSchedulableEntity) other; + int res = (int) Math.signum(getSerial() - r2.getSerial()); + return res; + } + return 1;//let other types go before this, if any + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java new file mode 100644 index 00000000000..e05ef261024 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; + +import java.util.*; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; + +public class TestFifoOrderingPolicy { + + @Test + public void testFifoOrderingPolicy() { + FifoOrderingPolicy policy = + new FifoOrderingPolicy(); + MockSchedulableEntity r1 = new MockSchedulableEntity(); + MockSchedulableEntity r2 = new MockSchedulableEntity(); + + Assert.assertEquals(policy.getComparator().compare(r1, r2), 0); + + r1.setSerial(1); + Assert.assertEquals(policy.getComparator().compare(r1, r2), 1); + + r2.setSerial(2); + Assert.assertEquals(policy.getComparator().compare(r1, r2), -1); + } + + @Test + public void testIterators() { + OrderingPolicy schedOrder = + new FifoOrderingPolicy(); + + MockSchedulableEntity msp1 = new MockSchedulableEntity(); + MockSchedulableEntity msp2 = new MockSchedulableEntity(); + MockSchedulableEntity msp3 = new MockSchedulableEntity(); + + msp1.setSerial(3); + msp2.setSerial(2); + msp3.setSerial(1); + + schedOrder.addSchedulableEntity(msp1); + schedOrder.addSchedulableEntity(msp2); + schedOrder.addSchedulableEntity(msp3); + + //Assignment, oldest to youngest + checkSerials(schedOrder.getAssignmentIterator(), new long[]{1, 2, 3}); + + //Preemption, youngest to oldest + checkSerials(schedOrder.getPreemptionIterator(), new long[]{3, 2, 1}); + } + + public void checkSerials(Iterator si, + long[] serials) { + for (int i = 0;i < serials.length;i++) { + Assert.assertEquals(si.next().getSerial(), + serials[i]); + } + } + +}