From 0aad6f2dac96546adf28135fbdc30f1656b1f4ee Mon Sep 17 00:00:00 2001 From: Thomas White Date: Thu, 11 Oct 2012 10:30:53 +0000 Subject: [PATCH] Merge -r 1396971:1396972 from trunk to branch-2. Fixes: YARN-146. Add unit tests for computing fair share in the fair scheduler. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1396977 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FakeSchedulable.java | 115 +++++++++++ .../scheduler/fair/TestComputeFairShares.java | 184 ++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0f245607f4f..27ac35441cf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -41,6 +41,9 @@ Release 2.0.3-alpha - Unreleased YARN-94. Modify DistributedShell to point to main-class by default, clean up the help message, and hard-code the AM class. (Hitesh Shah via vinodkv) + YARN-146. Add unit tests for computing fair share in the fair scheduler. + (Sandy Ryza via tomwhite) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java new file mode 100644 index 00000000000..11f96f48c50 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.util.Records; + +/** + * Dummy implementation of Schedulable for unit testing. + */ +public class FakeSchedulable extends Schedulable { + private Resource demand; + private Resource usage; + private Resource minShare; + private double weight; + private Priority priority; + private long startTime; + + public FakeSchedulable() { + this(0, 0, 1, 0, 0, 0); + } + + public FakeSchedulable(int demand) { + this(demand, 0, 1, 0, 0, 0); + } + + public FakeSchedulable(int demand, int minShare) { + this(demand, minShare, 1, 0, 0, 0); + } + + public FakeSchedulable(int demand, int minShare, double weight) { + this(demand, minShare, weight, 0, 0, 0); + } + + public FakeSchedulable(int demand, int minShare, double weight, int fairShare, int usage, + long startTime) { + this(Resources.createResource(demand), Resources.createResource(minShare), weight, + Resources.createResource(fairShare), Resources.createResource(usage), startTime); + } + + public FakeSchedulable(Resource demand, Resource minShare, double weight, Resource fairShare, + Resource usage, long startTime) { + this.demand = demand; + this.minShare = minShare; + this.weight = weight; + setFairShare(fairShare); + this.usage = usage; + this.priority = Records.newRecord(Priority.class); + this.startTime = startTime; + } + + @Override + public Resource assignContainer(FSSchedulerNode node, boolean reserved) { + return null; + } + + @Override + public Resource getDemand() { + return demand; + } + + @Override + public String getName() { + return "FakeSchedulable" + this.hashCode(); + } + + @Override + public Priority getPriority() { + return priority; + } + + @Override + public Resource getResourceUsage() { + return usage; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public double getWeight() { + return weight; + } + + @Override + public Resource getMinShare() { + return minShare; + } + + @Override + public void redistributeShare() {} + + @Override + public void updateDemand() {} +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java new file mode 100644 index 00000000000..05054b332e8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.junit.Before; +import org.junit.Test; + +/** + * Exercise the computeFairShares method in SchedulingAlgorithms. + */ +public class TestComputeFairShares { + private List scheds; + + @Before + public void setUp() throws Exception { + scheds = new ArrayList(); + } + + /** + * Basic test - pools with different demands that are all higher than their + * fair share (of 10 slots) should each get their fair share. + */ + @Test + public void testEqualSharing() { + scheds.add(new FakeSchedulable(100)); + scheds.add(new FakeSchedulable(50)); + scheds.add(new FakeSchedulable(30)); + scheds.add(new FakeSchedulable(20)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + verifyShares(10, 10, 10, 10); + } + + /** + * In this test, pool 4 has a smaller demand than the 40 / 4 = 10 slots that + * it would be assigned with equal sharing. It should only get the 3 slots + * it demands. The other pools must then split the remaining 37 slots, but + * pool 3, with 11 slots demanded, is now below its share of 37/3 ~= 12.3, + * so it only gets 11 slots. Pools 1 and 2 split the rest and get 13 each. + */ + @Test + public void testLowDemands() { + scheds.add(new FakeSchedulable(100)); + scheds.add(new FakeSchedulable(50)); + scheds.add(new FakeSchedulable(11)); + scheds.add(new FakeSchedulable(3)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + verifyShares(13, 13, 11, 3); + } + + /** + * In this test, some pools have minimum shares set. Pool 1 has a min share + * of 20 so it gets 20 slots. Pool 2 also has a min share of 20, but its + * demand is only 10 so it can only get 10 slots. The remaining pools have + * 10 slots to split between them. Pool 4 gets 3 slots because its demand is + * only 3, and pool 3 gets the remaining 7 slots. Pool 4 also had a min share + * of 2 slots but this should not affect the outcome. + */ + @Test + public void testMinShares() { + scheds.add(new FakeSchedulable(100, 20)); + scheds.add(new FakeSchedulable(10, 20)); + scheds.add(new FakeSchedulable(10, 0)); + scheds.add(new FakeSchedulable(3, 2)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + verifyShares(20, 10, 7, 3); + } + + /** + * Basic test for weighted shares with no minimum shares and no low demands. + * Each pool should get slots in proportion to its weight. + */ + @Test + public void testWeightedSharing() { + scheds.add(new FakeSchedulable(100, 0, 2.0)); + scheds.add(new FakeSchedulable(50, 0, 1.0)); + scheds.add(new FakeSchedulable(30, 0, 1.0)); + scheds.add(new FakeSchedulable(20, 0, 0.5)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + verifyShares(20, 10, 10, 5); + } + + /** + * Weighted sharing test where pools 1 and 2 are now given lower demands than + * above. Pool 1 stops at 10 slots, leaving 35. If the remaining pools split + * this into a 1:1:0.5 ratio, they would get 14:14:7 slots respectively, but + * pool 2's demand is only 11, so it only gets 11. The remaining 2 pools split + * the 24 slots left into a 1:0.5 ratio, getting 16 and 8 slots respectively. + */ + @Test + public void testWeightedSharingWithLowDemands() { + scheds.add(new FakeSchedulable(10, 0, 2.0)); + scheds.add(new FakeSchedulable(11, 0, 1.0)); + scheds.add(new FakeSchedulable(30, 0, 1.0)); + scheds.add(new FakeSchedulable(20, 0, 0.5)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + verifyShares(10, 11, 16, 8); + } + + /** + * Weighted fair sharing test with min shares. As in the min share test above, + * pool 1 has a min share greater than its demand so it only gets its demand. + * Pool 3 has a min share of 15 even though its weight is very small, so it + * gets 15 slots. The remaining pools share the remaining 20 slots equally, + * getting 10 each. Pool 3's min share of 5 slots doesn't affect this. + */ + @Test + public void testWeightedSharingWithMinShares() { + scheds.add(new FakeSchedulable(10, 20, 2.0)); + scheds.add(new FakeSchedulable(11, 0, 1.0)); + scheds.add(new FakeSchedulable(30, 5, 1.0)); + scheds.add(new FakeSchedulable(20, 15, 0.5)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + verifyShares(10, 10, 10, 15); + } + + /** + * Test that shares are computed accurately even when the number of slots is + * very large. + */ + @Test + public void testLargeShares() { + int million = 1000 * 1000; + scheds.add(new FakeSchedulable(100 * million)); + scheds.add(new FakeSchedulable(50 * million)); + scheds.add(new FakeSchedulable(30 * million)); + scheds.add(new FakeSchedulable(20 * million)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40 * million)); + verifyShares(10 * million, 10 * million, 10 * million, 10 * million); + } + + /** + * Test that having a pool with 0 demand doesn't confuse the algorithm. + */ + @Test + public void testZeroDemand() { + scheds.add(new FakeSchedulable(100)); + scheds.add(new FakeSchedulable(50)); + scheds.add(new FakeSchedulable(30)); + scheds.add(new FakeSchedulable(0)); + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(30)); + verifyShares(10, 10, 10, 0); + } + + /** + * Test that being called on an empty list doesn't confuse the algorithm. + */ + @Test + public void testEmptyList() { + SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + verifyShares(); + } + + /** + * Check that a given list of shares have been assigned to this.scheds. + */ + private void verifyShares(double... shares) { + Assert.assertEquals(scheds.size(), shares.length); + for (int i = 0; i < shares.length; i++) { + Assert.assertEquals(shares[i], scheds.get(i).getFairShare().getMemory(), 0.01); + } + } +}