From 5f5b031d1f20cb7f621db41979e963eaa42cf52f Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Mon, 27 Feb 2017 20:38:29 +0530 Subject: [PATCH] YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R. --- .../AbstractReservationSystem.java | 8 ++++++-- .../planning/AlignedPlannerWithGreedy.java | 16 +++++++++------- .../planning/GreedyReservationAgent.java | 19 ++++++------------- .../planning/PlanningAlgorithm.java | 4 ++++ .../planning/ReservationAgent.java | 8 ++++++++ .../planning/TryManyReservationAgents.java | 5 ++++- .../planning/TestAlignedPlanner.java | 3 +++ .../planning/TestGreedyReservationAgent.java | 4 ++-- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 8769ca139c2..5ef4912e670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService try { Class agentClazz = conf.getClassByName(agentClassName); if (ReservationAgent.class.isAssignableFrom(agentClazz)) { - return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf); + ReservationAgent resevertionAgent = + (ReservationAgent) agentClazz.newInstance(); + resevertionAgent.init(conf); + return resevertionAgent; } else { throw new YarnRuntimeException("Class: " + agentClassName + " not instance of " + ReservationAgent.class.getCanonicalName()); } - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | InstantiationException + | IllegalAccessException e) { throw new YarnRuntimeException("Could not instantiate Agent: " + agentClassName + " for queue: " + queueName, e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java index b23cf1e933e..00c233389d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.LinkedList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory; public class AlignedPlannerWithGreedy implements ReservationAgent { // Default smoothness factor - private static final int DEFAULT_SMOOTHNESS_FACTOR = 10; + public static final int DEFAULT_SMOOTHNESS_FACTOR = 10; + public static final String SMOOTHNESS_FACTOR = + "yarn.resourcemanager.reservation-system.smoothness-factor"; // Log private static final Logger LOG = LoggerFactory .getLogger(AlignedPlannerWithGreedy.class); // Smoothness factor - private final ReservationAgent planner; + private ReservationAgent planner; // Constructor public AlignedPlannerWithGreedy() { - this(DEFAULT_SMOOTHNESS_FACTOR); } - // Constructor - public AlignedPlannerWithGreedy(int smoothnessFactor) { + @Override + public void init(Configuration conf) { + int smoothnessFactor = + conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR); // List of algorithms List listAlg = new LinkedList(); @@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent { // 1. Attempt to execute algAligned // 2. If failed, fall back to algGreedy planner = new TryManyReservationAgents(listAlg); - } @Override @@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent { return planner.deleteReservation(reservationId, user, plan); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java index 915a834179f..1559b973f51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java @@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent { .getLogger(GreedyReservationAgent.class); // Greedy planner - private final ReservationAgent planner; - + private ReservationAgent planner; public final static String GREEDY_FAVOR_EARLY_ALLOCATION = "yarn.resourcemanager.reservation-system.favor-early-allocation"; - public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true; - - private final boolean allocateLeft; + private boolean allocateLeft; public GreedyReservationAgent() { - this(new Configuration()); } - public GreedyReservationAgent(Configuration yarnConfiguration) { - - allocateLeft = - yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION, - DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION); - + @Override + public void init(Configuration conf) { + allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION, + DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION); if (allocateLeft) { LOG.info("Initializing the GreedyReservationAgent to favor \"early\"" + " (left) allocations (controlled by parameter: " @@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent { planner = new IterativePlanner(new StageEarliestStartByJobArrival(), new StageAllocatorGreedyRLE(allocateLeft), allocateLeft); - } public boolean isAllocateLeft(){ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java index e1b508d22b9..199bfa54325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; @@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent { } + @Override + public void init(Configuration conf) { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java index bdea2f47d57..52e7055e0fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java @@ -17,6 +17,7 @@ *******************************************************************************/ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -70,4 +71,11 @@ public interface ReservationAgent { public boolean deleteReservation(ReservationId reservationId, String user, Plan plan) throws PlanningException; + /** + * Init configuration. + * + * @param conf Configuration + */ + void init(Configuration conf); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java index 1d37ce596be..ab2e1e0f916 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.LinkedList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent { return plan.deleteReservation(reservationId); } - + @Override + public void init(Configuration conf) { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java index 4b01eb90fd4..2645366b4dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java @@ -726,8 +726,11 @@ public class TestAlignedPlanner { QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); + conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR, + AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR); // Set planning agent agent = new AlignedPlannerWithGreedy(); + agent.init(conf); // Create Plan plan = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java index ec11ffe868e..6d1cfa87adf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java @@ -110,8 +110,8 @@ public class TestGreedyReservationAgent { // setting conf to conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION, allocateLeft); - - agent = new GreedyReservationAgent(conf); + agent = new GreedyReservationAgent(); + agent.init(conf); QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext();