YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.
(cherry picked from commit 5f5b031d1f)
This commit is contained in:
Naganarasimha
parent
68b08e96a0
commit
1eec911cd9
|
|
@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService
|
|||
try {
|
||||
Class<?> agentClazz = conf.getClassByName(agentClassName);
|
||||
if (ReservationAgent.class.isAssignableFrom(agentClazz)) {
|
||||
return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf);
|
||||
ReservationAgent resevertionAgent =
|
||||
(ReservationAgent) agentClazz.newInstance();
|
||||
resevertionAgent.init(conf);
|
||||
return resevertionAgent;
|
||||
} else {
|
||||
throw new YarnRuntimeException("Class: " + agentClassName
|
||||
+ " not instance of " + ReservationAgent.class.getCanonicalName());
|
||||
}
|
||||
} catch (ClassNotFoundException e) {
|
||||
} catch (ClassNotFoundException | InstantiationException
|
||||
| IllegalAccessException e) {
|
||||
throw new YarnRuntimeException("Could not instantiate Agent: "
|
||||
+ agentClassName + " for queue: " + queueName, e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
|
|||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
|
||||
|
|
@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory;
|
|||
public class AlignedPlannerWithGreedy implements ReservationAgent {
|
||||
|
||||
// Default smoothness factor
|
||||
private static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
|
||||
public static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
|
||||
public static final String SMOOTHNESS_FACTOR =
|
||||
"yarn.resourcemanager.reservation-system.smoothness-factor";
|
||||
|
||||
// Log
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(AlignedPlannerWithGreedy.class);
|
||||
|
||||
// Smoothness factor
|
||||
private final ReservationAgent planner;
|
||||
private ReservationAgent planner;
|
||||
|
||||
// Constructor
|
||||
public AlignedPlannerWithGreedy() {
|
||||
this(DEFAULT_SMOOTHNESS_FACTOR);
|
||||
}
|
||||
|
||||
// Constructor
|
||||
public AlignedPlannerWithGreedy(int smoothnessFactor) {
|
||||
@Override
|
||||
public void init(Configuration conf) {
|
||||
int smoothnessFactor =
|
||||
conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR);
|
||||
|
||||
// List of algorithms
|
||||
List<ReservationAgent> listAlg = new LinkedList<ReservationAgent>();
|
||||
|
|
@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
|
|||
// 1. Attempt to execute algAligned
|
||||
// 2. If failed, fall back to algGreedy
|
||||
planner = new TryManyReservationAgents(listAlg);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
|
|||
return planner.deleteReservation(reservationId, user, plan);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent {
|
|||
.getLogger(GreedyReservationAgent.class);
|
||||
|
||||
// Greedy planner
|
||||
private final ReservationAgent planner;
|
||||
|
||||
private ReservationAgent planner;
|
||||
public final static String GREEDY_FAVOR_EARLY_ALLOCATION =
|
||||
"yarn.resourcemanager.reservation-system.favor-early-allocation";
|
||||
|
||||
public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true;
|
||||
|
||||
private final boolean allocateLeft;
|
||||
private boolean allocateLeft;
|
||||
|
||||
public GreedyReservationAgent() {
|
||||
this(new Configuration());
|
||||
}
|
||||
|
||||
public GreedyReservationAgent(Configuration yarnConfiguration) {
|
||||
|
||||
allocateLeft =
|
||||
yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
|
||||
DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
|
||||
|
||||
@Override
|
||||
public void init(Configuration conf) {
|
||||
allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
|
||||
DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
|
||||
if (allocateLeft) {
|
||||
LOG.info("Initializing the GreedyReservationAgent to favor \"early\""
|
||||
+ " (left) allocations (controlled by parameter: "
|
||||
|
|
@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent {
|
|||
planner =
|
||||
new IterativePlanner(new StageEarliestStartByJobArrival(),
|
||||
new StageAllocatorGreedyRLE(allocateLeft), allocateLeft);
|
||||
|
||||
}
|
||||
|
||||
public boolean isAllocateLeft(){
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
|
|
@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
|
|||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Configuration conf) {
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
*******************************************************************************/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
|
||||
|
|
@ -70,4 +71,11 @@ public interface ReservationAgent {
|
|||
public boolean deleteReservation(ReservationId reservationId, String user,
|
||||
Plan plan) throws PlanningException;
|
||||
|
||||
/**
|
||||
* Init configuration.
|
||||
*
|
||||
* @param conf Configuration
|
||||
*/
|
||||
void init(Configuration conf);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
|
|||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
|
||||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
|
||||
|
|
@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent {
|
|||
return plan.deleteReservation(reservationId);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Configuration conf) {
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -726,8 +726,11 @@ public class TestAlignedPlanner {
|
|||
QueueMetrics queueMetrics = mock(QueueMetrics.class);
|
||||
RMContext context = ReservationSystemTestUtil.createMockRMContext();
|
||||
|
||||
conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR,
|
||||
AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR);
|
||||
// Set planning agent
|
||||
agent = new AlignedPlannerWithGreedy();
|
||||
agent.init(conf);
|
||||
|
||||
// Create Plan
|
||||
plan =
|
||||
|
|
|
|||
|
|
@ -110,8 +110,8 @@ public class TestGreedyReservationAgent {
|
|||
// setting conf to
|
||||
conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION,
|
||||
allocateLeft);
|
||||
|
||||
agent = new GreedyReservationAgent(conf);
|
||||
agent = new GreedyReservationAgent();
|
||||
agent.init(conf);
|
||||
|
||||
QueueMetrics queueMetrics = mock(QueueMetrics.class);
|
||||
RMContext context = ReservationSystemTestUtil.createMockRMContext();
|
||||
|
|
|
|||
Loading…
Reference in New Issue