YARN-8697. LocalityMulticastAMRMProxyPolicy should fallback to random sub-cluster when cannot resolve resource. Contributed by Botong Huang.
This commit is contained in:
parent
3e18b957eb
commit
7ed458b255
|
@ -21,8 +21,11 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Random;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
@ -123,6 +126,8 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
public static final Logger LOG =
|
public static final Logger LOG =
|
||||||
LoggerFactory.getLogger(LocalityMulticastAMRMProxyPolicy.class);
|
LoggerFactory.getLogger(LocalityMulticastAMRMProxyPolicy.class);
|
||||||
|
|
||||||
|
private static Random rand = new Random();
|
||||||
|
|
||||||
private Map<SubClusterId, Float> weights;
|
private Map<SubClusterId, Float> weights;
|
||||||
private SubClusterResolver resolver;
|
private SubClusterResolver resolver;
|
||||||
|
|
||||||
|
@ -275,26 +280,18 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle node/rack requests that the SubClusterResolver cannot map to
|
// Handle node/rack requests that the SubClusterResolver cannot map to
|
||||||
// any cluster. Defaulting to home subcluster.
|
// any cluster. Pick a random sub-cluster from active and enabled ones.
|
||||||
|
targetId = getSubClusterForUnResolvedRequest(bookkeeper,
|
||||||
|
rr.getAllocationRequestId());
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("ERROR resolving sub-cluster for resourceName: "
|
LOG.debug("ERROR resolving sub-cluster for resourceName: "
|
||||||
+ rr.getResourceName() + " we are falling back to homeSubCluster:"
|
+ rr.getResourceName() + ", picked a random subcluster to forward:"
|
||||||
+ homeSubcluster);
|
+ targetId);
|
||||||
}
|
}
|
||||||
|
if (targetIds != null && targetIds.size() > 0) {
|
||||||
// If home-subcluster is not active, ignore node/rack request
|
bookkeeper.addRackRR(targetId, rr);
|
||||||
if (bookkeeper.isActiveAndEnabled(homeSubcluster)) {
|
|
||||||
if (targetIds != null && targetIds.size() > 0) {
|
|
||||||
bookkeeper.addRackRR(homeSubcluster, rr);
|
|
||||||
} else {
|
|
||||||
bookkeeper.addLocalizedNodeRR(homeSubcluster, rr);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
bookkeeper.addLocalizedNodeRR(targetId, rr);
|
||||||
LOG.debug("The homeSubCluster (" + homeSubcluster + ") we are "
|
|
||||||
+ "defaulting to is not active, the ResourceRequest "
|
|
||||||
+ "will be ignored.");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -313,6 +310,14 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
return bookkeeper.getAnswer();
|
return bookkeeper.getAnswer();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For unit test to override.
|
||||||
|
*/
|
||||||
|
protected SubClusterId getSubClusterForUnResolvedRequest(
|
||||||
|
AllocationBookkeeper bookKeeper, long allocationId) {
|
||||||
|
return bookKeeper.getSubClusterForUnResolvedRequest(allocationId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* It splits a list of non-localized resource requests among sub-clusters.
|
* It splits a list of non-localized resource requests among sub-clusters.
|
||||||
*/
|
*/
|
||||||
|
@ -512,10 +517,11 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
* This helper class is used to book-keep the requests made to each
|
* This helper class is used to book-keep the requests made to each
|
||||||
* subcluster, and maintain useful statistics to split ANY requests.
|
* subcluster, and maintain useful statistics to split ANY requests.
|
||||||
*/
|
*/
|
||||||
private final class AllocationBookkeeper {
|
protected final class AllocationBookkeeper {
|
||||||
|
|
||||||
// the answer being accumulated
|
// the answer being accumulated
|
||||||
private Map<SubClusterId, List<ResourceRequest>> answer = new TreeMap<>();
|
private Map<SubClusterId, List<ResourceRequest>> answer = new TreeMap<>();
|
||||||
|
private Map<SubClusterId, Set<Long>> maskForRackDeletion = new HashMap<>();
|
||||||
|
|
||||||
// stores how many containers we have allocated in each RM for localized
|
// stores how many containers we have allocated in each RM for localized
|
||||||
// asks, used to correctly "spread" the corresponding ANY
|
// asks, used to correctly "spread" the corresponding ANY
|
||||||
|
@ -523,6 +529,10 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
new HashMap<>();
|
new HashMap<>();
|
||||||
private Map<Long, AtomicLong> totNumLocalizedContainers = new HashMap<>();
|
private Map<Long, AtomicLong> totNumLocalizedContainers = new HashMap<>();
|
||||||
|
|
||||||
|
// Store the randomly selected subClusterId for unresolved resource requests
|
||||||
|
// keyed by requestId
|
||||||
|
private Map<Long, SubClusterId> unResolvedRequestLocation = new HashMap<>();
|
||||||
|
|
||||||
private Set<SubClusterId> activeAndEnabledSC = new HashSet<>();
|
private Set<SubClusterId> activeAndEnabledSC = new HashSet<>();
|
||||||
private float totHeadroomMemory = 0;
|
private float totHeadroomMemory = 0;
|
||||||
private int totHeadRoomEnabledRMs = 0;
|
private int totHeadRoomEnabledRMs = 0;
|
||||||
|
@ -538,6 +548,7 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
|
|
||||||
// reset data structures
|
// reset data structures
|
||||||
answer.clear();
|
answer.clear();
|
||||||
|
maskForRackDeletion.clear();
|
||||||
countContainersPerRM.clear();
|
countContainersPerRM.clear();
|
||||||
totNumLocalizedContainers.clear();
|
totNumLocalizedContainers.clear();
|
||||||
activeAndEnabledSC.clear();
|
activeAndEnabledSC.clear();
|
||||||
|
@ -628,16 +639,16 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
.addAndGet(rr.getNumContainers());
|
.addAndGet(rr.getNumContainers());
|
||||||
}
|
}
|
||||||
|
|
||||||
internalAddToAnswer(targetId, rr);
|
internalAddToAnswer(targetId, rr, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add a rack-local request to the final asnwer.
|
* Add a rack-local request to the final asnwer.
|
||||||
*/
|
*/
|
||||||
public void addRackRR(SubClusterId targetId, ResourceRequest rr) {
|
private void addRackRR(SubClusterId targetId, ResourceRequest rr) {
|
||||||
Preconditions
|
Preconditions
|
||||||
.checkArgument(!ResourceRequest.isAnyLocation(rr.getResourceName()));
|
.checkArgument(!ResourceRequest.isAnyLocation(rr.getResourceName()));
|
||||||
internalAddToAnswer(targetId, rr);
|
internalAddToAnswer(targetId, rr, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -646,17 +657,45 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
private void addAnyRR(SubClusterId targetId, ResourceRequest rr) {
|
private void addAnyRR(SubClusterId targetId, ResourceRequest rr) {
|
||||||
Preconditions
|
Preconditions
|
||||||
.checkArgument(ResourceRequest.isAnyLocation(rr.getResourceName()));
|
.checkArgument(ResourceRequest.isAnyLocation(rr.getResourceName()));
|
||||||
internalAddToAnswer(targetId, rr);
|
internalAddToAnswer(targetId, rr, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void internalAddToAnswer(SubClusterId targetId,
|
private void internalAddToAnswer(SubClusterId targetId,
|
||||||
ResourceRequest partialRR) {
|
ResourceRequest partialRR, boolean isRack) {
|
||||||
|
if (!isRack) {
|
||||||
|
if (!maskForRackDeletion.containsKey(targetId)) {
|
||||||
|
maskForRackDeletion.put(targetId, new HashSet<Long>());
|
||||||
|
}
|
||||||
|
maskForRackDeletion.get(targetId)
|
||||||
|
.add(partialRR.getAllocationRequestId());
|
||||||
|
}
|
||||||
if (!answer.containsKey(targetId)) {
|
if (!answer.containsKey(targetId)) {
|
||||||
answer.put(targetId, new ArrayList<ResourceRequest>());
|
answer.put(targetId, new ArrayList<ResourceRequest>());
|
||||||
}
|
}
|
||||||
answer.get(targetId).add(partialRR);
|
answer.get(targetId).add(partialRR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For requests whose location cannot be resolved, choose an active and
|
||||||
|
* enabled sub-cluster to forward this requestId to.
|
||||||
|
*/
|
||||||
|
private SubClusterId getSubClusterForUnResolvedRequest(long allocationId) {
|
||||||
|
if (unResolvedRequestLocation.containsKey(allocationId)) {
|
||||||
|
return unResolvedRequestLocation.get(allocationId);
|
||||||
|
}
|
||||||
|
int id = rand.nextInt(activeAndEnabledSC.size());
|
||||||
|
for (SubClusterId subclusterId : activeAndEnabledSC) {
|
||||||
|
if (id == 0) {
|
||||||
|
unResolvedRequestLocation.put(allocationId, subclusterId);
|
||||||
|
return subclusterId;
|
||||||
|
}
|
||||||
|
id--;
|
||||||
|
}
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Should not be here. activeAndEnabledSC size = "
|
||||||
|
+ activeAndEnabledSC.size() + " id = " + id);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return all known subclusters associated with an allocation id.
|
* Return all known subclusters associated with an allocation id.
|
||||||
*
|
*
|
||||||
|
@ -678,6 +717,28 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
* @return the answer
|
* @return the answer
|
||||||
*/
|
*/
|
||||||
private Map<SubClusterId, List<ResourceRequest>> getAnswer() {
|
private Map<SubClusterId, List<ResourceRequest>> getAnswer() {
|
||||||
|
Iterator<Entry<SubClusterId, List<ResourceRequest>>> answerIter =
|
||||||
|
answer.entrySet().iterator();
|
||||||
|
// Remove redundant rack RR before returning the answer
|
||||||
|
while (answerIter.hasNext()) {
|
||||||
|
Entry<SubClusterId, List<ResourceRequest>> entry = answerIter.next();
|
||||||
|
SubClusterId scId = entry.getKey();
|
||||||
|
Set<Long> mask = maskForRackDeletion.get(scId);
|
||||||
|
if (mask != null) {
|
||||||
|
Iterator<ResourceRequest> rrIter = entry.getValue().iterator();
|
||||||
|
while (rrIter.hasNext()) {
|
||||||
|
ResourceRequest rr = rrIter.next();
|
||||||
|
if (!mask.contains(rr.getAllocationRequestId())) {
|
||||||
|
rrIter.remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (mask == null || entry.getValue().size() == 0) {
|
||||||
|
answerIter.remove();
|
||||||
|
LOG.info("removing {} from output because it has only rack RR",
|
||||||
|
scId);
|
||||||
|
}
|
||||||
|
}
|
||||||
return answer;
|
return answer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -69,12 +69,12 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
setPolicy(new LocalityMulticastAMRMProxyPolicy());
|
setPolicy(new TestableLocalityMulticastAMRMProxyPolicy());
|
||||||
setPolicyInfo(new WeightedPolicyInfo());
|
setPolicyInfo(new WeightedPolicyInfo());
|
||||||
Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
|
Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
|
||||||
Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
|
Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
|
||||||
|
|
||||||
// simulate 20 subclusters with a 5% chance of being inactive
|
// Six sub-clusters with one inactive and one disabled
|
||||||
for (int i = 0; i < 6; i++) {
|
for (int i = 0; i < 6; i++) {
|
||||||
SubClusterIdInfo sc = new SubClusterIdInfo("subcluster" + i);
|
SubClusterIdInfo sc = new SubClusterIdInfo("subcluster" + i);
|
||||||
// sub-cluster 3 is not active
|
// sub-cluster 3 is not active
|
||||||
|
@ -207,6 +207,7 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
getPolicyInfo().setHeadroomAlpha(1.0f);
|
getPolicyInfo().setHeadroomAlpha(1.0f);
|
||||||
|
|
||||||
initializePolicy();
|
initializePolicy();
|
||||||
|
addHomeSubClusterAsActive();
|
||||||
|
|
||||||
int numRR = 1000;
|
int numRR = 1000;
|
||||||
List<ResourceRequest> resourceRequests = createLargeRandomList(numRR);
|
List<ResourceRequest> resourceRequests = createLargeRandomList(numRR);
|
||||||
|
@ -324,14 +325,11 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
null, Collections.<NMToken> emptyList());
|
null, Collections.<NMToken> emptyList());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
/**
|
||||||
public void testSplitAllocateRequest() throws Exception {
|
* modify default initialization to include a "homesubcluster" which we will
|
||||||
|
* use as the default for when nodes or racks are unknown.
|
||||||
// Test a complex List<ResourceRequest> is split correctly
|
*/
|
||||||
initializePolicy();
|
private void addHomeSubClusterAsActive() {
|
||||||
|
|
||||||
// modify default initialization to include a "homesubcluster"
|
|
||||||
// which we will use as the default for when nodes or racks are unknown
|
|
||||||
SubClusterInfo sci = mock(SubClusterInfo.class);
|
SubClusterInfo sci = mock(SubClusterInfo.class);
|
||||||
when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
|
when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
|
||||||
when(sci.getSubClusterId()).thenReturn(getHomeSubCluster());
|
when(sci.getSubClusterId()).thenReturn(getHomeSubCluster());
|
||||||
|
@ -340,6 +338,14 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
|
|
||||||
getPolicyInfo().getRouterPolicyWeights().put(sc, 0.1f);
|
getPolicyInfo().getRouterPolicyWeights().put(sc, 0.1f);
|
||||||
getPolicyInfo().getAMRMPolicyWeights().put(sc, 0.1f);
|
getPolicyInfo().getAMRMPolicyWeights().put(sc, 0.1f);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSplitAllocateRequest() throws Exception {
|
||||||
|
|
||||||
|
// Test a complex List<ResourceRequest> is split correctly
|
||||||
|
initializePolicy();
|
||||||
|
addHomeSubClusterAsActive();
|
||||||
|
|
||||||
FederationPoliciesTestUtil.initializePolicyContext(
|
FederationPoliciesTestUtil.initializePolicyContext(
|
||||||
getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
|
getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
|
||||||
|
@ -502,7 +508,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
|
|
||||||
// Test target Ids
|
// Test target Ids
|
||||||
for (SubClusterId targetId : split.keySet()) {
|
for (SubClusterId targetId : split.keySet()) {
|
||||||
Assert.assertTrue("Target subclusters should be in the active set",
|
Assert.assertTrue(
|
||||||
|
"Target subcluster " + targetId + " should be in the active set",
|
||||||
getActiveSubclusters().containsKey(targetId));
|
getActiveSubclusters().containsKey(targetId));
|
||||||
Assert.assertTrue(
|
Assert.assertTrue(
|
||||||
"Target subclusters (" + targetId + ") should have weight >0 in "
|
"Target subclusters (" + targetId + ") should have weight >0 in "
|
||||||
|
@ -787,4 +794,28 @@ public class TestLocalityMulticastAMRMProxyPolicy
|
||||||
checkTotalContainerAllocation(response, 100);
|
checkTotalContainerAllocation(response, 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A testable version of LocalityMulticastAMRMProxyPolicy that
|
||||||
|
* deterministically falls back to home sub-cluster for unresolved requests.
|
||||||
|
*/
|
||||||
|
private class TestableLocalityMulticastAMRMProxyPolicy
|
||||||
|
extends LocalityMulticastAMRMProxyPolicy {
|
||||||
|
@Override
|
||||||
|
protected SubClusterId getSubClusterForUnResolvedRequest(
|
||||||
|
AllocationBookkeeper bookkeeper, long allocationId) {
|
||||||
|
SubClusterId originalResult =
|
||||||
|
super.getSubClusterForUnResolvedRequest(bookkeeper, allocationId);
|
||||||
|
Map<SubClusterId, SubClusterInfo> activeClusters = null;
|
||||||
|
try {
|
||||||
|
activeClusters = getActiveSubclusters();
|
||||||
|
} catch (YarnException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
// The randomly selected sub-cluster should at least be active
|
||||||
|
Assert.assertTrue(activeClusters.containsKey(originalResult));
|
||||||
|
|
||||||
|
// Alwasy use home sub-cluster so that unit test is deterministic
|
||||||
|
return getHomeSubCluster();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue