Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8517. getContainer and getContainers ResourceManager REST API methods are not documented (snemeth via rkanter)

This commit is contained in:
Robert Kanter 2018-07-27 14:35:03 -07:00
parent fecbac499e
commit 2cccf4061c
4 changed files with 430 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidResourceRequestException.java
View File

@ -30,19 +30,55 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
*
*/
public class InvalidResourceRequestException extends YarnException {
public static final String LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE =
"Invalid resource request! Cannot allocate containers as "
+ "requested resource is less than 0! "
+ "Requested resource type=[%s], " + "Requested resource=%s";
public static final String GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE =
"Invalid resource request! Cannot allocate containers as "
+ "requested resource is greater than " +
"maximum allowed allocation. "
+ "Requested resource type=[%s], "
+ "Requested resource=%s, maximum allowed allocation=%s, "
+ "please note that maximum allowed allocation is calculated "
+ "by scheduler based on maximum resource of registered "
+ "NodeManagers, which might be less than configured "
+ "maximum allocation=%s";
public static final String UNKNOWN_REASON_MESSAGE_TEMPLATE =
"Invalid resource request! "
+ "Cannot allocate containers for an unknown reason! "
+ "Requested resource type=[%s], Requested resource=%s";
public enum InvalidResourceType {
LESS_THAN_ZERO, GREATER_THEN_MAX_ALLOCATION, UNKNOWN;
}
private static final long serialVersionUID = 13498237L;
private final InvalidResourceType invalidResourceType;
public InvalidResourceRequestException(Throwable cause) {
super(cause);
this.invalidResourceType = InvalidResourceType.UNKNOWN;
}
public InvalidResourceRequestException(String message) {
this(message, InvalidResourceType.UNKNOWN);
}
public InvalidResourceRequestException(String message,
InvalidResourceType invalidResourceType) {
super(message);
this.invalidResourceType = invalidResourceType;
}
public InvalidResourceRequestException(String message, Throwable cause) {
super(message, cause);
this.invalidResourceType = InvalidResourceType.UNKNOWN;
}
public InvalidResourceType getInvalidResourceType() {
return invalidResourceType;
}
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -53,6 +53,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
.InvalidResourceType;
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@ -89,6 +91,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType
.GREATER_THEN_MAX_ALLOCATION;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType.LESS_THAN_ZERO;
/**
* This is the default Application Master Service processor. It has be the
* last processor in the @{@link AMSProcessingChain}.
@ -231,8 +239,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
maximumCapacity, app.getQueue(),
getScheduler(), getRmContext());
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw e;
RMAppAttempt rmAppAttempt = app.getRMAppAttempt(appAttemptId);
handleInvalidResourceException(e, rmAppAttempt);
}
try {
@ -336,6 +344,17 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
allocation.getPreviousAttemptContainers());
}
private void handleInvalidResourceException(InvalidResourceRequestException e,
RMAppAttempt rmAppAttempt) throws InvalidResourceRequestException {
if (e.getInvalidResourceType() == LESS_THAN_ZERO ||
e.getInvalidResourceType() == GREATER_THEN_MAX_ALLOCATION) {
rmAppAttempt.updateAMLaunchDiagnostics(e.getMessage());
}
LOG.warn("Invalid resource ask by application " +
rmAppAttempt.getAppAttemptId(), e);
throw e;
}
private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) {
Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>();
if(app.pullRMNodeUpdates(updatedNodes) > 0) {

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
View File

@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
.InvalidResourceType;
import org.apache.hadoop.yarn.exceptions
.SchedulerInvalidResoureRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@ -61,6 +63,15 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException
.GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException
.LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.UNKNOWN_REASON_MESSAGE_TEMPLATE;
/**
* Utilities shared by schedulers.
*/
@ -257,7 +268,7 @@ public class SchedulerUtils {
}
public static void normalizeAndValidateRequest(ResourceRequest resReq,
private static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
throws InvalidResourceRequestException {
@ -384,13 +395,13 @@ public class SchedulerUtils {
if (requestedRI.getValue() < 0) {
throwInvalidResourceException(reqResource, availableResource,
reqResourceName);
reqResourceName, InvalidResourceType.LESS_THAN_ZERO);
}
boolean valid = checkResource(requestedRI, availableResource);
if (!valid) {
throwInvalidResourceException(reqResource, availableResource,
reqResourceName);
reqResourceName, InvalidResourceType.GREATER_THEN_MAX_ALLOCATION);
}
}
}
@ -470,18 +481,30 @@ public class SchedulerUtils {
}
private static void throwInvalidResourceException(Resource reqResource,
Resource availableResource, String reqResourceName)
Resource maxAllowedAllocation, String reqResourceName,
InvalidResourceType invalidResourceType)
throws InvalidResourceRequestException {
throw new InvalidResourceRequestException(
"Invalid resource request, requested resource type=[" + reqResourceName
+ "] < 0 or greater than maximum allowed allocation. Requested "
+ "resource=" + reqResource + ", maximum allowed allocation="
+ availableResource
+ ", please note that maximum allowed allocation is calculated "
+ "by scheduler based on maximum resource of registered "
+ "NodeManagers, which might be less than configured "
+ "maximum allocation="
+ ResourceUtils.getResourceTypesMaximumAllocation());
final String message;
if (invalidResourceType == InvalidResourceType.LESS_THAN_ZERO) {
message = String.format(LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE,
reqResourceName, reqResource);
} else if (invalidResourceType ==
InvalidResourceType.GREATER_THEN_MAX_ALLOCATION) {
message = String.format(GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE,
reqResourceName, reqResource, maxAllowedAllocation,
ResourceUtils.getResourceTypesMaximumAllocation());
} else if (invalidResourceType == InvalidResourceType.UNKNOWN) {
message = String.format(UNKNOWN_REASON_MESSAGE_TEMPLATE, reqResourceName,
reqResource);
} else {
throw new IllegalArgumentException(String.format(
"InvalidResourceType argument should be either " + "%s, %s or %s",
InvalidResourceType.LESS_THAN_ZERO,
InvalidResourceType.GREATER_THEN_MAX_ALLOCATION,
InvalidResourceType.UNKNOWN));
}
throw new InvalidResourceRequestException(message, invalidResourceType);
}
private static void checkQueueLabelInLabelManager(String labelExpression,

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
View File

@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType
.GREATER_THEN_MAX_ALLOCATION;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType.LESS_THAN_ZERO;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -67,6 +72,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
.InvalidResourceType;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
@ -642,7 +649,7 @@ public class TestSchedulerUtils {
mockScheduler, rmContext);
fail("Negative memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
}
// negative vcores
@ -657,7 +664,7 @@ public class TestSchedulerUtils {
mockScheduler, rmContext);
fail("Negative vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
}
// more than max memory
@ -673,7 +680,7 @@ public class TestSchedulerUtils {
mockScheduler, rmContext);
fail("More than max memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
}
// more than max vcores
@ -688,7 +695,7 @@ public class TestSchedulerUtils {
mockScheduler, rmContext);
fail("More than max vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
}
}
@ -857,7 +864,9 @@ public class TestSchedulerUtils {
.create().withRequestedResourceType("custom-resource-1")
.withRequestedResource(requestedResource)
.withAvailableAllocation(availableResource)
.withMaxAllocation(configuredMaxAllocation).build());
.withMaxAllocation(configuredMaxAllocation)
.withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
.build());
SchedulerUtils.checkResourceRequestAgainstAvailableResource(
requestedResource, availableResource);
@ -891,7 +900,8 @@ public class TestSchedulerUtils {
ImmutableMap.of("custom-resource-1", "1M"));
Resource availableResource = ResourceTypesTestHelper.newResource(1, 1,
ImmutableMap.<String, String> builder().put("custom-resource-1", "120k")
ImmutableMap.<String, String>builder().put("custom-resource-1",
"120k")
.build());
exception.expect(InvalidResourceRequestException.class);
@ -899,7 +909,9 @@ public class TestSchedulerUtils {
.create().withRequestedResourceType("custom-resource-1")
.withRequestedResource(requestedResource)
.withAvailableAllocation(availableResource)
.withMaxAllocation(configuredMaxAllocation).build());
.withMaxAllocation(configuredMaxAllocation)
.withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
.build());
SchedulerUtils.checkResourceRequestAgainstAvailableResource(
requestedResource, availableResource);
}
@ -956,7 +968,9 @@ public class TestSchedulerUtils {
.create().withRequestedResourceType("custom-resource-1")
.withRequestedResource(requestedResource)
.withAvailableAllocation(availableResource)
.withMaxAllocation(configuredMaxAllocation).build());
.withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
.withMaxAllocation(configuredMaxAllocation)
.build());
SchedulerUtils.checkResourceRequestAgainstAvailableResource(
requestedResource, availableResource);
@ -1026,6 +1040,7 @@ public class TestSchedulerUtils {
private Resource availableAllocation;
private Resource configuredMaxAllowedAllocation;
private String resourceType;
private InvalidResourceType invalidResourceType;
InvalidResourceRequestExceptionMessageGenerator(StringBuilder sb) {
this.sb = sb;
@ -1060,18 +1075,41 @@ public class TestSchedulerUtils {
return this;
}
InvalidResourceRequestExceptionMessageGenerator
withInvalidResourceType(InvalidResourceType invalidResourceType) {
this.invalidResourceType = invalidResourceType;
return this;
}
public String build() {
return sb
.append("Invalid resource request, requested resource type=[")
if (invalidResourceType == LESS_THAN_ZERO) {
return sb.append("Invalid resource request! " +
"Cannot allocate containers as " +
"requested resource is less than 0! ")
.append("Requested resource type=[")
.append(resourceType).append("]")
.append(" < 0 or greater than maximum allowed allocation. ")
.append("Requested resource=").append(requestedResource).append(", ")
.append("maximum allowed allocation=").append(availableAllocation)
.append(", please note that maximum allowed allocation is calculated "
+ "by scheduler based on maximum resource of " +
"registered NodeManagers, which might be less than " +
"configured maximum allocation=")
.append(configuredMaxAllowedAllocation).toString();
.append(", Requested resource=")
.append(requestedResource).toString();
} else if (invalidResourceType == GREATER_THEN_MAX_ALLOCATION) {
return sb.append("Invalid resource request! " +
"Cannot allocate containers as "
+ "requested resource is greater than " +
"maximum allowed allocation. ")
.append("Requested resource type=[").append(resourceType)
.append("], ")
.append("Requested resource=").append(requestedResource)
.append(", maximum allowed allocation=")
.append(availableAllocation)
.append(", please note that maximum allowed allocation is " +
"calculated by scheduler based on maximum resource " +
"of registered NodeManagers, which might be less " +
"than configured maximum allocation=")
.append(configuredMaxAllowedAllocation)
.toString();
}
throw new IllegalStateException("Wrong type of InvalidResourceType is " +
"detected!");
}
}
}