YARN-8517. getContainer and getContainers ResourceManager REST API methods are not documented (snemeth via rkanter)

This commit is contained in:
Robert Kanter 2018-07-27 14:35:03 -07:00
parent fecbac499e
commit 2cccf4061c
4 changed files with 430 additions and 314 deletions

View File

@ -30,19 +30,55 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
*
*/
public class InvalidResourceRequestException extends YarnException {
public static final String LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE =
"Invalid resource request! Cannot allocate containers as "
+ "requested resource is less than 0! "
+ "Requested resource type=[%s], " + "Requested resource=%s";
public static final String GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE =
"Invalid resource request! Cannot allocate containers as "
+ "requested resource is greater than " +
"maximum allowed allocation. "
+ "Requested resource type=[%s], "
+ "Requested resource=%s, maximum allowed allocation=%s, "
+ "please note that maximum allowed allocation is calculated "
+ "by scheduler based on maximum resource of registered "
+ "NodeManagers, which might be less than configured "
+ "maximum allocation=%s";
public static final String UNKNOWN_REASON_MESSAGE_TEMPLATE =
"Invalid resource request! "
+ "Cannot allocate containers for an unknown reason! "
+ "Requested resource type=[%s], Requested resource=%s";
public enum InvalidResourceType {
LESS_THAN_ZERO, GREATER_THEN_MAX_ALLOCATION, UNKNOWN;
}
private static final long serialVersionUID = 13498237L;
private final InvalidResourceType invalidResourceType;
public InvalidResourceRequestException(Throwable cause) {
super(cause);
this.invalidResourceType = InvalidResourceType.UNKNOWN;
}
public InvalidResourceRequestException(String message) {
this(message, InvalidResourceType.UNKNOWN);
}
public InvalidResourceRequestException(String message,
InvalidResourceType invalidResourceType) {
super(message);
this.invalidResourceType = invalidResourceType;
}
public InvalidResourceRequestException(String message, Throwable cause) {
super(message, cause);
this.invalidResourceType = InvalidResourceType.UNKNOWN;
}
public InvalidResourceType getInvalidResourceType() {
return invalidResourceType;
}
}

View File

@ -53,6 +53,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
.InvalidResourceType;
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@ -89,6 +91,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType
.GREATER_THEN_MAX_ALLOCATION;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.InvalidResourceType.LESS_THAN_ZERO;
/**
* This is the default Application Master Service processor. It has be the
* last processor in the @{@link AMSProcessingChain}.
@ -231,8 +239,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
maximumCapacity, app.getQueue(),
getScheduler(), getRmContext());
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw e;
RMAppAttempt rmAppAttempt = app.getRMAppAttempt(appAttemptId);
handleInvalidResourceException(e, rmAppAttempt);
}
try {
@ -336,6 +344,17 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
allocation.getPreviousAttemptContainers());
}
private void handleInvalidResourceException(InvalidResourceRequestException e,
RMAppAttempt rmAppAttempt) throws InvalidResourceRequestException {
if (e.getInvalidResourceType() == LESS_THAN_ZERO ||
e.getInvalidResourceType() == GREATER_THEN_MAX_ALLOCATION) {
rmAppAttempt.updateAMLaunchDiagnostics(e.getMessage());
}
LOG.warn("Invalid resource ask by application " +
rmAppAttempt.getAppAttemptId(), e);
throw e;
}
private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) {
Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>();
if(app.pullRMNodeUpdates(updatedNodes) > 0) {

View File

@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
.InvalidResourceType;
import org.apache.hadoop.yarn.exceptions
.SchedulerInvalidResoureRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@ -61,6 +63,15 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException
.GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException
.LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE;
import static org.apache.hadoop.yarn.exceptions
.InvalidResourceRequestException.UNKNOWN_REASON_MESSAGE_TEMPLATE;
/**
* Utilities shared by schedulers.
*/
@ -257,9 +268,9 @@ public class SchedulerUtils {
}
public static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
private static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
throws InvalidResourceRequestException {
Configuration conf = rmContext.getYarnConfiguration();
// If Node label is not enabled throw exception
@ -384,13 +395,13 @@ public class SchedulerUtils {
if (requestedRI.getValue() < 0) {
throwInvalidResourceException(reqResource, availableResource,
reqResourceName);
reqResourceName, InvalidResourceType.LESS_THAN_ZERO);
}
boolean valid = checkResource(requestedRI, availableResource);
if (!valid) {
throwInvalidResourceException(reqResource, availableResource,
reqResourceName);
reqResourceName, InvalidResourceType.GREATER_THEN_MAX_ALLOCATION);
}
}
}
@ -470,18 +481,30 @@ public class SchedulerUtils {
}
private static void throwInvalidResourceException(Resource reqResource,
Resource availableResource, String reqResourceName)
Resource maxAllowedAllocation, String reqResourceName,
InvalidResourceType invalidResourceType)
throws InvalidResourceRequestException {
throw new InvalidResourceRequestException(
"Invalid resource request, requested resource type=[" + reqResourceName
+ "] < 0 or greater than maximum allowed allocation. Requested "
+ "resource=" + reqResource + ", maximum allowed allocation="
+ availableResource
+ ", please note that maximum allowed allocation is calculated "
+ "by scheduler based on maximum resource of registered "
+ "NodeManagers, which might be less than configured "
+ "maximum allocation="
+ ResourceUtils.getResourceTypesMaximumAllocation());
final String message;
if (invalidResourceType == InvalidResourceType.LESS_THAN_ZERO) {
message = String.format(LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE,
reqResourceName, reqResource);
} else if (invalidResourceType ==
InvalidResourceType.GREATER_THEN_MAX_ALLOCATION) {
message = String.format(GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE,
reqResourceName, reqResource, maxAllowedAllocation,
ResourceUtils.getResourceTypesMaximumAllocation());
} else if (invalidResourceType == InvalidResourceType.UNKNOWN) {
message = String.format(UNKNOWN_REASON_MESSAGE_TEMPLATE, reqResourceName,
reqResource);
} else {
throw new IllegalArgumentException(String.format(
"InvalidResourceType argument should be either " + "%s, %s or %s",
InvalidResourceType.LESS_THAN_ZERO,
InvalidResourceType.GREATER_THEN_MAX_ALLOCATION,
InvalidResourceType.UNKNOWN));
}
throw new InvalidResourceRequestException(message, invalidResourceType);
}
private static void checkQueueLabelInLabelManager(String labelExpression,