Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk" 

This reverts commit c163d1797a, reversing
changes made to 0d9804dcef.
This commit is contained in:
Anu Engineer 2018-07-05 12:22:18 -07:00
parent c163d1797a
commit 39ad98903a
28 changed files with 156 additions and 2404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
View File

@ -65,6 +65,11 @@
<Class name="~org\.apache\.hadoop\.yarn\.server\.resourcemanager\.rmapp\.attempt\.RMAppAttemptImpl.*" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
<Method name="getLocalityStatistics" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
<Method name="incNumAllocatedContainers"/>
@ -113,18 +118,6 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
<!-- Ignore exposed internal representations -->
<Match>
<Class name="org.apache.hadoop.yarn.api.records.Resource" />
<Method name="getResources" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
<Method name="getLocalityStatistics" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<!-- Object cast is based on the event type -->
<Match>
<Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
View File

@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
@ -76,18 +75,6 @@ public abstract class Resource implements Comparable<Resource> {
@Private
public static final int VCORES_INDEX = 1;
/**
* Return a new {@link Resource} instance with all resource values
* initialized to {@code value}.
* @param value the value to use for all resources
* @return a new {@link Resource} instance
*/
@Private
@Unstable
public static Resource newInstance(long value) {
return new LightWeightResource(value);
}
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
View File

@ -18,8 +18,9 @@
package org.apache.hadoop.yarn.api.records.impl;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@ -57,29 +58,13 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.*;
*
* @see Resource
*/
@Private
@InterfaceAudience.Private
@Unstable
public class LightWeightResource extends Resource {
private ResourceInformation memoryResInfo;
private ResourceInformation vcoresResInfo;
/**
* Create a new {@link LightWeightResource} instance with all resource values
* initialized to {@code value}.
* @param value the value to use for all resources
*/
public LightWeightResource(long value) {
ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
initResourceInformations(value, value, types.length);
for (int i = 2; i < types.length; i++) {
resources[i] = new ResourceInformation();
ResourceInformation.copy(types[i], resources[i]);
resources[i].setValue(value);
}
}
public LightWeightResource(long memory, int vcores) {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
@ -106,7 +91,7 @@ public class LightWeightResource extends Resource {
}
}
private void initResourceInformations(long memory, long vcores,
private void initResourceInformations(long memory, int vcores,
int numberOfKnownResourceTypes) {
this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(),
memory);

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
View File

@ -18,13 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* A {@code ConfigurableResource} object represents an entity that is used to
@ -37,53 +33,29 @@ public class ConfigurableResource {
private final Resource resource;
private final double[] percentages;
ConfigurableResource() {
this(getOneHundredPercentArray());
}
ConfigurableResource(double[] percentages) {
public ConfigurableResource(double[] percentages) {
this.percentages = percentages.clone();
this.resource = null;
}
ConfigurableResource(long value) {
this(Resource.newInstance(value));
}
public ConfigurableResource(Resource resource) {
this.percentages = null;
this.resource = resource;
}
private static double[] getOneHundredPercentArray() {
double[] resourcePercentages =
new double[ResourceUtils.getNumberOfKnownResourceTypes()];
Arrays.fill(resourcePercentages, 1.0);
return resourcePercentages;
}
/**
* Get resource by multiplying the cluster resource and the percentage of
* each resource respectively. Return the absolute resource if either
* {@code percentages} or {@code clusterResource} is null.
*
* @param clusterResource the cluster resource
* @return resource the resulting resource
* @return resource
*/
public Resource getResource(Resource clusterResource) {
if (percentages != null && clusterResource != null) {
long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
Resource res = Resource.newInstance(memory, vcore);
ResourceInformation[] clusterInfo = clusterResource.getResources();
for (int i = 2; i < clusterInfo.length; i++) {
res.setResourceValue(i,
(long)(clusterInfo[i].getValue() * percentages[i]));
}
return res;
return Resource.newInstance(memory, vcore);
} else {
return resource;
}
@ -97,39 +69,4 @@ public class ConfigurableResource {
public Resource getResource() {
return resource;
}
/**
* Set the value of the wrapped resource if this object isn't setup to use
* percentages. If this object is set to use percentages, this method has
* no effect.
*
* @param name the name of the resource
* @param value the value
*/
void setValue(String name, long value) {
if (resource != null) {
resource.setResourceValue(name, value);
}
}
/**
* Set the percentage of the resource if this object is setup to use
* percentages. If this object is set to use percentages, this method has
* no effect.
*
* @param name the name of the resource
* @param value the percentage
*/
void setPercentage(String name, double value) {
if (percentages != null) {
Integer index = ResourceUtils.getResourceTypeIndex().get(name);
if (index != null) {
percentages[index] = value;
} else {
throw new ResourceNotFoundException("The requested resource, \""
+ name + "\", could not be found.");
}
}
}
}

174
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -33,7 +33,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@ -214,9 +213,6 @@ public class FairSchedulerConfiguration extends Configuration {
CONF_PREFIX + "reservable-nodes";
public static final float RESERVABLE_NODES_DEFAULT = 0.05f;
private static final String INVALID_RESOURCE_DEFINITION_PREFIX =
"Error reading resource config--invalid resource definition: ";
public FairSchedulerConfiguration() {
super();
}
@ -411,167 +407,54 @@ public class FairSchedulerConfiguration extends Configuration {
}
/**
* Parses a resource config value in one of three forms:
* <ol>
* <li>Percentage: &quot;50%&quot; or &quot;40% memory, 60% cpu&quot;</li>
* <li>New style resources: &quot;vcores=10, memory-mb=1024&quot;
* or &quot;vcores=60%, memory-mb=40%&quot;</li>
* <li>Old style resources: &quot;1024 mb, 10 vcores&quot;</li>
* </ol>
* In new style resources, any resource that is not specified will be
* set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new
* style resources, units are not allowed. Units are assumed from the resource
* manager's settings for the resources when the value isn't a percentage.
*
* @param value the resource definition to parse
* @return a {@link ConfigurableResource} that represents the parsed value
* @throws AllocationConfigurationException if the raw value is not a valid
* resource definition
* Parses a resource config value of a form like "1024", "1024 mb",
* or "1024 mb, 3 vcores". If no units are given, megabytes are assumed.
*
* @throws AllocationConfigurationException
*/
public static ConfigurableResource parseResourceConfigValue(String value)
public static ConfigurableResource parseResourceConfigValue(String val)
throws AllocationConfigurationException {
return parseResourceConfigValue(value, Long.MAX_VALUE);
}
/**
* Parses a resource config value in one of three forms:
* <ol>
* <li>Percentage: &quot;50%&quot; or &quot;40% memory, 60% cpu&quot;</li>
* <li>New style resources: &quot;vcores=10, memory-mb=1024&quot;
* or &quot;vcores=60%, memory-mb=40%&quot;</li>
* <li>Old style resources: &quot;1024 mb, 10 vcores&quot;</li>
* </ol>
* In new style resources, any resource that is not specified will be
* set to {@code missing} or 0%, as appropriate. Also, in the new style
* resources, units are not allowed. Units are assumed from the resource
* manager's settings for the resources when the value isn't a percentage.
*
* The {@code missing} parameter is only used in the case of new style
* resources without percentages. With new style resources with percentages,
* any missing resources will be assumed to be 100% because percentages are
* only used with maximum resource limits.
*
* @param value the resource definition to parse
* @param missing the value to use for any unspecified resources
* @return a {@link ConfigurableResource} that represents the parsed value
* @throws AllocationConfigurationException if the raw value is not a valid
* resource definition
*/
public static ConfigurableResource parseResourceConfigValue(String value,
long missing) throws AllocationConfigurationException {
ConfigurableResource configurableResource;
if (value.trim().isEmpty()) {
throw new AllocationConfigurationException("Error reading resource "
+ "config--the resource string is empty.");
}
try {
if (value.contains("=")) {
configurableResource = parseNewStyleResource(value, missing);
} else if (value.contains("%")) {
configurableResource = parseOldStyleResourceAsPercentage(value);
val = StringUtils.toLowerCase(val);
if (val.contains("%")) {
configurableResource = new ConfigurableResource(
getResourcePercentage(val));
} else {
configurableResource = parseOldStyleResource(value);
int memory = findResource(val, "mb");
int vcores = findResource(val, "vcores");
configurableResource = new ConfigurableResource(
BuilderUtils.newResource(memory, vcores));
}
} catch (RuntimeException ex) {
} catch (AllocationConfigurationException ex) {
throw ex;
} catch (Exception ex) {
throw new AllocationConfigurationException(
"Error reading resource config", ex);
}
return configurableResource;
}
private static ConfigurableResource parseNewStyleResource(String value,
long missing) throws AllocationConfigurationException {
final ConfigurableResource configurableResource;
boolean asPercent = value.contains("%");
if (asPercent) {
configurableResource = new ConfigurableResource();
} else {
configurableResource = new ConfigurableResource(missing);
}
String[] resources = value.split(",");
for (String resource : resources) {
String[] parts = resource.split("=");
if (parts.length != 2) {
throw createConfigException(value,
"Every resource must be of the form: name=value.");
}
String resourceName = parts[0].trim();
String resourceValue = parts[1].trim();
try {
if (asPercent) {
configurableResource.setPercentage(resourceName,
findPercentage(resourceValue, ""));
} else {
configurableResource.setValue(resourceName,
Long.parseLong(resourceValue));
}
} catch (ResourceNotFoundException ex) {
throw createConfigException(value, "The "
+ "resource name, \"" + resourceName + "\" was not "
+ "recognized. Please check the value of "
+ YarnConfiguration.RESOURCE_TYPES + " in the Resource "
+ "Manager's configuration files.", ex);
} catch (NumberFormatException ex) {
// This only comes from Long.parseLong()
throw createConfigException(value, "The "
+ "resource values must all be integers. \"" + resourceValue
+ "\" is not an integer.", ex);
} catch (AllocationConfigurationException ex) {
// This only comes from findPercentage()
throw createConfigException(value, "The "
+ "resource values must all be percentages. \""
+ resourceValue + "\" is either not a number or does not "
+ "include the '%' symbol.", ex);
}
}
return configurableResource;
}
private static ConfigurableResource parseOldStyleResourceAsPercentage(
String value) throws AllocationConfigurationException {
return new ConfigurableResource(
getResourcePercentage(StringUtils.toLowerCase(value)));
}
private static ConfigurableResource parseOldStyleResource(String value)
throws AllocationConfigurationException {
final String lCaseValue = StringUtils.toLowerCase(value);
int memory = findResource(lCaseValue, "mb");
int vcores = findResource(lCaseValue, "vcores");
return new ConfigurableResource(
BuilderUtils.newResource(memory, vcores));
}
private static double[] getResourcePercentage(
String val) throws AllocationConfigurationException {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
double[] resourcePercentage = new double[numberOfKnownResourceTypes];
String[] strings = val.split(",");
if (strings.length == 1) {
double percentage = findPercentage(strings[0], "");
for (int i = 0; i < numberOfKnownResourceTypes; i++) {
resourcePercentage[i] = percentage;
resourcePercentage[i] = percentage/100;
}
} else {
resourcePercentage[0] = findPercentage(val, "memory");
resourcePercentage[1] = findPercentage(val, "cpu");
resourcePercentage[0] = findPercentage(val, "memory")/100;
resourcePercentage[1] = findPercentage(val, "cpu")/100;
}
return resourcePercentage;
}
private static double findPercentage(String val, String units)
throws AllocationConfigurationException {
throws AllocationConfigurationException {
final Pattern pattern =
Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units);
Matcher matcher = pattern.matcher(val);
@ -584,22 +467,7 @@ public class FairSchedulerConfiguration extends Configuration {
units);
}
}
return Double.parseDouble(matcher.group(1)) / 100.0;
}
private static AllocationConfigurationException createConfigException(
String value, String message) {
return createConfigException(value, message, null);
}
private static AllocationConfigurationException createConfigException(
String value, String message, Throwable t) {
String msg = INVALID_RESOURCE_DEFINITION_PREFIX + value + ". " + message;
if (t != null) {
return new AllocationConfigurationException(msg, t);
} else {
return new AllocationConfigurationException(msg);
}
return Double.parseDouble(matcher.group(1));
}
public long getUpdateInterval() {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
View File

@ -134,7 +134,7 @@ public class AllocationFileQueueParser {
if (MIN_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text, 0L);
FairSchedulerConfiguration.parseResourceConfigValue(text);
builder.minQueueResources(queueName, val.getResource());
} else if (MAX_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -479,7 +479,7 @@ public class AppInfo {
public int getNumNonAMContainersPreempted() {
return numNonAMContainerPreempted;
}
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
View File

@ -41,9 +41,8 @@ public class SchedulerInfo {
protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes;
protected int maximumClusterPriority;
// JAXB needs this
public SchedulerInfo() {
}
} // JAXB needs this
public SchedulerInfo(final ResourceManager rm) {
ResourceScheduler rs = rm.getResourceScheduler();
@ -75,10 +74,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
if (minAllocResource != null) {
return Arrays.toString(minAllocResource.getResource().getResources());
}
return null;
return Arrays.toString(minAllocResource.getResource().getResources());
}
public int getMaxClusterLevelAppPriority() {

158
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
View File

@ -48,9 +48,6 @@ import org.apache.log4j.spi.LoggingEvent;
import org.junit.Assert;
import org.junit.Test;
/**
* Tests fair scheduler configuration.
*/
public class TestFairSchedulerConfiguration {
private static final String A_CUSTOM_RESOURCE = "a-custom-resource";
@ -105,152 +102,67 @@ public class TestFairSchedulerConfiguration {
@Test
public void testParseResourceConfigValue() throws Exception {
Resource expected = BuilderUtils.newResource(5 * 1024, 2);
Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4);
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("2 vcores, 1024 mb").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("1024 mb, 2 vcores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("2vcores,1024mb").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("1024mb,2vcores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("1024 mb, 2 vcores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource());
assertEquals(expected,
parseResourceConfigValue("2 vcores, 5120 mb").getResource());
assertEquals(expected,
parseResourceConfigValue("5120 mb, 2 vcores").getResource());
assertEquals(expected,
parseResourceConfigValue("2vcores,5120mb").getResource());
assertEquals(expected,
parseResourceConfigValue("5120mb,2vcores").getResource());
assertEquals(expected,
parseResourceConfigValue("5120mb mb, 2 vcores").getResource());
assertEquals(expected,
parseResourceConfigValue("5120 Mb, 2 vCores").getResource());
assertEquals(expected,
parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource());
assertEquals(expected,
parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource());
assertEquals(expected,
parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource());
assertEquals(expected,
Resource clusterResource = BuilderUtils.newResource(2048, 4);
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50% memory, 50% cpu").
getResource(clusterResource));
assertEquals(expected,
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50% Memory, 50% CpU").
getResource(clusterResource));
assertEquals(BuilderUtils.newResource(5 * 1024, 4),
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50%").getResource(clusterResource));
assertEquals(BuilderUtils.newResource(1024, 4),
parseResourceConfigValue("50% memory, 100% cpu").
getResource(clusterResource));
assertEquals(BuilderUtils.newResource(5 * 1024, 4),
assertEquals(BuilderUtils.newResource(1024, 4),
parseResourceConfigValue(" 100% cpu, 50% memory").
getResource(clusterResource));
assertEquals(BuilderUtils.newResource(5 * 1024, 0),
assertEquals(BuilderUtils.newResource(1024, 0),
parseResourceConfigValue("50% memory, 0% cpu").
getResource(clusterResource));
assertEquals(expected,
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50 % memory, 50 % cpu").
getResource(clusterResource));
assertEquals(expected,
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50%memory,50%cpu").
getResource(clusterResource));
assertEquals(expected,
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue(" 50 % memory, 50 % cpu ").
getResource(clusterResource));
assertEquals(expected,
assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50.% memory, 50.% cpu").
getResource(clusterResource));
clusterResource = BuilderUtils.newResource(1024 * 10, 4);
assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
parseResourceConfigValue("10.9% memory, 50.6% cpu").
getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("50%").getResource(clusterResource));
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RESOURCE_TYPES, "test1");
ResourceUtils.resetResourceTypes(conf);
clusterResource = BuilderUtils.newResource(10 * 1024, 4);
expected = BuilderUtils.newResource(5 * 1024, 2);
expected.setResourceValue("test1", Long.MAX_VALUE);
assertEquals(expected,
parseResourceConfigValue("vcores=2, memory-mb=5120").getResource());
assertEquals(expected,
parseResourceConfigValue("memory-mb=5120, vcores=2").getResource());
assertEquals(expected,
parseResourceConfigValue("vcores=2,memory-mb=5120").getResource());
assertEquals(expected, parseResourceConfigValue(" vcores = 2 , "
+ "memory-mb = 5120 ").getResource());
expected.setResourceValue("test1", 0L);
assertEquals(expected,
parseResourceConfigValue("vcores=2, memory-mb=5120", 0L).getResource());
assertEquals(expected,
parseResourceConfigValue("memory-mb=5120, vcores=2", 0L).getResource());
assertEquals(expected,
parseResourceConfigValue("vcores=2,memory-mb=5120", 0L).getResource());
assertEquals(expected,
parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ",
0L).getResource());
clusterResource.setResourceValue("test1", 8L);
expected.setResourceValue("test1", 4L);
assertEquals(expected,
parseResourceConfigValue("50%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("vcores=2, memory-mb=5120, "
+ "test1=4").getResource());
assertEquals(expected,
parseResourceConfigValue("test1=4, vcores=2, "
+ "memory-mb=5120").getResource());
assertEquals(expected,
parseResourceConfigValue("memory-mb=5120, test1=4, "
+ "vcores=2").getResource());
assertEquals(expected,
parseResourceConfigValue("vcores=2,memory-mb=5120,"
+ "test1=4").getResource());
assertEquals(expected,
parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , "
+ "test1 = 4 ").getResource());
expected = BuilderUtils.newResource(4 * 1024, 3);
expected.setResourceValue("test1", 8L);
assertEquals(expected,
parseResourceConfigValue("vcores=75%, "
+ "memory-mb=40%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("memory-mb=40%, "
+ "vcores=75%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("vcores=75%,"
+ "memory-mb=40%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue(" vcores = 75 % , "
+ "memory-mb = 40 % ").getResource(clusterResource));
expected.setResourceValue("test1", 4L);
assertEquals(expected,
parseResourceConfigValue("vcores=75%, memory-mb=40%, "
+ "test1=50%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("test1=50%, vcores=75%, "
+ "memory-mb=40%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("memory-mb=40%, test1=50%, "
+ "vcores=75%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue("vcores=75%,memory-mb=40%,"
+ "test1=50%").getResource(clusterResource));
assertEquals(expected,
parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , "
+ "test1 = 50 % ").getResource(clusterResource));
}
@Test(expected = AllocationConfigurationException.class)
public void testNoUnits() throws Exception {
parseResourceConfigValue("1024");
}
@Test(expected = AllocationConfigurationException.class)
public void testOnlyMemory() throws Exception {
parseResourceConfigValue("1024mb");
@ -260,7 +172,7 @@ public class TestFairSchedulerConfiguration {
public void testOnlyCPU() throws Exception {
parseResourceConfigValue("1024vcores");
}
@Test(expected = AllocationConfigurationException.class)
public void testGibberish() throws Exception {
parseResourceConfigValue("1o24vc0res");

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -53,7 +53,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.*;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@ -72,12 +76,11 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.eclipse.jetty.server.Response;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@ -93,8 +96,6 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServices extends JerseyTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestRMWebServices.class);
private static MockRM rm;
@ -471,19 +472,19 @@ public class TestRMWebServices extends JerseyTestBase {
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
long totalMBExpect =
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
long totalVirtualCoresExpect =
long totalVirtualCoresExpect =
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
assertEquals("appsCompleted doesn't match",
metrics.getAppsCompleted(), completedApps);
assertEquals("reservedMB doesn't match",
metrics.getReservedMB(), reservedMB);
assertEquals("availableMB doesn't match",
assertEquals("availableMB doesn't match",
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
assertEquals("reservedVirtualCores doesn't match",
metrics.getReservedVirtualCores(), reservedVirtualCores);
@ -596,13 +597,11 @@ public class TestRMWebServices extends JerseyTestBase {
public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements in: " + json, 1, json.length());
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
LOG.debug("schedulerInfo: {}", info);
assertEquals("incorrect number of elements in: " + info, 11, info.length());
assertEquals("incorrect number of elements", 11, info.length());
verifyClusterSchedulerFifoGeneric(info.getString("type"),
info.getString("qstate"), (float) info.getDouble("capacity"),

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -79,7 +79,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServicesApps extends JerseyTestBase {
private static MockRM rm;
private static final int CONTAINER_MB = 1024;
private static class WebServletModule extends ServletModule {
@ -324,7 +324,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue("both app states of ACCEPTED and KILLED are not present",
assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
@ -375,12 +375,12 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue("both app states of ACCEPTED and KILLED are not present",
assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
array.getJSONObject(1).getString("state").equals("ACCEPTED")));
rm.stop();
}
@ -511,8 +511,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("finalStatus",
FinalApplicationStatus.UNDEFINED.toString())
.path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
@ -1805,8 +1804,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
int numAttempt = 1;
while (true) {
// fail the AM by sending CONTAINER_FINISHED event without registering.
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
ContainerState.COMPLETE);
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
if (numAttempt == maxAppAttempts) {
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);

242
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
View File

@ -1,242 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import static org.junit.Assert.assertEquals;
/**
* This test verifies that custom resource types are correctly serialized to XML
* and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
*/
public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
private static MockRM rm;
private static final int CONTAINER_MB = 1024;
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
ResourceScheduler.class);
initResourceTypes(conf);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
private void initResourceTypes(Configuration conf) {
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
CustomResourceTypesConfigurationProvider.class.getName());
ResourceUtils.resetResourceTypes(conf);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
createInjectorForWebServletModule();
}
private void createInjectorForWebServletModule() {
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesAppsCustomResourceTypes() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testRunningAppXml() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
am1.allocate("*", 2048, 1, new ArrayList<>());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
XmlCustomResourceTypeTestCase testCase =
new XmlCustomResourceTypeTestCase(path,
new BufferedClientResponse(response));
testCase.verify(document -> {
NodeList apps = document.getElementsByTagName("apps");
assertEquals("incorrect number of apps elements", 1, apps.getLength());
NodeList appArray = ((Element)(apps.item(0)))
.getElementsByTagName("app");
assertEquals("incorrect number of app elements", 1, appArray.getLength());
verifyAppsXML(appArray, app1);
});
rm.stop();
}
@Test
public void testRunningAppJson() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
am1.allocate("*", 2048, 1, new ArrayList<>());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
JsonCustomResourceTypeTestcase testCase =
new JsonCustomResourceTypeTestcase(path,
new BufferedClientResponse(response));
testCase.verify(json -> {
try {
assertEquals("incorrect number of apps elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of app elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect count of app", 1, array.length());
verifyAppInfoJson(array.getJSONObject(0), app1);
} catch (JSONException e) {
throw new RuntimeException(e);
}
});
rm.stop();
}
private void verifyAppsXML(NodeList appArray, RMApp app) {
for (int i = 0; i < appArray.getLength(); i++) {
Element element = (Element) appArray.item(i);
AppInfoXmlVerifications.verify(element, app);
NodeList resourceRequests =
element.getElementsByTagName("resourceRequests");
assertEquals(1, resourceRequests.getLength());
Node resourceRequest = resourceRequests.item(0);
ResourceRequest rr =
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
.getApplicationAttempt(
app.getCurrentAppAttempt().getAppAttemptId())
.getAppSchedulingInfo().getAllResourceRequests().get(0);
ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
(Element) resourceRequest, rr,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
}
private void verifyAppInfoJson(JSONObject info, RMApp app) throws
JSONException {
int expectedNumberOfElements = getExpectedNumberOfElements(app);
assertEquals("incorrect number of elements", expectedNumberOfElements,
info.length());
AppInfoJsonVerifications.verify(info, app);
JSONArray resourceRequests = info.getJSONArray("resourceRequests");
JSONObject requestInfo = resourceRequests.getJSONObject(0);
ResourceRequest rr =
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
.getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
.getAppSchedulingInfo().getAllResourceRequests().get(0);
ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
requestInfo, rr,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
private int getExpectedNumberOfElements(RMApp app) {
int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
expectedNumberOfElements++;
}
if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
expectedNumberOfElements++;
}
if (AppInfo
.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
expectedNumberOfElements++;
}
return expectedNumberOfElements;
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -146,7 +146,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
config.setUserLimitFactor(B2, 100.0f);
config.setCapacity(B3, 0.5f);
config.setUserLimitFactor(B3, 100.0f);
config.setQueues(A1, new String[] {"a1a", "a1b"});
final String A1A = A1 + ".a1a";
config.setCapacity(A1A, 85);
@ -254,7 +254,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
}
}
public void verifySubQueueXML(Element qElem, String q,
public void verifySubQueueXML(Element qElem, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws Exception {
NodeList children = qElem.getChildNodes();
@ -317,34 +317,30 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements in: " + json, 1, json.length());
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 8, info.length());
assertEquals("incorrect number of elements", 8, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements in: " + health, 3,
health.length());
assertEquals("incorrect number of elements", 3, health.length());
JSONArray operationsInfo = health.getJSONArray("operationsInfo");
assertEquals("incorrect number of elements in: " + health, 4,
operationsInfo.length());
assertEquals("incorrect number of elements", 4, operationsInfo.length());
JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
assertEquals("incorrect number of elements in: " + health, 3,
lastRunDetails.length());
assertEquals("incorrect number of elements", 3, lastRunDetails.length());
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
assertEquals("incorrect number of elements", 2, arr.length());
// test subqueues
for (int i = 0; i < arr.length(); i++) {
JSONObject obj = arr.getJSONObject(i);
String q = CapacitySchedulerConfiguration.ROOT + "." +
obj.getString("queueName");
String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName");
verifySubQueue(obj, q, 100, 100);
}
}
@ -359,7 +355,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
assertTrue("queueName doesn't match", "root".matches(queueName));
}
private void verifySubQueue(JSONObject info, String q,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 20;
@ -468,7 +464,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
}
//Return a child Node of node with the tagname or null if none exists
//Return a child Node of node with the tagname or null if none exists
private Node getChildNodeByName(Node node, String tagname) {
NodeList nodeList = node.getChildNodes();
for (int i=0; i < nodeList.getLength(); ++i) {
@ -518,7 +514,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
for (int j=0; j<users.getLength(); ++j) {
Node user = users.item(j);
String username = getChildNodeByName(user, "username")
.getTextContent();
.getTextContent();
assertTrue(username.equals("user1") || username.equals("user2"));
//Should be a parsable integer
Integer.parseInt(getChildNodeByName(getChildNodeByName(user,

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
View File

@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
@ -61,8 +59,6 @@ import static org.junit.Assert.assertNull;
* Test scheduler configuration mutation via REST API.
*/
public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
private static final Logger LOG = LoggerFactory
.getLogger(TestRMWebServicesConfigurationMutation.class);
private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@ -400,7 +396,6 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
LOG.debug("Response headers: " + response.getHeaders());
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
assertEquals(0.2f, newCSConf

95
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -16,14 +16,13 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import javax.ws.rs.core.MediaType;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@ -31,9 +30,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@ -42,18 +38,18 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import javax.ws.rs.core.MediaType;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
* Tests RM Webservices fair scheduler resources.
*/
public class TestRMWebServicesFairScheduler extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
@ -62,7 +58,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
@ -70,32 +66,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
static {
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesFairScheduler() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testClusterScheduler() throws JSONException {
public void testClusterScheduler() throws JSONException, Exception {
WebResource r = resource();
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
@ -103,51 +99,52 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
@Test
public void testClusterSchedulerSlash() throws JSONException {
public void testClusterSchedulerSlash() throws JSONException, Exception {
WebResource r = resource();
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler/")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler/").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
verifyClusterScheduler(json);
}
@Test
public void testClusterSchedulerWithSubQueues()
throws JSONException {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
public void testClusterSchedulerWithSubQueues() throws JSONException,
Exception {
FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueue
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
WebResource r = resource();
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
.getJSONObject("childQueues").getJSONArray("queue");
.getJSONObject("childQueues").getJSONArray("queue")
.getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
try {
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
fail("FairSchedulerQueueInfo should omit field 'childQueues'"
+ "if child queue is empty.");
fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
"if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
}
}
private void verifyClusterScheduler(JSONObject json) throws JSONException {
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
View File

@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities
if (object.getClass() == JSONObject.class) {
assertEquals("Number of allocations is wrong", 1, realValue);
} else if (object.getClass() == JSONArray.class) {
assertEquals("Number of allocations is wrong in: " + object,
assertEquals("Number of allocations is wrong",
((JSONArray) object).length(), realValue);
}
}

138
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
View File

@ -1,138 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static java.util.stream.Collectors.toList;
/**
* This class can generate an XML configuration file of custom resource types.
* See createInitialResourceTypes for the default values. All custom resource
* type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the
* getConfigurationInputStream method to get an InputStream of the XML. If you
* want to have different number of resources in your tests, please see usages
* of this class in this test class:
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}
*
*/
public class CustomResourceTypesConfigurationProvider
extends LocalConfigurationProvider {
private static class CustomResourceTypes {
private int count;
private String xml;
CustomResourceTypes(String xml, int count) {
this.xml = xml;
this.count = count;
}
public int getCount() {
return count;
}
public String getXml() {
return xml;
}
}
private static final String CUSTOM_RESOURCE_PREFIX = "customResource-";
private static CustomResourceTypes customResourceTypes =
createInitialResourceTypes();
private static CustomResourceTypes createInitialResourceTypes() {
return createCustomResourceTypes(2);
}
private static CustomResourceTypes createCustomResourceTypes(int count) {
List<String> resourceTypeNames = generateResourceTypeNames(count);
List<String> resourceUnitXmlElements = IntStream.range(0, count)
.boxed()
.map(i -> getResourceUnitsXml(resourceTypeNames.get(i)))
.collect(toList());
StringBuilder sb = new StringBuilder("<configuration>\n");
sb.append(getResourceTypesXml(resourceTypeNames));
for (String resourceUnitXml : resourceUnitXmlElements) {
sb.append(resourceUnitXml);
}
sb.append("</configuration>");
return new CustomResourceTypes(sb.toString(), count);
}
private static List<String> generateResourceTypeNames(int count) {
return IntStream.range(0, count)
.boxed()
.map(i -> CUSTOM_RESOURCE_PREFIX + i)
.collect(toList());
}
private static String getResourceUnitsXml(String resource) {
return "<property>\n" + "<name>yarn.resource-types." + resource
+ ".units</name>\n" + "<value>k</value>\n" + "</property>\n";
}
private static String getResourceTypesXml(List<String> resources) {
final String resourceTypes = makeCommaSeparatedString(resources);
return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>"
+ resourceTypes + "</value>\n" + "</property>\n";
}
private static String makeCommaSeparatedString(List<String> resources) {
return resources.stream().collect(Collectors.joining(","));
}
@Override
public InputStream getConfigurationInputStream(Configuration bootstrapConf,
String name) throws YarnException, IOException {
if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
return new ByteArrayInputStream(
customResourceTypes.getXml().getBytes());
} else {
return super.getConfigurationInputStream(bootstrapConf, name);
}
}
public static void reset() {
customResourceTypes = createInitialResourceTypes();
}
public static void setNumberOfResourceTypes(int count) {
customResourceTypes = createCustomResourceTypes(count);
}
public static List<String> getCustomResourceTypes() {
return generateResourceTypeNames(customResourceTypes.getCount());
}
}

139
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
View File

@ -1,139 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* This test helper class is primarily used by
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
*/
public class FairSchedulerJsonVerifications {
private static final Set<String> RESOURCE_FIELDS =
Sets.newHashSet("minResources", "amUsedResources", "amMaxResources",
"fairResources", "clusterResources", "reservedResources",
"maxResources", "usedResources", "steadyFairResources",
"demandResources");
private final Set<String> customResourceTypes;
FairSchedulerJsonVerifications(List<String> customResourceTypes) {
this.customResourceTypes = Sets.newHashSet(customResourceTypes);
}
public void verify(JSONObject jsonObject) {
try {
verifyResourcesContainDefaultResourceTypes(jsonObject, RESOURCE_FIELDS);
verifyResourcesContainCustomResourceTypes(jsonObject, RESOURCE_FIELDS);
} catch (JSONException e) {
throw new RuntimeException(e);
}
}
private void verifyResourcesContainDefaultResourceTypes(JSONObject queue,
Set<String> resourceCategories) throws JSONException {
for (String resourceCategory : resourceCategories) {
boolean hasResourceCategory = queue.has(resourceCategory);
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, hasResourceCategory);
verifyResourceContainsDefaultResourceTypes(
queue.getJSONObject(resourceCategory));
}
}
private void verifyResourceContainsDefaultResourceTypes(
JSONObject jsonObject) {
Object memory = jsonObject.opt("memory");
Object vCores = jsonObject.opt("vCores");
assertNotNull("Key 'memory' not found in: " + jsonObject, memory);
assertNotNull("Key 'vCores' not found in: " + jsonObject, vCores);
}
private void verifyResourcesContainCustomResourceTypes(JSONObject queue,
Set<String> resourceCategories) throws JSONException {
for (String resourceCategory : resourceCategories) {
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, queue.has(resourceCategory));
verifyResourceContainsAllCustomResourceTypes(
queue.getJSONObject(resourceCategory));
}
}
private void verifyResourceContainsAllCustomResourceTypes(
JSONObject resourceCategory) throws JSONException {
assertTrue("resourceCategory does not have resourceInformations: "
+ resourceCategory, resourceCategory.has("resourceInformations"));
JSONObject resourceInformations =
resourceCategory.getJSONObject("resourceInformations");
assertTrue(
"resourceInformations does not have resourceInformation object: "
+ resourceInformations,
resourceInformations.has("resourceInformation"));
JSONArray customResources =
resourceInformations.getJSONArray("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
customResourceTypes.size(), customResources.length() - 2);
for (int i = 0; i < customResources.length(); i++) {
JSONObject customResource = customResources.getJSONObject(i);
assertTrue("Resource type does not have name field: " + customResource,
customResource.has("name"));
assertTrue("Resource type does not have name resourceType field: "
+ customResource, customResource.has("resourceType"));
assertTrue(
"Resource type does not have name units field: " + customResource,
customResource.has("units"));
assertTrue(
"Resource type does not have name value field: " + customResource,
customResource.has("value"));
String name = customResource.getString("name");
String unit = customResource.getString("units");
String resourceType = customResource.getString("resourceType");
Long value = customResource.getLong("value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
customResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Custom resource value " + value + " is null!", value);
}
}
}

153
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
View File

@ -1,153 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.List;
import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* This test helper class is primarily used by
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
*/
public class FairSchedulerXmlVerifications {
private static final Set<String> RESOURCE_FIELDS = Sets.newHashSet(
"minResources", "amUsedResources", "amMaxResources", "fairResources",
"clusterResources", "reservedResources", "maxResources", "usedResources",
"steadyFairResources", "demandResources");
private final Set<String> customResourceTypes;
FairSchedulerXmlVerifications(List<String> customResourceTypes) {
this.customResourceTypes = Sets.newHashSet(customResourceTypes);
}
public void verify(Element element) {
verifyResourcesContainDefaultResourceTypes(element, RESOURCE_FIELDS);
verifyResourcesContainCustomResourceTypes(element, RESOURCE_FIELDS);
}
private void verifyResourcesContainDefaultResourceTypes(Element queue,
Set<String> resourceCategories) {
for (String resourceCategory : resourceCategories) {
boolean hasResourceCategory = hasChild(queue, resourceCategory);
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, hasResourceCategory);
verifyResourceContainsDefaultResourceTypes(
(Element) queue.getElementsByTagName(resourceCategory).item(0));
}
}
private void verifyResourceContainsDefaultResourceTypes(
Element element) {
Object memory = opt(element, "memory");
Object vCores = opt(element, "vCores");
assertNotNull("Key 'memory' not found in: " + element, memory);
assertNotNull("Key 'vCores' not found in: " + element, vCores);
}
private void verifyResourcesContainCustomResourceTypes(Element queue,
Set<String> resourceCategories) {
for (String resourceCategory : resourceCategories) {
assertTrue("Queue " + queue + " does not have key for resourceCategory: "
+ resourceCategory, hasChild(queue, resourceCategory));
verifyResourceContainsCustomResourceTypes(
(Element) queue.getElementsByTagName(resourceCategory).item(0));
}
}
private void verifyResourceContainsCustomResourceTypes(
Element resourceCategory) {
assertEquals(
toXml(resourceCategory)
+ " should have only one resourceInformations child!",
1, resourceCategory.getElementsByTagName("resourceInformations")
.getLength());
Element resourceInformations = (Element) resourceCategory
.getElementsByTagName("resourceInformations").item(0);
NodeList customResources =
resourceInformations.getElementsByTagName("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
customResourceTypes.size(), customResources.getLength() - 2);
for (int i = 0; i < customResources.getLength(); i++) {
Element customResource = (Element) customResources.item(i);
String name = getXmlString(customResource, "name");
String unit = getXmlString(customResource, "units");
String resourceType = getXmlString(customResource, "resourceType");
Long value = getXmlLong(customResource, "value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
customResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Resource value should not be null for resource type "
+ resourceType + ", listing xml contents: " + toXml(customResource),
value);
}
}
private Object opt(Node node, String child) {
NodeList nodes = getElementsByTagNameInternal(node, child);
if (nodes.getLength() > 0) {
return nodes.item(0);
}
return null;
}
private boolean hasChild(Node node, String child) {
return getElementsByTagNameInternal(node, child).getLength() > 0;
}
private NodeList getElementsByTagNameInternal(Node node, String child) {
if (node instanceof Element) {
return ((Element) node).getElementsByTagName(child);
} else if (node instanceof Document) {
return ((Document) node).getElementsByTagName(child);
} else {
throw new IllegalStateException("Unknown type of wrappedObject: " + node
+ ", type: " + node.getClass());
}
}
}

271
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
View File

@ -1,271 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.*;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Element;
import javax.ws.rs.core.MediaType;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
/**
* This class is to test response representations of queue resources,
* explicitly setting custom resource types. with the help of
* {@link CustomResourceTypesConfigurationProvider}
*/
public class TestRMWebServicesFairSchedulerCustomResourceTypes
extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
initResourceTypes(conf);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
private void initResourceTypes(YarnConfiguration conf) {
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
CustomResourceTypesConfigurationProvider.class.getName());
ResourceUtils.resetResourceTypes(conf);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
createInjectorForWebServletModule();
}
@After
public void tearDown() {
ResourceUtils.resetResourceTypes(new Configuration());
}
private void createInjectorForWebServletModule() {
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
@After
public void teardown() {
CustomResourceTypesConfigurationProvider.reset();
}
public TestRMWebServicesFairSchedulerCustomResourceTypes() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testClusterSchedulerWithCustomResourceTypesJson() {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
verifyJsonResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerWithCustomResourceTypesXml() {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
verifyXmlResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerWithElevenCustomResourceTypesXml() {
CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
createInjectorForWebServletModule();
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
verifyXmlResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerElevenWithCustomResourceTypesJson() {
CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
createInjectorForWebServletModule();
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
verifyJsonResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
private void verifyJsonResponse(WebResource path, ClientResponse response,
List<String> customResourceTypes) {
JsonCustomResourceTypeTestcase testCase =
new JsonCustomResourceTypeTestcase(path,
new BufferedClientResponse(response));
testCase.verify(json -> {
try {
JSONArray queues = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue");
// childQueueInfo consists of subqueue1 and subqueue2 info
assertEquals(2, queues.length());
JSONObject firstChildQueue = queues.getJSONObject(0);
new FairSchedulerJsonVerifications(customResourceTypes)
.verify(firstChildQueue);
} catch (JSONException e) {
throw new RuntimeException(e);
}
});
}
private void verifyXmlResponse(WebResource path, ClientResponse response,
List<String> customResourceTypes) {
XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase(
path, new BufferedClientResponse(response));
testCase.verify(xml -> {
Element scheduler =
(Element) xml.getElementsByTagName("scheduler").item(0);
Element schedulerInfo =
(Element) scheduler.getElementsByTagName("schedulerInfo").item(0);
Element rootQueue =
(Element) schedulerInfo.getElementsByTagName("rootQueue").item(0);
Element childQueues =
(Element) rootQueue.getElementsByTagName("childQueues").item(0);
Element queue =
(Element) childQueues.getElementsByTagName("queue").item(0);
new FairSchedulerXmlVerifications(customResourceTypes).verify(queue);
});
}
private void incrementUsedResourcesOnQueue(final FSLeafQueue queue,
final long value) {
try {
Method incUsedResourceMethod = queue.getClass().getSuperclass()
.getDeclaredMethod("incUsedResource", Resource.class);
incUsedResourceMethod.setAccessible(true);
Map<String, Long> customResources =
CustomResourceTypesConfigurationProvider.getCustomResourceTypes()
.stream()
.collect(Collectors.toMap(Function.identity(), v -> value));
incUsedResourceMethod.invoke(queue,
Resource.newInstance(20, 30, customResources));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

123
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
View File

@ -1,123 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringEqual;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
import static org.junit.Assert.*;
/**
* Contains all value verifications that are needed to verify {@link AppInfo}
* JSON objects.
*/
public final class AppInfoJsonVerifications {
private AppInfoJsonVerifications() {
//utility class
}
/**
* Tests whether {@link AppInfo} representation object contains the required
* values as per defined in the specified app parameter.
* @param app an RMApp instance that contains the required values
* to test against.
*/
public static void verify(JSONObject info, RMApp app) throws JSONException {
checkStringMatch("id", app.getApplicationId().toString(),
info.getString("id"));
checkStringMatch("user", app.getUser(), info.getString("user"));
checkStringMatch("name", app.getName(), info.getString("name"));
checkStringMatch("applicationType", app.getApplicationType(),
info.getString("applicationType"));
checkStringMatch("queue", app.getQueue(), info.getString("queue"));
assertEquals("priority doesn't match", 0, info.getInt("priority"));
checkStringMatch("state", app.getState().toString(),
info.getString("state"));
checkStringMatch("finalStatus", app.getFinalApplicationStatus().toString(),
info.getString("finalStatus"));
assertEquals("progress doesn't match", 0,
(float) info.getDouble("progress"), 0.0);
if ("UNASSIGNED".equals(info.getString("trackingUI"))) {
checkStringMatch("trackingUI", "UNASSIGNED",
info.getString("trackingUI"));
}
checkStringEqual("diagnostics", app.getDiagnostics().toString(),
info.getString("diagnostics"));
assertEquals("clusterId doesn't match",
ResourceManager.getClusterTimeStamp(), info.getLong("clusterId"));
assertEquals("startedTime doesn't match", app.getStartTime(),
info.getLong("startedTime"));
assertEquals("finishedTime doesn't match", app.getFinishTime(),
info.getLong("finishedTime"));
assertTrue("elapsed time not greater than 0",
info.getLong("elapsedTime") > 0);
checkStringMatch("amHostHttpAddress",
app.getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(),
info.getString("amHostHttpAddress"));
assertTrue("amContainerLogs doesn't match",
info.getString("amContainerLogs").startsWith("http://"));
assertTrue("amContainerLogs doesn't contain user info",
info.getString("amContainerLogs").endsWith("/" + app.getUser()));
assertEquals("allocatedMB doesn't match", 1024, info.getInt("allocatedMB"));
assertEquals("allocatedVCores doesn't match", 1,
info.getInt("allocatedVCores"));
assertEquals("queueUsagePerc doesn't match", 50.0f,
(float) info.getDouble("queueUsagePercentage"), 0.01f);
assertEquals("clusterUsagePerc doesn't match", 50.0f,
(float) info.getDouble("clusterUsagePercentage"), 0.01f);
assertEquals("numContainers doesn't match", 1,
info.getInt("runningContainers"));
assertNotNull("preemptedResourceSecondsMap should not be null",
info.getJSONObject("preemptedResourceSecondsMap"));
assertEquals("preemptedResourceMB doesn't match",
app.getRMAppMetrics().getResourcePreempted().getMemorySize(),
info.getInt("preemptedResourceMB"));
assertEquals("preemptedResourceVCores doesn't match",
app.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
info.getInt("preemptedResourceVCores"));
assertEquals("numNonAMContainerPreempted doesn't match",
app.getRMAppMetrics().getNumNonAMContainersPreempted(),
info.getInt("numNonAMContainerPreempted"));
assertEquals("numAMContainerPreempted doesn't match",
app.getRMAppMetrics().getNumAMContainersPreempted(),
info.getInt("numAMContainerPreempted"));
assertEquals("Log aggregation Status doesn't match",
app.getLogAggregationStatusForAppReport().toString(),
info.getString("logAggregationStatus"));
assertEquals("unmanagedApplication doesn't match",
app.getApplicationSubmissionContext().getUnmanagedAM(),
info.getBoolean("unmanagedApplication"));
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
assertEquals("appNodeLabelExpression doesn't match",
app.getApplicationSubmissionContext().getNodeLabelExpression(),
info.getString("appNodeLabelExpression"));
}
assertEquals("amNodeLabelExpression doesn't match",
app.getAMResourceRequests().get(0).getNodeLabelExpression(),
info.getString("amNodeLabelExpression"));
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
info.getString("amRPCAddress"));
}
}

132
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
View File

@ -1,132 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.w3c.dom.Element;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* Contains all value verifications that are needed to verify {@link AppInfo}
* XML documents.
*/
public final class AppInfoXmlVerifications {
private AppInfoXmlVerifications() {
//utility class
}
/**
* Tests whether {@link AppInfo} representation object contains the required
* values as per defined in the specified app parameter.
* @param info
* @param app an RMApp instance that contains the required values
*/
public static void verify(Element info, RMApp app) {
checkStringMatch("id", app.getApplicationId()
.toString(), getXmlString(info, "id"));
checkStringMatch("user", app.getUser(),
getXmlString(info, "user"));
checkStringMatch("name", app.getName(),
getXmlString(info, "name"));
checkStringMatch("applicationType",
app.getApplicationType(), getXmlString(info, "applicationType"));
checkStringMatch("queue", app.getQueue(),
getXmlString(info, "queue"));
assertEquals("priority doesn't match", 0, getXmlInt(info, "priority"));
checkStringMatch("state", app.getState().toString(),
getXmlString(info, "state"));
checkStringMatch("finalStatus", app
.getFinalApplicationStatus().toString(),
getXmlString(info, "finalStatus"));
assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"),
0.0);
if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) {
checkStringMatch("trackingUI", "UNASSIGNED",
getXmlString(info, "trackingUI"));
}
WebServicesTestUtils.checkStringEqual("diagnostics",
app.getDiagnostics().toString(), getXmlString(info, "diagnostics"));
assertEquals("clusterId doesn't match",
ResourceManager.getClusterTimeStamp(),
getXmlLong(info, "clusterId"));
assertEquals("startedTime doesn't match", app.getStartTime(),
getXmlLong(info, "startedTime"));
assertEquals("finishedTime doesn't match", app.getFinishTime(),
getXmlLong(info, "finishedTime"));
assertTrue("elapsed time not greater than 0",
getXmlLong(info, "elapsedTime") > 0);
checkStringMatch("amHostHttpAddress", app
.getCurrentAppAttempt().getMasterContainer()
.getNodeHttpAddress(),
getXmlString(info, "amHostHttpAddress"));
assertTrue("amContainerLogs doesn't match",
getXmlString(info, "amContainerLogs").startsWith("http://"));
assertTrue("amContainerLogs doesn't contain user info",
getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser()));
assertEquals("allocatedMB doesn't match", 1024,
getXmlInt(info, "allocatedMB"));
assertEquals("allocatedVCores doesn't match", 1,
getXmlInt(info, "allocatedVCores"));
assertEquals("queueUsagePerc doesn't match", 50.0f,
getXmlFloat(info, "queueUsagePercentage"), 0.01f);
assertEquals("clusterUsagePerc doesn't match", 50.0f,
getXmlFloat(info, "clusterUsagePercentage"), 0.01f);
assertEquals("numContainers doesn't match", 1,
getXmlInt(info, "runningContainers"));
assertNotNull("preemptedResourceSecondsMap should not be null",
info.getElementsByTagName("preemptedResourceSecondsMap"));
assertEquals("preemptedResourceMB doesn't match", app
.getRMAppMetrics().getResourcePreempted().getMemorySize(),
getXmlInt(info, "preemptedResourceMB"));
assertEquals("preemptedResourceVCores doesn't match", app
.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
getXmlInt(info, "preemptedResourceVCores"));
assertEquals("numNonAMContainerPreempted doesn't match", app
.getRMAppMetrics().getNumNonAMContainersPreempted(),
getXmlInt(info, "numNonAMContainerPreempted"));
assertEquals("numAMContainerPreempted doesn't match", app
.getRMAppMetrics().getNumAMContainersPreempted(),
getXmlInt(info, "numAMContainerPreempted"));
assertEquals("Log aggregation Status doesn't match", app
.getLogAggregationStatusForAppReport().toString(),
getXmlString(info, "logAggregationStatus"));
assertEquals("unmanagedApplication doesn't match", app
.getApplicationSubmissionContext().getUnmanagedAM(),
getXmlBoolean(info, "unmanagedApplication"));
assertEquals("unmanagedApplication doesn't match",
app.getApplicationSubmissionContext().getNodeLabelExpression(),
getXmlString(info, "appNodeLabelExpression"));
assertEquals("unmanagedApplication doesn't match",
app.getAMResourceRequests().get(0).getNodeLabelExpression(),
getXmlString(info, "amNodeLabelExpression"));
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
getXmlString(info, "amRPCAddress"));
}
}

57
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
/**
* This class is merely a wrapper for {@link ClientResponse}. Given that the
* entity input stream of {@link ClientResponse} can be read only once by
* default and for some tests it is convenient to read the input stream many
* times, this class hides the details of how to do that and prevents
* unnecessary code duplication in tests.
*/
public class BufferedClientResponse {
private ClientResponse response;
public BufferedClientResponse(ClientResponse response) {
response.bufferEntity();
this.response = response;
}
public <T> T getEntity(Class<T> clazz)
throws ClientHandlerException, UniformInterfaceException {
try {
response.getEntityInputStream().reset();
} catch (IOException e) {
throw new RuntimeException(e);
}
return response.getEntity(clazz);
}
public MediaType getType() {
return response.getType();
}
}

77
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
View File

@ -1,77 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.http.JettyUtils;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.util.function.Consumer;
import static org.junit.Assert.*;
/**
* This class hides the implementation details of how to verify the structure of
* JSON responses. Tests should only provide the path of the
* {@link WebResource}, the response from the resource and
* the verifier Consumer to
* {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of
* {@link JSONObject} will be passed to that consumer to be able to
* verify the response.
*/
public class JsonCustomResourceTypeTestcase {
private static final Logger LOG =
LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class);
private final WebResource path;
private final BufferedClientResponse response;
private final JSONObject parsedResponse;
public JsonCustomResourceTypeTestcase(WebResource path,
BufferedClientResponse response) {
this.path = path;
this.response = response;
this.parsedResponse = response.getEntity(JSONObject.class);
}
public void verify(Consumer<JSONObject> verifier) {
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
logResponse();
String responseStr = response.getEntity(String.class);
if (responseStr == null || responseStr.isEmpty()) {
throw new IllegalStateException("Response is null or empty!");
}
verifier.accept(parsedResponse);
}
private void logResponse() {
String responseStr = response.getEntity(String.class);
LOG.info("Raw response from service URL {}: {}", path.toString(),
responseStr);
LOG.info("Parsed response from service URL {}: {}", path.toString(),
parsedResponse);
}
}

252
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
View File

@ -1,252 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
import java.util.Map;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* Performs value verifications on
* {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
* objects against the values of {@link ResourceRequest}. With the help of the
* {@link Builder}, users can also make verifications of the custom resource
* types and its values.
*/
public class ResourceRequestsJsonVerifications {
private final ResourceRequest resourceRequest;
private final JSONObject requestInfo;
private final Map<String, Long> customResourceTypes;
private final List<String> expectedCustomResourceTypes;
ResourceRequestsJsonVerifications(Builder builder) {
this.resourceRequest = builder.resourceRequest;
this.requestInfo = builder.requestInfo;
this.customResourceTypes = builder.customResourceTypes;
this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
}
public static void verify(JSONObject requestInfo, ResourceRequest rr)
throws JSONException {
createDefaultBuilder(requestInfo, rr).build().verify();
}
public static void verifyWithCustomResourceTypes(JSONObject requestInfo,
ResourceRequest resourceRequest, List<String> expectedResourceTypes)
throws JSONException {
createDefaultBuilder(requestInfo, resourceRequest)
.withExpectedCustomResourceTypes(expectedResourceTypes)
.withCustomResourceTypes(
extractActualCustomResourceTypes(requestInfo, expectedResourceTypes))
.build().verify();
}
private static Builder createDefaultBuilder(JSONObject requestInfo,
ResourceRequest resourceRequest) {
return new ResourceRequestsJsonVerifications.Builder()
.withRequest(resourceRequest)
.withRequestInfoJson(requestInfo);
}
private static Map<String, Long> extractActualCustomResourceTypes(
JSONObject requestInfo, List<String> expectedResourceTypes)
throws JSONException {
JSONObject capability = requestInfo.getJSONObject("capability");
Map<String, Long> resourceAndValue =
extractCustomResorceTypeValues(capability, expectedResourceTypes);
Map.Entry<String, Long> resourceEntry =
resourceAndValue.entrySet().iterator().next();
assertTrue(
"Found resource type: " + resourceEntry.getKey()
+ " is not in expected resource types: " + expectedResourceTypes,
expectedResourceTypes.contains(resourceEntry.getKey()));
return resourceAndValue;
}
private static Map<String, Long> extractCustomResorceTypeValues(
JSONObject capability, List<String> expectedResourceTypes)
throws JSONException {
assertTrue(
"resourceCategory does not have resourceInformations: " + capability,
capability.has("resourceInformations"));
JSONObject resourceInformations =
capability.getJSONObject("resourceInformations");
assertTrue(
"resourceInformations does not have resourceInformation object: "
+ resourceInformations,
resourceInformations.has("resourceInformation"));
JSONArray customResources =
resourceInformations.getJSONArray("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
expectedResourceTypes.size(), customResources.length() - 2);
Map<String, Long> resourceValues = Maps.newHashMap();
for (int i = 0; i < customResources.length(); i++) {
JSONObject customResource = customResources.getJSONObject(i);
assertTrue("Resource type does not have name field: " + customResource,
customResource.has("name"));
assertTrue("Resource type does not have name resourceType field: "
+ customResource, customResource.has("resourceType"));
assertTrue(
"Resource type does not have name units field: " + customResource,
customResource.has("units"));
assertTrue(
"Resource type does not have name value field: " + customResource,
customResource.has("value"));
String name = customResource.getString("name");
String unit = customResource.getString("units");
String resourceType = customResource.getString("resourceType");
Long value = customResource.getLong("value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
expectedResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Custom resource value " + value + " is null!", value);
resourceValues.put(name, value);
}
return resourceValues;
}
private void verify() throws JSONException {
assertEquals("nodeLabelExpression doesn't match",
resourceRequest.getNodeLabelExpression(),
requestInfo.getString("nodeLabelExpression"));
assertEquals("numContainers doesn't match",
resourceRequest.getNumContainers(),
requestInfo.getInt("numContainers"));
assertEquals("relaxLocality doesn't match",
resourceRequest.getRelaxLocality(),
requestInfo.getBoolean("relaxLocality"));
assertEquals("priority does not match",
resourceRequest.getPriority().getPriority(),
requestInfo.getInt("priority"));
assertEquals("resourceName does not match",
resourceRequest.getResourceName(),
requestInfo.getString("resourceName"));
assertEquals("memory does not match",
resourceRequest.getCapability().getMemorySize(),
requestInfo.getJSONObject("capability").getLong("memory"));
assertEquals("vCores does not match",
resourceRequest.getCapability().getVirtualCores(),
requestInfo.getJSONObject("capability").getLong("vCores"));
verifyAtLeastOneCustomResourceIsSerialized();
JSONObject executionTypeRequest =
requestInfo.getJSONObject("executionTypeRequest");
assertEquals("executionType does not match",
resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
executionTypeRequest.getString("executionType"));
assertEquals("enforceExecutionType does not match",
resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
executionTypeRequest.getBoolean("enforceExecutionType"));
}
/**
* JSON serialization produces "invalid JSON" by default as maps are
* serialized like this:
* "customResources":{"entry":{"key":"customResource-1","value":"0"}}
* If the map has multiple keys then multiple entries will be serialized.
* Our json parser in tests cannot handle duplicates therefore only one
* custom resource will be in the parsed json. See:
* https://issues.apache.org/jira/browse/YARN-7505
*/
private void verifyAtLeastOneCustomResourceIsSerialized() {
boolean resourceFound = false;
for (String expectedCustomResourceType : expectedCustomResourceTypes) {
if (customResourceTypes.containsKey(expectedCustomResourceType)) {
resourceFound = true;
Long resourceValue =
customResourceTypes.get(expectedCustomResourceType);
assertNotNull("Resource value should not be null!", resourceValue);
}
}
assertTrue("No custom resource type can be found in the response!",
resourceFound);
}
/**
* Builder class for {@link ResourceRequestsJsonVerifications}.
*/
public static final class Builder {
private List<String> expectedCustomResourceTypes = Lists.newArrayList();
private Map<String, Long> customResourceTypes;
private ResourceRequest resourceRequest;
private JSONObject requestInfo;
Builder() {
}
public static Builder create() {
return new Builder();
}
Builder withExpectedCustomResourceTypes(
List<String> expectedCustomResourceTypes) {
this.expectedCustomResourceTypes = expectedCustomResourceTypes;
return this;
}
Builder withCustomResourceTypes(
Map<String, Long> customResourceTypes) {
this.customResourceTypes = customResourceTypes;
return this;
}
Builder withRequest(ResourceRequest resourceRequest) {
this.resourceRequest = resourceRequest;
return this;
}
Builder withRequestInfoJson(JSONObject requestInfo) {
this.requestInfo = requestInfo;
return this;
}
public ResourceRequestsJsonVerifications build() {
return new ResourceRequestsJsonVerifications(this);
}
}
}

215
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
View File

@ -1,215 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static junit.framework.TestCase.assertTrue;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* Performs value verifications on
* {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
* objects against the values of {@link ResourceRequest}. With the help of the
* {@link Builder}, users can also make verifications of the custom resource
* types and its values.
*/
public class ResourceRequestsXmlVerifications {
private final ResourceRequest resourceRequest;
private final Element requestInfo;
private final Map<String, Long> customResourceTypes;
private final List<String> expectedCustomResourceTypes;
ResourceRequestsXmlVerifications(Builder builder) {
this.resourceRequest = builder.resourceRequest;
this.requestInfo = builder.requestInfo;
this.customResourceTypes = builder.customResourceTypes;
this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
}
public static void verifyWithCustomResourceTypes(Element requestInfo,
ResourceRequest resourceRequest, List<String> expectedResourceTypes) {
createDefaultBuilder(requestInfo, resourceRequest)
.withExpectedCustomResourceTypes(expectedResourceTypes)
.withCustomResourceTypes(extractActualCustomResourceType(requestInfo,
expectedResourceTypes))
.build().verify();
}
private static Builder createDefaultBuilder(Element requestInfo,
ResourceRequest resourceRequest) {
return new ResourceRequestsXmlVerifications.Builder()
.withRequest(resourceRequest).withRequestInfo(requestInfo);
}
private static Map<String, Long> extractActualCustomResourceType(
Element requestInfo, List<String> expectedResourceTypes) {
Element capability =
(Element) requestInfo.getElementsByTagName("capability").item(0);
return extractCustomResorceTypes(capability,
Sets.newHashSet(expectedResourceTypes));
}
private static Map<String, Long> extractCustomResorceTypes(Element capability,
Set<String> expectedResourceTypes) {
assertEquals(
toXml(capability) + " should have only one resourceInformations child!",
1, capability.getElementsByTagName("resourceInformations").getLength());
Element resourceInformations = (Element) capability
.getElementsByTagName("resourceInformations").item(0);
NodeList customResources =
resourceInformations.getElementsByTagName("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
expectedResourceTypes.size(), customResources.getLength() - 2);
Map<String, Long> resourceTypesAndValues = Maps.newHashMap();
for (int i = 0; i < customResources.getLength(); i++) {
Element customResource = (Element) customResources.item(i);
String name = getXmlString(customResource, "name");
String unit = getXmlString(customResource, "units");
String resourceType = getXmlString(customResource, "resourceType");
Long value = getXmlLong(customResource, "value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
expectedResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Resource value should not be null for resource type "
+ resourceType + ", listing xml contents: " + toXml(customResource),
value);
resourceTypesAndValues.put(name, value);
}
return resourceTypesAndValues;
}
private void verify() {
assertEquals("nodeLabelExpression doesn't match",
resourceRequest.getNodeLabelExpression(),
getXmlString(requestInfo, "nodeLabelExpression"));
assertEquals("numContainers doesn't match",
resourceRequest.getNumContainers(),
getXmlInt(requestInfo, "numContainers"));
assertEquals("relaxLocality doesn't match",
resourceRequest.getRelaxLocality(),
getXmlBoolean(requestInfo, "relaxLocality"));
assertEquals("priority does not match",
resourceRequest.getPriority().getPriority(),
getXmlInt(requestInfo, "priority"));
assertEquals("resourceName does not match",
resourceRequest.getResourceName(),
getXmlString(requestInfo, "resourceName"));
Element capability = (Element) requestInfo
.getElementsByTagName("capability").item(0);
assertEquals("memory does not match",
resourceRequest.getCapability().getMemorySize(),
getXmlLong(capability, "memory"));
assertEquals("vCores does not match",
resourceRequest.getCapability().getVirtualCores(),
getXmlLong(capability, "vCores"));
for (String expectedCustomResourceType : expectedCustomResourceTypes) {
assertTrue(
"Custom resource type " + expectedCustomResourceType
+ " cannot be found!",
customResourceTypes.containsKey(expectedCustomResourceType));
Long resourceValue = customResourceTypes.get(expectedCustomResourceType);
assertNotNull("Resource value should not be null!", resourceValue);
}
Element executionTypeRequest = (Element) requestInfo
.getElementsByTagName("executionTypeRequest").item(0);
assertEquals("executionType does not match",
resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
getXmlString(executionTypeRequest, "executionType"));
assertEquals("enforceExecutionType does not match",
resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
getXmlBoolean(executionTypeRequest, "enforceExecutionType"));
}
/**
* Builder class for {@link ResourceRequestsXmlVerifications}.
*/
public static final class Builder {
private List<String> expectedCustomResourceTypes = Lists.newArrayList();
private Map<String, Long> customResourceTypes;
private ResourceRequest resourceRequest;
private Element requestInfo;
Builder() {
}
public static Builder create() {
return new Builder();
}
Builder withExpectedCustomResourceTypes(
List<String> expectedCustomResourceTypes) {
this.expectedCustomResourceTypes = expectedCustomResourceTypes;
return this;
}
Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) {
this.customResourceTypes = customResourceTypes;
return this;
}
Builder withRequest(ResourceRequest resourceRequest) {
this.resourceRequest = resourceRequest;
return this;
}
Builder withRequestInfo(Element requestInfo) {
this.requestInfo = requestInfo;
return this;
}
public ResourceRequestsXmlVerifications build() {
return new ResourceRequestsXmlVerifications(this);
}
}
}

112
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
View File

@ -1,112 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.http.JettyUtils;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.function.Consumer;
import static org.junit.Assert.assertEquals;
/**
* This class hides the implementation details of how to verify the structure of
* XML responses. Tests should only provide the path of the
* {@link WebResource}, the response from the resource and
* the verifier Consumer to
* {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of
* {@link JSONObject} will be passed to that consumer to be able to
* verify the response.
*/
public class XmlCustomResourceTypeTestCase {
private static final Logger LOG =
LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class);
private WebResource path;
private BufferedClientResponse response;
private Document parsedResponse;
public XmlCustomResourceTypeTestCase(WebResource path,
BufferedClientResponse response) {
this.path = path;
this.response = response;
}
public void verify(Consumer<Document> verifier) {
assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
response.getType().toString());
parsedResponse = parseXml(response);
logResponse(parsedResponse);
verifier.accept(parsedResponse);
}
private Document parseXml(BufferedClientResponse response) {
try {
String xml = response.getEntity(String.class);
DocumentBuilder db =
DocumentBuilderFactory.newInstance().newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(xml));
return db.parse(is);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private void logResponse(Document doc) {
String responseStr = response.getEntity(String.class);
LOG.info("Raw response from service URL {}: {}", path.toString(),
responseStr);
LOG.info("Parsed response from service URL {}: {}", path.toString(),
toXml(doc));
}
public static String toXml(Node node) {
StringWriter writer;
try {
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.setOutputProperty(
"{http://xml.apache.org/xslt}indent" + "-amount", "2");
writer = new StringWriter();
transformer.transform(new DOMSource(node), new StreamResult(writer));
} catch (TransformerException e) {
throw new RuntimeException(e);
}
return writer.getBuffer().toString();
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
View File

@ -86,11 +86,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties:
* **minResources**: minimum resources the queue is entitled to, in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and its minimum. Note that it is possible for a queue that is below its minimum to not immediately get up to its minimum when an application is submitted to the queue, because already-running jobs may be using those resources.
* **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
* **maxResources**: maximum resources a queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxChildResources**: maximum resources an ad hoc child queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxRunningApps**: limit the number of apps from the queue to run at once