Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7451. Add missing tests to verify the presence of custom resources of RM apps and scheduler webservice endpoints (snemeth via rkanter)

This commit is contained in:
Robert Kanter 2018-07-05 10:54:19 -07:00
parent 1726247024
commit 99febe7fd5
21 changed files with 2020 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
View File

@ -41,8 +41,9 @@ public class SchedulerInfo {
protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes;
protected int maximumClusterPriority;
// JAXB needs this
public SchedulerInfo() {
} // JAXB needs this
}
public SchedulerInfo(final ResourceManager rm) {
ResourceScheduler rs = rm.getResourceScheduler();
@ -74,7 +75,10 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
return Arrays.toString(minAllocResource.getResource().getResources());
if (minAllocResource != null) {
return Arrays.toString(minAllocResource.getResource().getResources());
}
return null;
}
public int getMaxClusterLevelAppPriority() {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
View File

@ -48,6 +48,9 @@ import org.apache.log4j.spi.LoggingEvent;
import org.junit.Assert;
import org.junit.Test;
/**
* Tests fair scheduler configuration.
*/
public class TestFairSchedulerConfiguration {
private static final String A_CUSTOM_RESOURCE = "a-custom-resource";

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -53,11 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.*;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@ -76,11 +72,12 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.eclipse.jetty.server.Response;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@ -96,6 +93,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServices extends JerseyTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestRMWebServices.class);
private static MockRM rm;
@ -597,11 +596,13 @@ public class TestRMWebServices extends JerseyTestBase {
public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("incorrect number of elements in: " + json, 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 11, info.length());
LOG.debug("schedulerInfo: {}", info);
assertEquals("incorrect number of elements in: " + info, 11, info.length());
verifyClusterSchedulerFifoGeneric(info.getString("type"),
info.getString("qstate"), (float) info.getDouble("capacity"),

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -511,7 +511,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
.path("apps").queryParam("finalStatus",
FinalApplicationStatus.UNDEFINED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
@ -1804,7 +1805,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
int numAttempt = 1;
while (true) {
// fail the AM by sending CONTAINER_FINISHED event without registering.
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
if (numAttempt == maxAppAttempts) {
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);

242
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java Normal file
View File

@ -0,0 +1,242 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import static org.junit.Assert.assertEquals;
/**
* This test verifies that custom resource types are correctly serialized to XML
* and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
*/
public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
private static MockRM rm;
private static final int CONTAINER_MB = 1024;
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
ResourceScheduler.class);
initResourceTypes(conf);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
private void initResourceTypes(Configuration conf) {
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
CustomResourceTypesConfigurationProvider.class.getName());
ResourceUtils.resetResourceTypes(conf);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
createInjectorForWebServletModule();
}
private void createInjectorForWebServletModule() {
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesAppsCustomResourceTypes() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testRunningAppXml() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
am1.allocate("*", 2048, 1, new ArrayList<>());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
XmlCustomResourceTypeTestCase testCase =
new XmlCustomResourceTypeTestCase(path,
new BufferedClientResponse(response));
testCase.verify(document -> {
NodeList apps = document.getElementsByTagName("apps");
assertEquals("incorrect number of apps elements", 1, apps.getLength());
NodeList appArray = ((Element)(apps.item(0)))
.getElementsByTagName("app");
assertEquals("incorrect number of app elements", 1, appArray.getLength());
verifyAppsXML(appArray, app1);
});
rm.stop();
}
@Test
public void testRunningAppJson() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
am1.allocate("*", 2048, 1, new ArrayList<>());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
JsonCustomResourceTypeTestcase testCase =
new JsonCustomResourceTypeTestcase(path,
new BufferedClientResponse(response));
testCase.verify(json -> {
try {
assertEquals("incorrect number of apps elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of app elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect count of app", 1, array.length());
verifyAppInfoJson(array.getJSONObject(0), app1);
} catch (JSONException e) {
throw new RuntimeException(e);
}
});
rm.stop();
}
private void verifyAppsXML(NodeList appArray, RMApp app) {
for (int i = 0; i < appArray.getLength(); i++) {
Element element = (Element) appArray.item(i);
AppInfoXmlVerifications.verify(element, app);
NodeList resourceRequests =
element.getElementsByTagName("resourceRequests");
assertEquals(1, resourceRequests.getLength());
Node resourceRequest = resourceRequests.item(0);
ResourceRequest rr =
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
.getApplicationAttempt(
app.getCurrentAppAttempt().getAppAttemptId())
.getAppSchedulingInfo().getAllResourceRequests().get(0);
ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
(Element) resourceRequest, rr,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
}
private void verifyAppInfoJson(JSONObject info, RMApp app) throws
JSONException {
int expectedNumberOfElements = getExpectedNumberOfElements(app);
assertEquals("incorrect number of elements", expectedNumberOfElements,
info.length());
AppInfoJsonVerifications.verify(info, app);
JSONArray resourceRequests = info.getJSONArray("resourceRequests");
JSONObject requestInfo = resourceRequests.getJSONObject(0);
ResourceRequest rr =
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
.getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
.getAppSchedulingInfo().getAllResourceRequests().get(0);
ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
requestInfo, rr,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
private int getExpectedNumberOfElements(RMApp app) {
int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
expectedNumberOfElements++;
}
if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
expectedNumberOfElements++;
}
if (AppInfo
.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
expectedNumberOfElements++;
}
return expectedNumberOfElements;
}
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -317,30 +317,34 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("incorrect number of elements in: " + json, 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 8, info.length());
assertEquals("incorrect number of elements in: " + info, 8, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements", 3, health.length());
assertEquals("incorrect number of elements in: " + health, 3,
health.length());
JSONArray operationsInfo = health.getJSONArray("operationsInfo");
assertEquals("incorrect number of elements", 4, operationsInfo.length());
assertEquals("incorrect number of elements in: " + health, 4,
operationsInfo.length());
JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
assertEquals("incorrect number of elements", 3, lastRunDetails.length());
assertEquals("incorrect number of elements in: " + health, 3,
lastRunDetails.length());
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
assertEquals("incorrect number of elements", 2, arr.length());
assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
// test subqueues
for (int i = 0; i < arr.length(); i++) {
JSONObject obj = arr.getJSONObject(i);
String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName");
String q = CapacitySchedulerConfiguration.ROOT + "." +
obj.getString("queueName");
verifySubQueue(obj, q, 100, 100);
}
}
@ -514,7 +518,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
for (int j=0; j<users.getLength(); ++j) {
Node user = users.item(j);
String username = getChildNodeByName(user, "username")
.getTextContent();
.getTextContent();
assertTrue(username.equals("user1") || username.equals("user2"));
//Should be a parsable integer
Integer.parseInt(getChildNodeByName(getChildNodeByName(user,

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
View File

@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
@ -59,6 +61,8 @@ import static org.junit.Assert.assertNull;
* Test scheduler configuration mutation via REST API.
*/
public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
private static final Logger LOG = LoggerFactory
.getLogger(TestRMWebServicesConfigurationMutation.class);
private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@ -396,6 +400,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
LOG.debug("Response headers: " + response.getHeaders());
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
assertEquals(0.2f, newCSConf

89
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -16,13 +16,14 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import javax.ws.rs.core.MediaType;
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@ -30,6 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@ -38,14 +42,14 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import javax.ws.rs.core.MediaType;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* Tests RM Webservices fair scheduler resources.
*/
public class TestRMWebServicesFairScheduler extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
@ -58,7 +62,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
@ -66,32 +70,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
static {
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesFairScheduler() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testClusterScheduler() throws JSONException, Exception {
public void testClusterScheduler() throws JSONException {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
@ -99,11 +103,11 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
@Test
public void testClusterSchedulerSlash() throws JSONException, Exception {
public void testClusterSchedulerSlash() throws JSONException {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler/").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler/")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
@ -111,40 +115,39 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
@Test
public void testClusterSchedulerWithSubQueues() throws JSONException,
Exception {
FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
public void testClusterSchedulerWithSubQueues()
throws JSONException {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueue
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
ClientResponse response =
r.path("ws").path("v1").path("cluster").path("scheduler")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue")
.getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
.getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
.getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
try {
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
"if child queue is empty.");
fail("FairSchedulerQueueInfo should omit field 'childQueues'"
+ "if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
}
}
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
private void verifyClusterScheduler(JSONObject json) throws JSONException {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
View File

@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities
if (object.getClass() == JSONObject.class) {
assertEquals("Number of allocations is wrong", 1, realValue);
} else if (object.getClass() == JSONArray.class) {
assertEquals("Number of allocations is wrong",
assertEquals("Number of allocations is wrong in: " + object,
((JSONArray) object).length(), realValue);
}
}

138
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static java.util.stream.Collectors.toList;
/**
* This class can generate an XML configuration file of custom resource types.
* See createInitialResourceTypes for the default values. All custom resource
* type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the
* getConfigurationInputStream method to get an InputStream of the XML. If you
* want to have different number of resources in your tests, please see usages
* of this class in this test class:
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}
*
*/
public class CustomResourceTypesConfigurationProvider
extends LocalConfigurationProvider {
private static class CustomResourceTypes {
private int count;
private String xml;
CustomResourceTypes(String xml, int count) {
this.xml = xml;
this.count = count;
}
public int getCount() {
return count;
}
public String getXml() {
return xml;
}
}
private static final String CUSTOM_RESOURCE_PREFIX = "customResource-";
private static CustomResourceTypes customResourceTypes =
createInitialResourceTypes();
private static CustomResourceTypes createInitialResourceTypes() {
return createCustomResourceTypes(2);
}
private static CustomResourceTypes createCustomResourceTypes(int count) {
List<String> resourceTypeNames = generateResourceTypeNames(count);
List<String> resourceUnitXmlElements = IntStream.range(0, count)
.boxed()
.map(i -> getResourceUnitsXml(resourceTypeNames.get(i)))
.collect(toList());
StringBuilder sb = new StringBuilder("<configuration>\n");
sb.append(getResourceTypesXml(resourceTypeNames));
for (String resourceUnitXml : resourceUnitXmlElements) {
sb.append(resourceUnitXml);
}
sb.append("</configuration>");
return new CustomResourceTypes(sb.toString(), count);
}
private static List<String> generateResourceTypeNames(int count) {
return IntStream.range(0, count)
.boxed()
.map(i -> CUSTOM_RESOURCE_PREFIX + i)
.collect(toList());
}
private static String getResourceUnitsXml(String resource) {
return "<property>\n" + "<name>yarn.resource-types." + resource
+ ".units</name>\n" + "<value>k</value>\n" + "</property>\n";
}
private static String getResourceTypesXml(List<String> resources) {
final String resourceTypes = makeCommaSeparatedString(resources);
return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>"
+ resourceTypes + "</value>\n" + "</property>\n";
}
private static String makeCommaSeparatedString(List<String> resources) {
return resources.stream().collect(Collectors.joining(","));
}
@Override
public InputStream getConfigurationInputStream(Configuration bootstrapConf,
String name) throws YarnException, IOException {
if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
return new ByteArrayInputStream(
customResourceTypes.getXml().getBytes());
} else {
return super.getConfigurationInputStream(bootstrapConf, name);
}
}
public static void reset() {
customResourceTypes = createInitialResourceTypes();
}
public static void setNumberOfResourceTypes(int count) {
customResourceTypes = createCustomResourceTypes(count);
}
public static List<String> getCustomResourceTypes() {
return generateResourceTypeNames(customResourceTypes.getCount());
}
}

139
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* This test helper class is primarily used by
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
*/
public class FairSchedulerJsonVerifications {
private static final Set<String> RESOURCE_FIELDS =
Sets.newHashSet("minResources", "amUsedResources", "amMaxResources",
"fairResources", "clusterResources", "reservedResources",
"maxResources", "usedResources", "steadyFairResources",
"demandResources");
private final Set<String> customResourceTypes;
FairSchedulerJsonVerifications(List<String> customResourceTypes) {
this.customResourceTypes = Sets.newHashSet(customResourceTypes);
}
public void verify(JSONObject jsonObject) {
try {
verifyResourcesContainDefaultResourceTypes(jsonObject, RESOURCE_FIELDS);
verifyResourcesContainCustomResourceTypes(jsonObject, RESOURCE_FIELDS);
} catch (JSONException e) {
throw new RuntimeException(e);
}
}
private void verifyResourcesContainDefaultResourceTypes(JSONObject queue,
Set<String> resourceCategories) throws JSONException {
for (String resourceCategory : resourceCategories) {
boolean hasResourceCategory = queue.has(resourceCategory);
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, hasResourceCategory);
verifyResourceContainsDefaultResourceTypes(
queue.getJSONObject(resourceCategory));
}
}
private void verifyResourceContainsDefaultResourceTypes(
JSONObject jsonObject) {
Object memory = jsonObject.opt("memory");
Object vCores = jsonObject.opt("vCores");
assertNotNull("Key 'memory' not found in: " + jsonObject, memory);
assertNotNull("Key 'vCores' not found in: " + jsonObject, vCores);
}
private void verifyResourcesContainCustomResourceTypes(JSONObject queue,
Set<String> resourceCategories) throws JSONException {
for (String resourceCategory : resourceCategories) {
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, queue.has(resourceCategory));
verifyResourceContainsAllCustomResourceTypes(
queue.getJSONObject(resourceCategory));
}
}
private void verifyResourceContainsAllCustomResourceTypes(
JSONObject resourceCategory) throws JSONException {
assertTrue("resourceCategory does not have resourceInformations: "
+ resourceCategory, resourceCategory.has("resourceInformations"));
JSONObject resourceInformations =
resourceCategory.getJSONObject("resourceInformations");
assertTrue(
"resourceInformations does not have resourceInformation object: "
+ resourceInformations,
resourceInformations.has("resourceInformation"));
JSONArray customResources =
resourceInformations.getJSONArray("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
customResourceTypes.size(), customResources.length() - 2);
for (int i = 0; i < customResources.length(); i++) {
JSONObject customResource = customResources.getJSONObject(i);
assertTrue("Resource type does not have name field: " + customResource,
customResource.has("name"));
assertTrue("Resource type does not have name resourceType field: "
+ customResource, customResource.has("resourceType"));
assertTrue(
"Resource type does not have name units field: " + customResource,
customResource.has("units"));
assertTrue(
"Resource type does not have name value field: " + customResource,
customResource.has("value"));
String name = customResource.getString("name");
String unit = customResource.getString("units");
String resourceType = customResource.getString("resourceType");
Long value = customResource.getLong("value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
customResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Custom resource value " + value + " is null!", value);
}
}
}

153
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.List;
import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* This test helper class is primarily used by
* {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
*/
public class FairSchedulerXmlVerifications {
private static final Set<String> RESOURCE_FIELDS = Sets.newHashSet(
"minResources", "amUsedResources", "amMaxResources", "fairResources",
"clusterResources", "reservedResources", "maxResources", "usedResources",
"steadyFairResources", "demandResources");
private final Set<String> customResourceTypes;
FairSchedulerXmlVerifications(List<String> customResourceTypes) {
this.customResourceTypes = Sets.newHashSet(customResourceTypes);
}
public void verify(Element element) {
verifyResourcesContainDefaultResourceTypes(element, RESOURCE_FIELDS);
verifyResourcesContainCustomResourceTypes(element, RESOURCE_FIELDS);
}
private void verifyResourcesContainDefaultResourceTypes(Element queue,
Set<String> resourceCategories) {
for (String resourceCategory : resourceCategories) {
boolean hasResourceCategory = hasChild(queue, resourceCategory);
assertTrue("Queue " + queue + " does not have resource category key: "
+ resourceCategory, hasResourceCategory);
verifyResourceContainsDefaultResourceTypes(
(Element) queue.getElementsByTagName(resourceCategory).item(0));
}
}
private void verifyResourceContainsDefaultResourceTypes(
Element element) {
Object memory = opt(element, "memory");
Object vCores = opt(element, "vCores");
assertNotNull("Key 'memory' not found in: " + element, memory);
assertNotNull("Key 'vCores' not found in: " + element, vCores);
}
private void verifyResourcesContainCustomResourceTypes(Element queue,
Set<String> resourceCategories) {
for (String resourceCategory : resourceCategories) {
assertTrue("Queue " + queue + " does not have key for resourceCategory: "
+ resourceCategory, hasChild(queue, resourceCategory));
verifyResourceContainsCustomResourceTypes(
(Element) queue.getElementsByTagName(resourceCategory).item(0));
}
}
private void verifyResourceContainsCustomResourceTypes(
Element resourceCategory) {
assertEquals(
toXml(resourceCategory)
+ " should have only one resourceInformations child!",
1, resourceCategory.getElementsByTagName("resourceInformations")
.getLength());
Element resourceInformations = (Element) resourceCategory
.getElementsByTagName("resourceInformations").item(0);
NodeList customResources =
resourceInformations.getElementsByTagName("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
customResourceTypes.size(), customResources.getLength() - 2);
for (int i = 0; i < customResources.getLength(); i++) {
Element customResource = (Element) customResources.item(i);
String name = getXmlString(customResource, "name");
String unit = getXmlString(customResource, "units");
String resourceType = getXmlString(customResource, "resourceType");
Long value = getXmlLong(customResource, "value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
customResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Resource value should not be null for resource type "
+ resourceType + ", listing xml contents: " + toXml(customResource),
value);
}
}
private Object opt(Node node, String child) {
NodeList nodes = getElementsByTagNameInternal(node, child);
if (nodes.getLength() > 0) {
return nodes.item(0);
}
return null;
}
private boolean hasChild(Node node, String child) {
return getElementsByTagNameInternal(node, child).getLength() > 0;
}
private NodeList getElementsByTagNameInternal(Node node, String child) {
if (node instanceof Element) {
return ((Element) node).getElementsByTagName(child);
} else if (node instanceof Document) {
return ((Document) node).getElementsByTagName(child);
} else {
throw new IllegalStateException("Unknown type of wrappedObject: " + node
+ ", type: " + node.getClass());
}
}
}

271
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java Normal file
View File

@ -0,0 +1,271 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.*;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Element;
import javax.ws.rs.core.MediaType;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
/**
* This class is to test response representations of queue resources,
* explicitly setting custom resource types. with the help of
* {@link CustomResourceTypesConfigurationProvider}
*/
public class TestRMWebServicesFairSchedulerCustomResourceTypes
extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
initResourceTypes(conf);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
private void initResourceTypes(YarnConfiguration conf) {
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
CustomResourceTypesConfigurationProvider.class.getName());
ResourceUtils.resetResourceTypes(conf);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
createInjectorForWebServletModule();
}
@After
public void tearDown() {
ResourceUtils.resetResourceTypes(new Configuration());
}
private void createInjectorForWebServletModule() {
GuiceServletConfig
.setInjector(Guice.createInjector(new WebServletModule()));
}
@After
public void teardown() {
CustomResourceTypesConfigurationProvider.reset();
}
public TestRMWebServicesFairSchedulerCustomResourceTypes() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Test
public void testClusterSchedulerWithCustomResourceTypesJson() {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
verifyJsonResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerWithCustomResourceTypesXml() {
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
verifyXmlResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerWithElevenCustomResourceTypesXml() {
CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
createInjectorForWebServletModule();
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
verifyXmlResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
@Test
public void testClusterSchedulerElevenWithCustomResourceTypesJson() {
CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
createInjectorForWebServletModule();
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueues
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
FSLeafQueue subqueue1 =
queueManager.getLeafQueue("root.q.subqueue1", false);
incrementUsedResourcesOnQueue(subqueue1, 33L);
WebResource path =
resource().path("ws").path("v1").path("cluster").path("scheduler");
ClientResponse response =
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
verifyJsonResponse(path, response,
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
}
private void verifyJsonResponse(WebResource path, ClientResponse response,
List<String> customResourceTypes) {
JsonCustomResourceTypeTestcase testCase =
new JsonCustomResourceTypeTestcase(path,
new BufferedClientResponse(response));
testCase.verify(json -> {
try {
JSONArray queues = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue");
// childQueueInfo consists of subqueue1 and subqueue2 info
assertEquals(2, queues.length());
JSONObject firstChildQueue = queues.getJSONObject(0);
new FairSchedulerJsonVerifications(customResourceTypes)
.verify(firstChildQueue);
} catch (JSONException e) {
throw new RuntimeException(e);
}
});
}
private void verifyXmlResponse(WebResource path, ClientResponse response,
List<String> customResourceTypes) {
XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase(
path, new BufferedClientResponse(response));
testCase.verify(xml -> {
Element scheduler =
(Element) xml.getElementsByTagName("scheduler").item(0);
Element schedulerInfo =
(Element) scheduler.getElementsByTagName("schedulerInfo").item(0);
Element rootQueue =
(Element) schedulerInfo.getElementsByTagName("rootQueue").item(0);
Element childQueues =
(Element) rootQueue.getElementsByTagName("childQueues").item(0);
Element queue =
(Element) childQueues.getElementsByTagName("queue").item(0);
new FairSchedulerXmlVerifications(customResourceTypes).verify(queue);
});
}
private void incrementUsedResourcesOnQueue(final FSLeafQueue queue,
final long value) {
try {
Method incUsedResourceMethod = queue.getClass().getSuperclass()
.getDeclaredMethod("incUsedResource", Resource.class);
incUsedResourceMethod.setAccessible(true);
Map<String, Long> customResources =
CustomResourceTypesConfigurationProvider.getCustomResourceTypes()
.stream()
.collect(Collectors.toMap(Function.identity(), v -> value));
incUsedResourceMethod.invoke(queue,
Resource.newInstance(20, 30, customResources));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

123
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringEqual;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
import static org.junit.Assert.*;
/**
* Contains all value verifications that are needed to verify {@link AppInfo}
* JSON objects.
*/
public final class AppInfoJsonVerifications {
private AppInfoJsonVerifications() {
//utility class
}
/**
* Tests whether {@link AppInfo} representation object contains the required
* values as per defined in the specified app parameter.
* @param app an RMApp instance that contains the required values
* to test against.
*/
public static void verify(JSONObject info, RMApp app) throws JSONException {
checkStringMatch("id", app.getApplicationId().toString(),
info.getString("id"));
checkStringMatch("user", app.getUser(), info.getString("user"));
checkStringMatch("name", app.getName(), info.getString("name"));
checkStringMatch("applicationType", app.getApplicationType(),
info.getString("applicationType"));
checkStringMatch("queue", app.getQueue(), info.getString("queue"));
assertEquals("priority doesn't match", 0, info.getInt("priority"));
checkStringMatch("state", app.getState().toString(),
info.getString("state"));
checkStringMatch("finalStatus", app.getFinalApplicationStatus().toString(),
info.getString("finalStatus"));
assertEquals("progress doesn't match", 0,
(float) info.getDouble("progress"), 0.0);
if ("UNASSIGNED".equals(info.getString("trackingUI"))) {
checkStringMatch("trackingUI", "UNASSIGNED",
info.getString("trackingUI"));
}
checkStringEqual("diagnostics", app.getDiagnostics().toString(),
info.getString("diagnostics"));
assertEquals("clusterId doesn't match",
ResourceManager.getClusterTimeStamp(), info.getLong("clusterId"));
assertEquals("startedTime doesn't match", app.getStartTime(),
info.getLong("startedTime"));
assertEquals("finishedTime doesn't match", app.getFinishTime(),
info.getLong("finishedTime"));
assertTrue("elapsed time not greater than 0",
info.getLong("elapsedTime") > 0);
checkStringMatch("amHostHttpAddress",
app.getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(),
info.getString("amHostHttpAddress"));
assertTrue("amContainerLogs doesn't match",
info.getString("amContainerLogs").startsWith("http://"));
assertTrue("amContainerLogs doesn't contain user info",
info.getString("amContainerLogs").endsWith("/" + app.getUser()));
assertEquals("allocatedMB doesn't match", 1024, info.getInt("allocatedMB"));
assertEquals("allocatedVCores doesn't match", 1,
info.getInt("allocatedVCores"));
assertEquals("queueUsagePerc doesn't match", 50.0f,
(float) info.getDouble("queueUsagePercentage"), 0.01f);
assertEquals("clusterUsagePerc doesn't match", 50.0f,
(float) info.getDouble("clusterUsagePercentage"), 0.01f);
assertEquals("numContainers doesn't match", 1,
info.getInt("runningContainers"));
assertNotNull("preemptedResourceSecondsMap should not be null",
info.getJSONObject("preemptedResourceSecondsMap"));
assertEquals("preemptedResourceMB doesn't match",
app.getRMAppMetrics().getResourcePreempted().getMemorySize(),
info.getInt("preemptedResourceMB"));
assertEquals("preemptedResourceVCores doesn't match",
app.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
info.getInt("preemptedResourceVCores"));
assertEquals("numNonAMContainerPreempted doesn't match",
app.getRMAppMetrics().getNumNonAMContainersPreempted(),
info.getInt("numNonAMContainerPreempted"));
assertEquals("numAMContainerPreempted doesn't match",
app.getRMAppMetrics().getNumAMContainersPreempted(),
info.getInt("numAMContainerPreempted"));
assertEquals("Log aggregation Status doesn't match",
app.getLogAggregationStatusForAppReport().toString(),
info.getString("logAggregationStatus"));
assertEquals("unmanagedApplication doesn't match",
app.getApplicationSubmissionContext().getUnmanagedAM(),
info.getBoolean("unmanagedApplication"));
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
assertEquals("appNodeLabelExpression doesn't match",
app.getApplicationSubmissionContext().getNodeLabelExpression(),
info.getString("appNodeLabelExpression"));
}
assertEquals("amNodeLabelExpression doesn't match",
app.getAMResourceRequests().get(0).getNodeLabelExpression(),
info.getString("amNodeLabelExpression"));
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
info.getString("amRPCAddress"));
}
}

132
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.w3c.dom.Element;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* Contains all value verifications that are needed to verify {@link AppInfo}
* XML documents.
*/
public final class AppInfoXmlVerifications {
private AppInfoXmlVerifications() {
//utility class
}
/**
* Tests whether {@link AppInfo} representation object contains the required
* values as per defined in the specified app parameter.
* @param info
* @param app an RMApp instance that contains the required values
*/
public static void verify(Element info, RMApp app) {
checkStringMatch("id", app.getApplicationId()
.toString(), getXmlString(info, "id"));
checkStringMatch("user", app.getUser(),
getXmlString(info, "user"));
checkStringMatch("name", app.getName(),
getXmlString(info, "name"));
checkStringMatch("applicationType",
app.getApplicationType(), getXmlString(info, "applicationType"));
checkStringMatch("queue", app.getQueue(),
getXmlString(info, "queue"));
assertEquals("priority doesn't match", 0, getXmlInt(info, "priority"));
checkStringMatch("state", app.getState().toString(),
getXmlString(info, "state"));
checkStringMatch("finalStatus", app
.getFinalApplicationStatus().toString(),
getXmlString(info, "finalStatus"));
assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"),
0.0);
if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) {
checkStringMatch("trackingUI", "UNASSIGNED",
getXmlString(info, "trackingUI"));
}
WebServicesTestUtils.checkStringEqual("diagnostics",
app.getDiagnostics().toString(), getXmlString(info, "diagnostics"));
assertEquals("clusterId doesn't match",
ResourceManager.getClusterTimeStamp(),
getXmlLong(info, "clusterId"));
assertEquals("startedTime doesn't match", app.getStartTime(),
getXmlLong(info, "startedTime"));
assertEquals("finishedTime doesn't match", app.getFinishTime(),
getXmlLong(info, "finishedTime"));
assertTrue("elapsed time not greater than 0",
getXmlLong(info, "elapsedTime") > 0);
checkStringMatch("amHostHttpAddress", app
.getCurrentAppAttempt().getMasterContainer()
.getNodeHttpAddress(),
getXmlString(info, "amHostHttpAddress"));
assertTrue("amContainerLogs doesn't match",
getXmlString(info, "amContainerLogs").startsWith("http://"));
assertTrue("amContainerLogs doesn't contain user info",
getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser()));
assertEquals("allocatedMB doesn't match", 1024,
getXmlInt(info, "allocatedMB"));
assertEquals("allocatedVCores doesn't match", 1,
getXmlInt(info, "allocatedVCores"));
assertEquals("queueUsagePerc doesn't match", 50.0f,
getXmlFloat(info, "queueUsagePercentage"), 0.01f);
assertEquals("clusterUsagePerc doesn't match", 50.0f,
getXmlFloat(info, "clusterUsagePercentage"), 0.01f);
assertEquals("numContainers doesn't match", 1,
getXmlInt(info, "runningContainers"));
assertNotNull("preemptedResourceSecondsMap should not be null",
info.getElementsByTagName("preemptedResourceSecondsMap"));
assertEquals("preemptedResourceMB doesn't match", app
.getRMAppMetrics().getResourcePreempted().getMemorySize(),
getXmlInt(info, "preemptedResourceMB"));
assertEquals("preemptedResourceVCores doesn't match", app
.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
getXmlInt(info, "preemptedResourceVCores"));
assertEquals("numNonAMContainerPreempted doesn't match", app
.getRMAppMetrics().getNumNonAMContainersPreempted(),
getXmlInt(info, "numNonAMContainerPreempted"));
assertEquals("numAMContainerPreempted doesn't match", app
.getRMAppMetrics().getNumAMContainersPreempted(),
getXmlInt(info, "numAMContainerPreempted"));
assertEquals("Log aggregation Status doesn't match", app
.getLogAggregationStatusForAppReport().toString(),
getXmlString(info, "logAggregationStatus"));
assertEquals("unmanagedApplication doesn't match", app
.getApplicationSubmissionContext().getUnmanagedAM(),
getXmlBoolean(info, "unmanagedApplication"));
assertEquals("unmanagedApplication doesn't match",
app.getApplicationSubmissionContext().getNodeLabelExpression(),
getXmlString(info, "appNodeLabelExpression"));
assertEquals("unmanagedApplication doesn't match",
app.getAMResourceRequests().get(0).getNodeLabelExpression(),
getXmlString(info, "amNodeLabelExpression"));
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
getXmlString(info, "amRPCAddress"));
}
}

57
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
/**
* This class is merely a wrapper for {@link ClientResponse}. Given that the
* entity input stream of {@link ClientResponse} can be read only once by
* default and for some tests it is convenient to read the input stream many
* times, this class hides the details of how to do that and prevents
* unnecessary code duplication in tests.
*/
public class BufferedClientResponse {
private ClientResponse response;
public BufferedClientResponse(ClientResponse response) {
response.bufferEntity();
this.response = response;
}
public <T> T getEntity(Class<T> clazz)
throws ClientHandlerException, UniformInterfaceException {
try {
response.getEntityInputStream().reset();
} catch (IOException e) {
throw new RuntimeException(e);
}
return response.getEntity(clazz);
}
public MediaType getType() {
return response.getType();
}
}

77
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java Normal file
View File

@ -0,0 +1,77 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.http.JettyUtils;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.util.function.Consumer;
import static org.junit.Assert.*;
/**
* This class hides the implementation details of how to verify the structure of
* JSON responses. Tests should only provide the path of the
* {@link WebResource}, the response from the resource and
* the verifier Consumer to
* {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of
* {@link JSONObject} will be passed to that consumer to be able to
* verify the response.
*/
public class JsonCustomResourceTypeTestcase {
private static final Logger LOG =
LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class);
private final WebResource path;
private final BufferedClientResponse response;
private final JSONObject parsedResponse;
public JsonCustomResourceTypeTestcase(WebResource path,
BufferedClientResponse response) {
this.path = path;
this.response = response;
this.parsedResponse = response.getEntity(JSONObject.class);
}
public void verify(Consumer<JSONObject> verifier) {
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
logResponse();
String responseStr = response.getEntity(String.class);
if (responseStr == null || responseStr.isEmpty()) {
throw new IllegalStateException("Response is null or empty!");
}
verifier.accept(parsedResponse);
}
private void logResponse() {
String responseStr = response.getEntity(String.class);
LOG.info("Raw response from service URL {}: {}", path.toString(),
responseStr);
LOG.info("Parsed response from service URL {}: {}", path.toString(),
parsedResponse);
}
}

252
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java Normal file
View File

@ -0,0 +1,252 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
import java.util.Map;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* Performs value verifications on
* {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
* objects against the values of {@link ResourceRequest}. With the help of the
* {@link Builder}, users can also make verifications of the custom resource
* types and its values.
*/
public class ResourceRequestsJsonVerifications {
private final ResourceRequest resourceRequest;
private final JSONObject requestInfo;
private final Map<String, Long> customResourceTypes;
private final List<String> expectedCustomResourceTypes;
ResourceRequestsJsonVerifications(Builder builder) {
this.resourceRequest = builder.resourceRequest;
this.requestInfo = builder.requestInfo;
this.customResourceTypes = builder.customResourceTypes;
this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
}
public static void verify(JSONObject requestInfo, ResourceRequest rr)
throws JSONException {
createDefaultBuilder(requestInfo, rr).build().verify();
}
public static void verifyWithCustomResourceTypes(JSONObject requestInfo,
ResourceRequest resourceRequest, List<String> expectedResourceTypes)
throws JSONException {
createDefaultBuilder(requestInfo, resourceRequest)
.withExpectedCustomResourceTypes(expectedResourceTypes)
.withCustomResourceTypes(
extractActualCustomResourceTypes(requestInfo, expectedResourceTypes))
.build().verify();
}
private static Builder createDefaultBuilder(JSONObject requestInfo,
ResourceRequest resourceRequest) {
return new ResourceRequestsJsonVerifications.Builder()
.withRequest(resourceRequest)
.withRequestInfoJson(requestInfo);
}
private static Map<String, Long> extractActualCustomResourceTypes(
JSONObject requestInfo, List<String> expectedResourceTypes)
throws JSONException {
JSONObject capability = requestInfo.getJSONObject("capability");
Map<String, Long> resourceAndValue =
extractCustomResorceTypeValues(capability, expectedResourceTypes);
Map.Entry<String, Long> resourceEntry =
resourceAndValue.entrySet().iterator().next();
assertTrue(
"Found resource type: " + resourceEntry.getKey()
+ " is not in expected resource types: " + expectedResourceTypes,
expectedResourceTypes.contains(resourceEntry.getKey()));
return resourceAndValue;
}
private static Map<String, Long> extractCustomResorceTypeValues(
JSONObject capability, List<String> expectedResourceTypes)
throws JSONException {
assertTrue(
"resourceCategory does not have resourceInformations: " + capability,
capability.has("resourceInformations"));
JSONObject resourceInformations =
capability.getJSONObject("resourceInformations");
assertTrue(
"resourceInformations does not have resourceInformation object: "
+ resourceInformations,
resourceInformations.has("resourceInformation"));
JSONArray customResources =
resourceInformations.getJSONArray("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
expectedResourceTypes.size(), customResources.length() - 2);
Map<String, Long> resourceValues = Maps.newHashMap();
for (int i = 0; i < customResources.length(); i++) {
JSONObject customResource = customResources.getJSONObject(i);
assertTrue("Resource type does not have name field: " + customResource,
customResource.has("name"));
assertTrue("Resource type does not have name resourceType field: "
+ customResource, customResource.has("resourceType"));
assertTrue(
"Resource type does not have name units field: " + customResource,
customResource.has("units"));
assertTrue(
"Resource type does not have name value field: " + customResource,
customResource.has("value"));
String name = customResource.getString("name");
String unit = customResource.getString("units");
String resourceType = customResource.getString("resourceType");
Long value = customResource.getLong("value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
expectedResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Custom resource value " + value + " is null!", value);
resourceValues.put(name, value);
}
return resourceValues;
}
private void verify() throws JSONException {
assertEquals("nodeLabelExpression doesn't match",
resourceRequest.getNodeLabelExpression(),
requestInfo.getString("nodeLabelExpression"));
assertEquals("numContainers doesn't match",
resourceRequest.getNumContainers(),
requestInfo.getInt("numContainers"));
assertEquals("relaxLocality doesn't match",
resourceRequest.getRelaxLocality(),
requestInfo.getBoolean("relaxLocality"));
assertEquals("priority does not match",
resourceRequest.getPriority().getPriority(),
requestInfo.getInt("priority"));
assertEquals("resourceName does not match",
resourceRequest.getResourceName(),
requestInfo.getString("resourceName"));
assertEquals("memory does not match",
resourceRequest.getCapability().getMemorySize(),
requestInfo.getJSONObject("capability").getLong("memory"));
assertEquals("vCores does not match",
resourceRequest.getCapability().getVirtualCores(),
requestInfo.getJSONObject("capability").getLong("vCores"));
verifyAtLeastOneCustomResourceIsSerialized();
JSONObject executionTypeRequest =
requestInfo.getJSONObject("executionTypeRequest");
assertEquals("executionType does not match",
resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
executionTypeRequest.getString("executionType"));
assertEquals("enforceExecutionType does not match",
resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
executionTypeRequest.getBoolean("enforceExecutionType"));
}
/**
* JSON serialization produces "invalid JSON" by default as maps are
* serialized like this:
* "customResources":{"entry":{"key":"customResource-1","value":"0"}}
* If the map has multiple keys then multiple entries will be serialized.
* Our json parser in tests cannot handle duplicates therefore only one
* custom resource will be in the parsed json. See:
* https://issues.apache.org/jira/browse/YARN-7505
*/
private void verifyAtLeastOneCustomResourceIsSerialized() {
boolean resourceFound = false;
for (String expectedCustomResourceType : expectedCustomResourceTypes) {
if (customResourceTypes.containsKey(expectedCustomResourceType)) {
resourceFound = true;
Long resourceValue =
customResourceTypes.get(expectedCustomResourceType);
assertNotNull("Resource value should not be null!", resourceValue);
}
}
assertTrue("No custom resource type can be found in the response!",
resourceFound);
}
/**
* Builder class for {@link ResourceRequestsJsonVerifications}.
*/
public static final class Builder {
private List<String> expectedCustomResourceTypes = Lists.newArrayList();
private Map<String, Long> customResourceTypes;
private ResourceRequest resourceRequest;
private JSONObject requestInfo;
Builder() {
}
public static Builder create() {
return new Builder();
}
Builder withExpectedCustomResourceTypes(
List<String> expectedCustomResourceTypes) {
this.expectedCustomResourceTypes = expectedCustomResourceTypes;
return this;
}
Builder withCustomResourceTypes(
Map<String, Long> customResourceTypes) {
this.customResourceTypes = customResourceTypes;
return this;
}
Builder withRequest(ResourceRequest resourceRequest) {
this.resourceRequest = resourceRequest;
return this;
}
Builder withRequestInfoJson(JSONObject requestInfo) {
this.requestInfo = requestInfo;
return this;
}
public ResourceRequestsJsonVerifications build() {
return new ResourceRequestsJsonVerifications(this);
}
}
}

215
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java Normal file
View File

@ -0,0 +1,215 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static junit.framework.TestCase.assertTrue;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
* Performs value verifications on
* {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
* objects against the values of {@link ResourceRequest}. With the help of the
* {@link Builder}, users can also make verifications of the custom resource
* types and its values.
*/
public class ResourceRequestsXmlVerifications {
private final ResourceRequest resourceRequest;
private final Element requestInfo;
private final Map<String, Long> customResourceTypes;
private final List<String> expectedCustomResourceTypes;
ResourceRequestsXmlVerifications(Builder builder) {
this.resourceRequest = builder.resourceRequest;
this.requestInfo = builder.requestInfo;
this.customResourceTypes = builder.customResourceTypes;
this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
}
public static void verifyWithCustomResourceTypes(Element requestInfo,
ResourceRequest resourceRequest, List<String> expectedResourceTypes) {
createDefaultBuilder(requestInfo, resourceRequest)
.withExpectedCustomResourceTypes(expectedResourceTypes)
.withCustomResourceTypes(extractActualCustomResourceType(requestInfo,
expectedResourceTypes))
.build().verify();
}
private static Builder createDefaultBuilder(Element requestInfo,
ResourceRequest resourceRequest) {
return new ResourceRequestsXmlVerifications.Builder()
.withRequest(resourceRequest).withRequestInfo(requestInfo);
}
private static Map<String, Long> extractActualCustomResourceType(
Element requestInfo, List<String> expectedResourceTypes) {
Element capability =
(Element) requestInfo.getElementsByTagName("capability").item(0);
return extractCustomResorceTypes(capability,
Sets.newHashSet(expectedResourceTypes));
}
private static Map<String, Long> extractCustomResorceTypes(Element capability,
Set<String> expectedResourceTypes) {
assertEquals(
toXml(capability) + " should have only one resourceInformations child!",
1, capability.getElementsByTagName("resourceInformations").getLength());
Element resourceInformations = (Element) capability
.getElementsByTagName("resourceInformations").item(0);
NodeList customResources =
resourceInformations.getElementsByTagName("resourceInformation");
// customResources will include vcores / memory as well
assertEquals(
"Different number of custom resource types found than expected",
expectedResourceTypes.size(), customResources.getLength() - 2);
Map<String, Long> resourceTypesAndValues = Maps.newHashMap();
for (int i = 0; i < customResources.getLength(); i++) {
Element customResource = (Element) customResources.item(i);
String name = getXmlString(customResource, "name");
String unit = getXmlString(customResource, "units");
String resourceType = getXmlString(customResource, "resourceType");
Long value = getXmlLong(customResource, "value");
if (ResourceInformation.MEMORY_URI.equals(name)
|| ResourceInformation.VCORES_URI.equals(name)) {
continue;
}
assertTrue("Custom resource type " + name + " not found",
expectedResourceTypes.contains(name));
assertEquals("k", unit);
assertEquals(ResourceTypes.COUNTABLE,
ResourceTypes.valueOf(resourceType));
assertNotNull("Resource value should not be null for resource type "
+ resourceType + ", listing xml contents: " + toXml(customResource),
value);
resourceTypesAndValues.put(name, value);
}
return resourceTypesAndValues;
}
private void verify() {
assertEquals("nodeLabelExpression doesn't match",
resourceRequest.getNodeLabelExpression(),
getXmlString(requestInfo, "nodeLabelExpression"));
assertEquals("numContainers doesn't match",
resourceRequest.getNumContainers(),
getXmlInt(requestInfo, "numContainers"));
assertEquals("relaxLocality doesn't match",
resourceRequest.getRelaxLocality(),
getXmlBoolean(requestInfo, "relaxLocality"));
assertEquals("priority does not match",
resourceRequest.getPriority().getPriority(),
getXmlInt(requestInfo, "priority"));
assertEquals("resourceName does not match",
resourceRequest.getResourceName(),
getXmlString(requestInfo, "resourceName"));
Element capability = (Element) requestInfo
.getElementsByTagName("capability").item(0);
assertEquals("memory does not match",
resourceRequest.getCapability().getMemorySize(),
getXmlLong(capability, "memory"));
assertEquals("vCores does not match",
resourceRequest.getCapability().getVirtualCores(),
getXmlLong(capability, "vCores"));
for (String expectedCustomResourceType : expectedCustomResourceTypes) {
assertTrue(
"Custom resource type " + expectedCustomResourceType
+ " cannot be found!",
customResourceTypes.containsKey(expectedCustomResourceType));
Long resourceValue = customResourceTypes.get(expectedCustomResourceType);
assertNotNull("Resource value should not be null!", resourceValue);
}
Element executionTypeRequest = (Element) requestInfo
.getElementsByTagName("executionTypeRequest").item(0);
assertEquals("executionType does not match",
resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
getXmlString(executionTypeRequest, "executionType"));
assertEquals("enforceExecutionType does not match",
resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
getXmlBoolean(executionTypeRequest, "enforceExecutionType"));
}
/**
* Builder class for {@link ResourceRequestsXmlVerifications}.
*/
public static final class Builder {
private List<String> expectedCustomResourceTypes = Lists.newArrayList();
private Map<String, Long> customResourceTypes;
private ResourceRequest resourceRequest;
private Element requestInfo;
Builder() {
}
public static Builder create() {
return new Builder();
}
Builder withExpectedCustomResourceTypes(
List<String> expectedCustomResourceTypes) {
this.expectedCustomResourceTypes = expectedCustomResourceTypes;
return this;
}
Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) {
this.customResourceTypes = customResourceTypes;
return this;
}
Builder withRequest(ResourceRequest resourceRequest) {
this.resourceRequest = resourceRequest;
return this;
}
Builder withRequestInfo(Element requestInfo) {
this.requestInfo = requestInfo;
return this;
}
public ResourceRequestsXmlVerifications build() {
return new ResourceRequestsXmlVerifications(this);
}
}
}

112
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java Normal file
View File

@ -0,0 +1,112 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.http.JettyUtils;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.function.Consumer;
import static org.junit.Assert.assertEquals;
/**
* This class hides the implementation details of how to verify the structure of
* XML responses. Tests should only provide the path of the
* {@link WebResource}, the response from the resource and
* the verifier Consumer to
* {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of
* {@link JSONObject} will be passed to that consumer to be able to
* verify the response.
*/
public class XmlCustomResourceTypeTestCase {
private static final Logger LOG =
LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class);
private WebResource path;
private BufferedClientResponse response;
private Document parsedResponse;
public XmlCustomResourceTypeTestCase(WebResource path,
BufferedClientResponse response) {
this.path = path;
this.response = response;
}
public void verify(Consumer<Document> verifier) {
assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
response.getType().toString());
parsedResponse = parseXml(response);
logResponse(parsedResponse);
verifier.accept(parsedResponse);
}
private Document parseXml(BufferedClientResponse response) {
try {
String xml = response.getEntity(String.class);
DocumentBuilder db =
DocumentBuilderFactory.newInstance().newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(xml));
return db.parse(is);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private void logResponse(Document doc) {
String responseStr = response.getEntity(String.class);
LOG.info("Raw response from service URL {}: {}", path.toString(),
responseStr);
LOG.info("Parsed response from service URL {}: {}", path.toString(),
toXml(doc));
}
public static String toXml(Node node) {
StringWriter writer;
try {
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.setOutputProperty(
"{http://xml.apache.org/xslt}indent" + "-amount", "2");
writer = new StringWriter();
transformer.transform(new DOMSource(node), new StreamResult(writer));
} catch (TransformerException e) {
throw new RuntimeException(e);
}
return writer.getBuffer().toString();
}
}