diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 40f169d948a..4ecda2cae91 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -87,6 +87,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3391. Making a trivial change to correct a log message in DistributedShell app's AM. (Subroto Sanyal via vinodkv) + MAPREDUCE-3547. Added a bunch of unit tests for the the RM/NM webservices. + (Thomas Graves via acmurthy) + OPTIMIZATIONS MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java new file mode 100644 index 00000000000..2cb67724b99 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.webapp; + +import static org.junit.Assert.assertTrue; + +import org.w3c.dom.Attr; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class WebServicesTestUtils { + + public static long getXmlLong(Element element, String name) { + String val = getXmlString(element, name); + return Long.parseLong(val); + } + + public static int getXmlInt(Element element, String name) { + String val = getXmlString(element, name); + return Integer.parseInt(val); + } + + public static Boolean getXmlBoolean(Element element, String name) { + String val = getXmlString(element, name); + return Boolean.parseBoolean(val); + } + + public static float getXmlFloat(Element element, String name) { + String val = getXmlString(element, name); + return Float.parseFloat(val); + } + + public static String getXmlString(Element element, String name) { + NodeList id = element.getElementsByTagName(name); + Element line = (Element) id.item(0); + Node first = line.getFirstChild(); + // handle empty + if (first == null) { + return ""; + } + String val = first.getNodeValue(); + if (val == null) { + return ""; + } + return val; + } + + public static String getXmlAttrString(Element element, String name) { + Attr at = element.getAttributeNode(name); + if (at != null) { + return at.getValue(); + } + return null; + } + + public static void checkStringMatch(String print, String expected, String got) { + assertTrue( + print + " doesn't match, got: " + got + " expected: " + expected, + got.matches(expected)); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 9f7b90189a7..909679606de 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; @@ -92,12 +93,16 @@ public class NMWebServices { AppInfo appInfo = new AppInfo(entry.getValue()); if (stateQuery != null && !stateQuery.isEmpty()) { - ApplicationState state = ApplicationState.valueOf(stateQuery); + ApplicationState.valueOf(stateQuery); if (!appInfo.getState().equalsIgnoreCase(stateQuery)) { continue; } } - if (userQuery != null && !userQuery.isEmpty()) { + if (userQuery != null) { + if (userQuery.isEmpty()) { + String msg = "Error: You must specify a non-empty string for the user"; + throw new BadRequestException(msg); + } if (!appInfo.getUser().toString().equals(userQuery)) { continue; } @@ -146,11 +151,12 @@ public class NMWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public ContainerInfo getNodeContainer(@PathParam("containerid") String id) { ContainerId containerId = null; - containerId = ConverterUtils.toContainerId(id); - if (containerId == null) { - throw new NotFoundException("container with id, " + id - + ", is empty or null"); + try { + containerId = ConverterUtils.toContainerId(id); + } catch (Exception e) { + throw new BadRequestException("invalid container id, " + id); } + Container container = nmContext.getContainers().get(containerId); if (container == null) { throw new NotFoundException("container with id, " + id + ", not found"); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 5c79a7b52c1..41c649eea2c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -56,7 +57,7 @@ public class ContainerInfo { } public ContainerInfo(final Context nmContext, final Container container, - final String requestUri, final String pathPrefix) { + String requestUri, String pathPrefix) { this.id = container.getContainerID().toString(); this.nodeId = nmContext.getNodeId().toString(); @@ -71,10 +72,19 @@ public class ContainerInfo { } this.user = container.getUser(); - this.totalMemoryNeededMB = container.getLaunchContext().getResource() - .getMemory(); + Resource res = container.getLaunchContext().getResource(); + if (res != null) { + this.totalMemoryNeededMB = res.getMemory(); + } this.containerLogsShortLink = ujoin("containerlogs", this.id, container.getUser()); + + if (requestUri == null) { + requestUri = ""; + } + if (pathPrefix == null) { + pathPrefix = ""; + } this.containerLogsLink = join(requestUri, pathPrefix, this.containerLogsShortLink); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockApp.java new file mode 100644 index 00000000000..7a9dac69483 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockApp.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.util.BuilderUtils; + +public class MockApp implements Application { + + final String user; + final ApplicationId appId; + Map containers = new HashMap(); + ApplicationState appState; + Application app; + + public MockApp(int uniqId) { + this("mockUser", 1234, uniqId); + } + + public MockApp(String user, long clusterTimeStamp, int uniqId) { + super(); + this.user = user; + // Add an application and the corresponding containers + RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(new Configuration()); + this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, + uniqId); + appState = ApplicationState.NEW; + } + + public void setState(ApplicationState state) { + this.appState = state; + } + + public String getUser() { + return user; + } + + public Map getContainers() { + return containers; + } + + public ApplicationId getAppId() { + return appId; + } + + public ApplicationState getApplicationState() { + return appState; + } + + public void handle(ApplicationEvent event) {} + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java new file mode 100644 index 00000000000..eacdb1c4241 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.util.BuilderUtils; + +public class MockContainer implements Container { + + private ContainerId id; + private ContainerState state; + private String user; + private ContainerLaunchContext launchContext; + private final Map resource = new HashMap(); + private RecordFactory recordFactory; + + public MockContainer(ApplicationAttemptId appAttemptId, + Dispatcher dispatcher, Configuration conf, String user, + ApplicationId appId, int uniqId) { + + this.user = user; + this.recordFactory = RecordFactoryProvider.getRecordFactory(conf); + this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, + uniqId); + this.launchContext = recordFactory + .newRecordInstance(ContainerLaunchContext.class); + launchContext.setContainerId(id); + launchContext.setUser(user); + this.state = ContainerState.NEW; + + } + + public void setState(ContainerState state) { + this.state = state; + } + + @Override + public ContainerId getContainerID() { + return id; + } + + @Override + public String getUser() { + return user; + } + + @Override + public ContainerState getContainerState() { + return state; + } + + @Override + public ContainerLaunchContext getLaunchContext() { + return launchContext; + } + + @Override + public Credentials getCredentials() { + return null; + } + + @Override + public Map getLocalizedResources() { + return resource; + } + + @Override + public ContainerStatus cloneAndGetContainerStatus() { + ContainerStatus containerStatus = recordFactory + .newRecordInstance(ContainerStatus.class); + containerStatus + .setState(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING); + containerStatus.setContainerId(this.launchContext.getContainerId()); + containerStatus.setDiagnostics("testing"); + containerStatus.setExitStatus(0); + return containerStatus; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void handle(ContainerEvent event) { + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java new file mode 100644 index 00000000000..3b1d2cf81c7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -0,0 +1,361 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.StringReader; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.ResourceView; +import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +/** + * Test the nodemanager node info web services api's + */ +public class TestNMWebServices extends JerseyTest { + + private static Context nmContext; + private static ResourceView resourceView; + private static ApplicationACLsManager aclsManager; + private static LocalDirsHandlerService dirsHandler; + private static WebApp nmWebApp; + + private static final File testRootDir = new File("target", + TestNMWebServices.class.getSimpleName()); + private static File testLogDir = new File("target", + TestNMWebServices.class.getSimpleName() + "LogDir"); + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + nmContext = new NodeManager.NMContext(); + nmContext.getNodeId().setHost("testhost.foo.com"); + nmContext.getNodeId().setPort(9999); + resourceView = new ResourceView() { + @Override + public long getVmemAllocatedForContainers() { + // 15.5G in bytes + return new Long("16642998272"); + } + + @Override + public long getPmemAllocatedForContainers() { + // 16G in bytes + return new Long("17179869184"); + } + }; + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); + conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + dirsHandler = healthChecker.getDiskHandler(); + aclsManager = new ApplicationACLsManager(conf); + nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); + bind(JAXBContextResolver.class); + bind(NMWebServices.class); + bind(GenericExceptionHandler.class); + bind(Context.class).toInstance(nmContext); + bind(WebApp.class).toInstance(nmWebApp); + bind(ResourceView.class).toInstance(resourceView); + bind(ApplicationACLsManager.class).toInstance(aclsManager); + bind(LocalDirsHandlerService.class).toInstance(dirsHandler); + + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + testRootDir.mkdirs(); + testLogDir.mkdir(); + } + + @AfterClass + static public void stop() { + FileUtil.fullyDelete(testRootDir); + FileUtil.fullyDelete(testLogDir); + } + + public TestNMWebServices() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.nodemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testInvalidUri() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.path("ws").path("v1").path("node").path("bogus") + .accept(MediaType.APPLICATION_JSON).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + + @Test + public void testInvalidAccept() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.path("ws").path("v1").path("node") + .accept(MediaType.TEXT_PLAIN).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.INTERNAL_SERVER_ERROR, + response.getClientResponseStatus()); + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + + @Test + public void testInvalidUri2() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.accept(MediaType.APPLICATION_JSON).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + + @Test + public void testNode() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + @Test + public void testNodeSlash() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node/") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + // make sure default is json output + @Test + public void testNodeDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node") + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + @Test + public void testNodeInfo() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node").path("info") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + @Test + public void testNodeInfoSlash() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node") + .path("info/").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + // make sure default is json output + @Test + public void testNodeInfoDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node").path("info") + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeInfo(json); + } + + @Test + public void testSingleNodesXML() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node") + .path("info/").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("nodeInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyNodesXML(nodes); + } + + public void verifyNodesXML(NodeList nodes) throws JSONException, Exception { + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyNodeInfoGeneric(WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "healthReport"), + WebServicesTestUtils.getXmlLong(element, + "totalVmemAllocatedContainersMB"), + WebServicesTestUtils.getXmlLong(element, + "totalPmemAllocatedContainersMB"), + WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"), + WebServicesTestUtils.getXmlBoolean(element, "nodeHealthy"), + WebServicesTestUtils.getXmlString(element, "nodeHostName"), + WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), + WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), + WebServicesTestUtils.getXmlString(element, "hadoopVersion"), + WebServicesTestUtils.getXmlString(element, + "nodeManagerVersionBuiltOn"), WebServicesTestUtils.getXmlString( + element, "nodeManagerBuildVersion"), + WebServicesTestUtils.getXmlString(element, "nodeManagerVersion")); + } + } + + public void verifyNodeInfo(JSONObject json) throws JSONException, Exception { + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject info = json.getJSONObject("nodeInfo"); + assertEquals("incorrect number of elements", 13, info.length()); + verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"), + info.getLong("totalVmemAllocatedContainersMB"), + info.getLong("totalPmemAllocatedContainersMB"), + info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"), + info.getString("nodeHostName"), info.getString("hadoopVersionBuiltOn"), + info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), + info.getString("nodeManagerVersionBuiltOn"), + info.getString("nodeManagerBuildVersion"), + info.getString("nodeManagerVersion")); + + } + + public void verifyNodeInfoGeneric(String id, String healthReport, + long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB, + long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName, + String hadoopVersionBuiltOn, String hadoopBuildVersion, + String hadoopVersion, String resourceManagerVersionBuiltOn, + String resourceManagerBuildVersion, String resourceManagerVersion) { + + WebServicesTestUtils.checkStringMatch("id", "testhost.foo.com:9999", id); + WebServicesTestUtils.checkStringMatch("healthReport", "Healthy", + healthReport); + assertEquals("totalVmemAllocatedContainersMB incorrect", 15872, + totalVmemAllocatedContainersMB); + assertEquals("totalPmemAllocatedContainersMB incorrect", 16384, + totalPmemAllocatedContainersMB); + assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext + .getNodeHealthStatus().getLastHealthReportTime()); + assertTrue("nodeHealthy isn't true", nodeHealthy); + WebServicesTestUtils.checkStringMatch("nodeHostName", "testhost.foo.com", + nodeHostName); + + WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn", + VersionInfo.getDate(), hadoopVersionBuiltOn); + WebServicesTestUtils.checkStringMatch("hadoopBuildVersion", + VersionInfo.getBuildVersion(), hadoopBuildVersion); + WebServicesTestUtils.checkStringMatch("hadoopVersion", + VersionInfo.getVersion(), hadoopVersion); + + WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn", + YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn); + WebServicesTestUtils.checkStringMatch("resourceManagerBuildVersion", + YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion); + WebServicesTestUtils.checkStringMatch("resourceManagerVersion", + YarnVersionInfo.getVersion(), resourceManagerVersion); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java new file mode 100644 index 00000000000..f61fdbd4761 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -0,0 +1,607 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.StringReader; +import java.util.HashMap; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.MockApp; +import org.apache.hadoop.yarn.server.nodemanager.MockContainer; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.ResourceView; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestNMWebServicesApps extends JerseyTest { + + private static Context nmContext; + private static ResourceView resourceView; + private static ApplicationACLsManager aclsManager; + private static LocalDirsHandlerService dirsHandler; + private static WebApp nmWebApp; + private static Configuration conf = new Configuration(); + + private static final File testRootDir = new File("target", + TestNMWebServicesApps.class.getSimpleName()); + private static File testLogDir = new File("target", + TestNMWebServicesApps.class.getSimpleName() + "LogDir"); + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + nmContext = new NodeManager.NMContext(); + nmContext.getNodeId().setHost("testhost.foo.com"); + nmContext.getNodeId().setPort(9999); + resourceView = new ResourceView() { + @Override + public long getVmemAllocatedForContainers() { + // 15.5G in bytes + return new Long("16642998272"); + } + + @Override + public long getPmemAllocatedForContainers() { + // 16G in bytes + return new Long("17179869184"); + } + }; + conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); + conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + dirsHandler = healthChecker.getDiskHandler(); + aclsManager = new ApplicationACLsManager(conf); + nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); + bind(JAXBContextResolver.class); + bind(NMWebServices.class); + bind(GenericExceptionHandler.class); + bind(Context.class).toInstance(nmContext); + bind(WebApp.class).toInstance(nmWebApp); + bind(ResourceView.class).toInstance(resourceView); + bind(ApplicationACLsManager.class).toInstance(aclsManager); + bind(LocalDirsHandlerService.class).toInstance(dirsHandler); + + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + testRootDir.mkdirs(); + testLogDir.mkdir(); + } + + @AfterClass + static public void cleanup() { + FileUtil.fullyDelete(testRootDir); + FileUtil.fullyDelete(testLogDir); + } + + public TestNMWebServicesApps() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.nodemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testNodeAppsNone() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps")); + } + + private HashMap addAppContainers(Application app) { + Dispatcher dispatcher = new AsyncDispatcher(); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + app.getAppId(), 1); + Container container1 = new MockContainer(appAttemptId, dispatcher, conf, + app.getUser(), app.getAppId(), 1); + Container container2 = new MockContainer(appAttemptId, dispatcher, conf, + app.getUser(), app.getAppId(), 2); + nmContext.getContainers().put(container1.getContainerID(), container1); + nmContext.getContainers().put(container2.getContainerID(), container2); + + app.getContainers().put(container1.getContainerID(), container1); + app.getContainers().put(container2.getContainerID(), container2); + HashMap hash = new HashMap(); + hash.put(container1.getContainerID().toString(), container1 + .getContainerID().toString()); + hash.put(container2.getContainerID().toString(), container2 + .getContainerID().toString()); + return hash; + } + + @Test + public void testNodeApps() throws JSONException, Exception { + testNodeHelper("apps", MediaType.APPLICATION_JSON); + } + + @Test + public void testNodeAppsSlash() throws JSONException, Exception { + testNodeHelper("apps/", MediaType.APPLICATION_JSON); + } + + // make sure default is json output + @Test + public void testNodeAppsDefault() throws JSONException, Exception { + testNodeHelper("apps/", ""); + + } + + public void testNodeHelper(String path, String media) throws JSONException, + Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + HashMap hash2 = addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path(path) + .accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + JSONObject info = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, info.length()); + JSONArray appInfo = info.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, appInfo.length()); + String id = appInfo.getJSONObject(0).getString("id"); + if (id.matches(app.getAppId().toString())) { + verifyNodeAppInfo(appInfo.getJSONObject(0), app, hash); + verifyNodeAppInfo(appInfo.getJSONObject(1), app2, hash2); + } else { + verifyNodeAppInfo(appInfo.getJSONObject(0), app2, hash2); + verifyNodeAppInfo(appInfo.getJSONObject(1), app, hash); + } + } + + @Test + public void testNodeAppsUser() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .queryParam("user", "mockUser").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + JSONObject info = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, info.length()); + JSONArray appInfo = info.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, appInfo.length()); + verifyNodeAppInfo(appInfo.getJSONObject(0), app, hash); + } + + @Test + public void testNodeAppsUserNone() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .queryParam("user", "george").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + } + + @Test + public void testNodeAppsUserEmpty() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + try { + r.path("ws").path("v1").path("node").path("apps").queryParam("user", "") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "java.lang.Exception: Error: You must specify a non-empty string for the user", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } + } + + @Test + public void testNodeAppsState() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + MockApp app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + HashMap hash2 = addAppContainers(app2); + app2.setState(ApplicationState.RUNNING); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .queryParam("state", ApplicationState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + JSONObject info = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, info.length()); + JSONArray appInfo = info.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, appInfo.length()); + verifyNodeAppInfo(appInfo.getJSONObject(0), app2, hash2); + + } + + @Test + public void testNodeAppsStateNone() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .queryParam("state", ApplicationState.INITING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + } + + @Test + public void testNodeAppsStateInvalid() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp("foo", 1234, 2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + try { + r.path("ws").path("v1").path("node").path("apps") + .queryParam("state", "FOO_STATE").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "No enum const class org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + } + } + + @Test + public void testNodeSingleApps() throws JSONException, Exception { + testNodeSingleAppHelper(MediaType.APPLICATION_JSON); + } + + // make sure default is json output + @Test + public void testNodeSingleAppsDefault() throws JSONException, Exception { + testNodeSingleAppHelper(""); + } + + public void testNodeSingleAppHelper(String media) throws JSONException, + Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .path(app.getAppId().toString()).accept(media) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeAppInfo(json.getJSONObject("app"), app, hash); + } + + @Test + public void testNodeSingleAppsSlash() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .path(app.getAppId().toString() + "/") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeAppInfo(json.getJSONObject("app"), app, hash); + } + + @Test + public void testNodeSingleAppsInvalid() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + try { + r.path("ws").path("v1").path("node").path("apps").path("app_foo_0000") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "For input string: \"foo\"", message); + WebServicesTestUtils.checkStringMatch("exception type", + "NumberFormatException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.NumberFormatException", classname); + } + } + + @Test + public void testNodeSingleAppsMissing() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + try { + r.path("ws").path("v1").path("node").path("apps") + .path("application_1234_0009").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "java.lang.Exception: app with id application_1234_0009 not found", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "NotFoundException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); + } + } + + @Test + public void testNodeAppsXML() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 2, nodes.getLength()); + } + + @Test + public void testNodeSingleAppsXML() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + .path(app.getAppId().toString() + "/") + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyNodeAppInfoXML(nodes, app, hash); + } + + public void verifyNodeAppInfoXML(NodeList nodes, Application app, + HashMap hash) throws JSONException, Exception { + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyNodeAppInfoGeneric(app, + WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "user")); + + NodeList ids = element.getElementsByTagName("containerids"); + for (int j = 0; j < ids.getLength(); j++) { + Element line = (Element) ids.item(j); + Node first = line.getFirstChild(); + String val = first.getNodeValue(); + assertEquals("extra containerid: " + val, val, hash.remove(val)); + } + assertTrue("missing containerids", hash.isEmpty()); + } + } + + public void verifyNodeAppInfo(JSONObject info, Application app, + HashMap hash) throws JSONException, Exception { + assertEquals("incorrect number of elements", 4, info.length()); + + verifyNodeAppInfoGeneric(app, info.getString("id"), + info.getString("state"), info.getString("user")); + + JSONArray containerids = info.getJSONArray("containerids"); + for (int i = 0; i < containerids.length(); i++) { + String id = containerids.getString(i); + assertEquals("extra containerid: " + id, id, hash.remove(id)); + } + assertTrue("missing containerids", hash.isEmpty()); + } + + public void verifyNodeAppInfoGeneric(Application app, String id, + String state, String user) throws JSONException, Exception { + WebServicesTestUtils.checkStringMatch("id", app.getAppId().toString(), id); + WebServicesTestUtils.checkStringMatch("state", app.getApplicationState() + .toString(), state); + WebServicesTestUtils.checkStringMatch("user", app.getUser().toString(), + user); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java new file mode 100644 index 00000000000..11ec3401015 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -0,0 +1,481 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import static org.apache.hadoop.yarn.util.StringHelper.ujoin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.StringReader; +import java.util.HashMap; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.MockApp; +import org.apache.hadoop.yarn.server.nodemanager.MockContainer; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.ResourceView; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestNMWebServicesContainers extends JerseyTest { + + private static Context nmContext; + private static ResourceView resourceView; + private static ApplicationACLsManager aclsManager; + private static LocalDirsHandlerService dirsHandler; + private static WebApp nmWebApp; + private static Configuration conf = new Configuration(); + + private static final File testRootDir = new File("target", + TestNMWebServicesContainers.class.getSimpleName()); + private static File testLogDir = new File("target", + TestNMWebServicesContainers.class.getSimpleName() + "LogDir"); + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + nmContext = new NodeManager.NMContext(); + nmContext.getNodeId().setHost("testhost.foo.com"); + nmContext.getNodeId().setPort(9999); + resourceView = new ResourceView() { + @Override + public long getVmemAllocatedForContainers() { + // 15.5G in bytes + return new Long("16642998272"); + } + + @Override + public long getPmemAllocatedForContainers() { + // 16G in bytes + return new Long("17179869184"); + } + }; + conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); + conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + dirsHandler = healthChecker.getDiskHandler(); + aclsManager = new ApplicationACLsManager(conf); + nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); + bind(JAXBContextResolver.class); + bind(NMWebServices.class); + bind(GenericExceptionHandler.class); + bind(Context.class).toInstance(nmContext); + bind(WebApp.class).toInstance(nmWebApp); + bind(ResourceView.class).toInstance(resourceView); + bind(ApplicationACLsManager.class).toInstance(aclsManager); + bind(LocalDirsHandlerService.class).toInstance(dirsHandler); + + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + testRootDir.mkdirs(); + testLogDir.mkdir(); + } + + @AfterClass + static public void cleanup() { + FileUtil.fullyDelete(testRootDir); + FileUtil.fullyDelete(testLogDir); + } + + public TestNMWebServicesContainers() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.nodemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testNodeContainersNone() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node") + .path("containers").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("apps isn't NULL", JSONObject.NULL, json.get("containers")); + } + + private HashMap addAppContainers(Application app) { + Dispatcher dispatcher = new AsyncDispatcher(); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + app.getAppId(), 1); + Container container1 = new MockContainer(appAttemptId, dispatcher, conf, + app.getUser(), app.getAppId(), 1); + Container container2 = new MockContainer(appAttemptId, dispatcher, conf, + app.getUser(), app.getAppId(), 2); + nmContext.getContainers().put(container1.getContainerID(), container1); + nmContext.getContainers().put(container2.getContainerID(), container2); + + app.getContainers().put(container1.getContainerID(), container1); + app.getContainers().put(container2.getContainerID(), container2); + HashMap hash = new HashMap(); + hash.put(container1.getContainerID().toString(), container1 + .getContainerID().toString()); + hash.put(container2.getContainerID().toString(), container2 + .getContainerID().toString()); + return hash; + } + + @Test + public void testNodeContainers() throws JSONException, Exception { + testNodeHelper("containers", MediaType.APPLICATION_JSON); + } + + @Test + public void testNodeContainersSlash() throws JSONException, Exception { + testNodeHelper("containers/", MediaType.APPLICATION_JSON); + } + + // make sure default is json output + @Test + public void testNodeContainersDefault() throws JSONException, Exception { + testNodeHelper("containers/", ""); + + } + + public void testNodeHelper(String path, String media) throws JSONException, + Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node").path(path) + .accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + JSONObject info = json.getJSONObject("containers"); + assertEquals("incorrect number of elements", 1, info.length()); + JSONArray conInfo = info.getJSONArray("container"); + assertEquals("incorrect number of elements", 4, conInfo.length()); + + for (int i = 0; i < conInfo.length(); i++) { + verifyNodeContainerInfo( + conInfo.getJSONObject(i), + nmContext.getContainers().get( + ConverterUtils.toContainerId(conInfo.getJSONObject(i).getString( + "id")))); + } + } + + @Test + public void testNodeSingleContainers() throws JSONException, Exception { + testNodeSingleContainersHelper(MediaType.APPLICATION_JSON); + } + + @Test + public void testNodeSingleContainersSlash() throws JSONException, Exception { + testNodeSingleContainersHelper(MediaType.APPLICATION_JSON); + } + + @Test + public void testNodeSingleContainersDefault() throws JSONException, Exception { + testNodeSingleContainersHelper(""); + } + + public void testNodeSingleContainersHelper(String media) + throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + for (String id : hash.keySet()) { + ClientResponse response = r.path("ws").path("v1").path("node") + .path("containers").path(id).accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyNodeContainerInfo(json.getJSONObject("container"), nmContext + .getContainers().get(ConverterUtils.toContainerId(id))); + } + } + + @Test + public void testSingleContainerInvalid() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + try { + r.path("ws").path("v1").path("node").path("containers") + .path("container_foo_1234").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "java.lang.Exception: invalid container id, container_foo_1234", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } + } + + @Test + public void testSingleContainerInvalid2() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + try { + r.path("ws").path("v1").path("node").path("containers") + .path("container_1234_0001").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "java.lang.Exception: invalid container id, container_1234_0001", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } + } + + @Test + public void testSingleContainerWrong() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + try { + r.path("ws").path("v1").path("node").path("containers") + .path("container_1234_0001_01_000005") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "java.lang.Exception: container with id, container_1234_0001_01_000005, not found", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "NotFoundException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); + } + } + + @Test + public void testNodeSingleContainerXML() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + HashMap hash = addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + for (String id : hash.keySet()) { + ClientResponse response = r.path("ws").path("v1").path("node") + .path("containers").path(id).accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("container"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyContainersInfoXML(nodes, + nmContext.getContainers().get(ConverterUtils.toContainerId(id))); + + } + } + + @Test + public void testNodeContainerXML() throws JSONException, Exception { + WebResource r = resource(); + Application app = new MockApp(1); + nmContext.getApplications().put(app.getAppId(), app); + addAppContainers(app); + Application app2 = new MockApp(2); + nmContext.getApplications().put(app2.getAppId(), app2); + addAppContainers(app2); + + ClientResponse response = r.path("ws").path("v1").path("node") + .path("containers").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("container"); + assertEquals("incorrect number of elements", 4, nodes.getLength()); + } + + public void verifyContainersInfoXML(NodeList nodes, Container cont) + throws JSONException, Exception { + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyNodeContainerInfoGeneric(cont, + WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "user"), + WebServicesTestUtils.getXmlInt(element, "exitCode"), + WebServicesTestUtils.getXmlString(element, "diagnostics"), + WebServicesTestUtils.getXmlString(element, "nodeId"), + WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"), + WebServicesTestUtils.getXmlString(element, "containerLogsLink")); + } + } + + public void verifyNodeContainerInfo(JSONObject info, Container cont) + throws JSONException, Exception { + assertEquals("incorrect number of elements", 8, info.length()); + + verifyNodeContainerInfoGeneric(cont, info.getString("id"), + info.getString("state"), info.getString("user"), + info.getInt("exitCode"), info.getString("diagnostics"), + info.getString("nodeId"), info.getInt("totalMemoryNeededMB"), + info.getString("containerLogsLink")); + } + + public void verifyNodeContainerInfoGeneric(Container cont, String id, + String state, String user, int exitCode, String diagnostics, + String nodeId, int totalMemoryNeededMB, String logsLink) + throws JSONException, Exception { + WebServicesTestUtils.checkStringMatch("id", cont.getContainerID() + .toString(), id); + WebServicesTestUtils.checkStringMatch("state", cont.getContainerState() + .toString(), state); + WebServicesTestUtils.checkStringMatch("user", cont.getUser().toString(), + user); + assertEquals("exitCode wrong", 0, exitCode); + WebServicesTestUtils + .checkStringMatch("diagnostics", "testing", diagnostics); + + WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId() + .toString(), nodeId); + assertEquals("totalMemoryNeededMB wrong", 0, totalMemoryNeededMB); + String shortLink = ujoin("containerlogs", cont.getContainerID().toString(), + cont.getUser()); + assertTrue("containerLogsLink wrong", logsLink.contains(shortLink)); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index c66fa938873..921b5ea9e18 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; @@ -90,7 +91,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { if (max < EPSILON || max > 1f) max = 1f; float maxCapacity = max * 100; - String state = queue.getState().toString(); + QueueState state = queue.getState(); CapacitySchedulerQueueInfo info = new CapacitySchedulerQueueInfo( capacity, usedCapacity, maxCapacity, queueName, state, queuePath); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f346fb0c87a..14cffd8e5eb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; @XmlRootElement @@ -43,14 +44,14 @@ public class CapacitySchedulerQueueInfo { protected float usedCapacity; protected float maxCapacity; protected String queueName; - protected String state; + protected QueueState state; protected ArrayList subQueues; CapacitySchedulerQueueInfo() { }; CapacitySchedulerQueueInfo(float cap, float used, float max, String name, - String state, String path) { + QueueState state, String path) { this.capacity = cap; this.usedCapacity = used; this.maxCapacity = max; @@ -84,7 +85,7 @@ public class CapacitySchedulerQueueInfo { } public String getQueueState() { - return this.state; + return this.state.toString(); } public String getQueuePath() { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java index b4511bc1767..a68fe0a7d1f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java @@ -23,6 +23,7 @@ import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.service.Service.STATE; import org.apache.hadoop.yarn.util.YarnVersionInfo; @XmlRootElement @@ -31,7 +32,7 @@ public class ClusterInfo { protected long id; protected long startedOn; - protected String state; + protected STATE state; protected String resourceManagerVersion; protected String resourceManagerBuildVersion; protected String resourceManagerVersionBuiltOn; @@ -46,7 +47,7 @@ public class ClusterInfo { long ts = ResourceManager.clusterTimeStamp; this.id = ts; - this.state = rm.getServiceState().toString(); + this.state = rm.getServiceState(); this.startedOn = ts; this.resourceManagerVersion = YarnVersionInfo.getVersion(); this.resourceManagerBuildVersion = YarnVersionInfo.getBuildVersion(); @@ -57,7 +58,7 @@ public class ClusterInfo { } public String getState() { - return this.state; + return this.state.toString(); } public String getRMVersion() { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index e80e629aa54..cdcfd2590ec 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -87,8 +87,12 @@ public class MockRM extends ResourceManager { .newRecord(GetNewApplicationRequest.class)); } - // client public RMApp submitApp(int masterMemory) throws Exception { + return submitApp(masterMemory, "", ""); + } + + // client + public RMApp submitApp(int masterMemory, String name, String user) throws Exception { ClientRMProtocol client = getClientRMService(); GetNewApplicationResponse resp = client.getNewApplication(Records .newRecord(GetNewApplicationRequest.class)); @@ -99,8 +103,8 @@ public class MockRM extends ResourceManager { ApplicationSubmissionContext sub = Records .newRecord(ApplicationSubmissionContext.class); sub.setApplicationId(appId); - sub.setApplicationName(""); - sub.setUser(""); + sub.setApplicationName(name); + sub.setUser(user); ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); Resource capability = Records.newRecord(Resource.class); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index b19e7b54e5e..51d2fa7d0dd 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -22,34 +22,34 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.util.ArrayList; +import java.io.StringReader; import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeHealthStatus; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.server.resourcemanager.MockAM; -import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.service.Service.STATE; +import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.codehaus.jettison.json.JSONArray; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Before; import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; import com.google.inject.Guice; import com.google.inject.Injector; @@ -104,11 +104,73 @@ public class TestRMWebServices extends JerseyTest { .contextPath("jersey-guice-filter").servletPath("/").build()); } + @Test + public void testInfoXML() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("info").accept("application/xml").get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + verifyClusterInfoXML(xml); + } + + @Test + public void testInvalidUri() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.path("ws").path("v1").path("cluster").path("bogus") + .accept(MediaType.APPLICATION_JSON).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + + @Test + public void testInvalidUri2() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.accept(MediaType.APPLICATION_JSON).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + + @Test + public void testInvalidAccept() throws JSONException, Exception { + WebResource r = resource(); + String responseStr = ""; + try { + responseStr = r.path("ws").path("v1").path("cluster") + .accept(MediaType.TEXT_PLAIN).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.INTERNAL_SERVER_ERROR, + response.getClientResponseStatus()); + WebServicesTestUtils.checkStringMatch( + "error string exists and shouldn't", "", responseStr); + } + } + @Test public void testCluster() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterInfo(json); } @@ -116,986 +178,366 @@ public class TestRMWebServices extends JerseyTest { public void testClusterSlash() throws JSONException, Exception { WebResource r = resource(); // test with trailing "/" to make sure acts same as without slash - JSONObject json = r.path("ws").path("v1").path("cluster/") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + ClientResponse response = r.path("ws").path("v1").path("cluster/") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterInfo(json); + } + + @Test + public void testClusterDefault() throws JSONException, Exception { + WebResource r = resource(); + // test with trailing "/" to make sure acts same as without slash + ClientResponse response = r.path("ws").path("v1").path("cluster") + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterInfo(json); } @Test public void testInfo() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("info") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("info").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterInfo(json); } @Test public void testInfoSlash() throws JSONException, Exception { - WebResource r = resource(); // test with trailing "/" to make sure acts same as without slash - JSONObject json = r.path("ws").path("v1").path("cluster").path("info/") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("info/").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterInfo(json); } + @Test + public void testInfoDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("info").get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterInfo(json); + } + + public void verifyClusterInfoXML(String xml) throws JSONException, Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("clusterInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyClusterGeneric(WebServicesTestUtils.getXmlLong(element, "id"), + WebServicesTestUtils.getXmlLong(element, "startedOn"), + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), + WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), + WebServicesTestUtils.getXmlString(element, "hadoopVersion"), + WebServicesTestUtils.getXmlString(element, + "resourceManagerVersionBuiltOn"), + WebServicesTestUtils.getXmlString(element, + "resourceManagerBuildVersion"), + WebServicesTestUtils.getXmlString(element, "resourceManagerVersion")); + } + } + public void verifyClusterInfo(JSONObject json) throws JSONException, Exception { - assertEquals("correct number of elements", 1, json.length()); - JSONObject clusterinfo = json.getJSONObject("clusterInfo"); - assertEquals("correct number of elements", 9, clusterinfo.length()); - String clusterid = clusterinfo.get("id").toString(); - assertTrue("clusterId doesn't match: " + clusterid, clusterid.toString() - .matches("^\\d+")); - String startedon = clusterinfo.get("startedOn").toString(); - assertTrue("startedOn doesn't match: " + startedon, - startedon.matches("^\\d+")); - String state = clusterinfo.get("state").toString(); - assertTrue("stated doesn't match: " + state, state.matches("INITED")); - String rmVersion = clusterinfo.get("resourceManagerVersion").toString(); - assertTrue("rm version doesn't match: " + rmVersion, - rmVersion.matches(".*")); - String rmBuildVersion = clusterinfo.get("resourceManagerBuildVersion") - .toString(); - assertTrue("rm Build version doesn't match: " + rmBuildVersion, - rmBuildVersion.matches(".*")); - String rmVersionBuiltOn = clusterinfo.get("resourceManagerVersionBuiltOn") - .toString(); - assertTrue( - "rm version built on doesn't match: " + rmVersionBuiltOn, - rmVersionBuiltOn - .matches("^\\w+\\s+\\w+\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+\\w+\\s+\\d\\d\\d\\d")); - String hadoopVersion = clusterinfo.get("hadoopVersion").toString(); - assertTrue("hadoop version doesn't match: " + hadoopVersion, - hadoopVersion.matches(".*")); - String hadoopBuildVersion = clusterinfo.get("hadoopBuildVersion") - .toString(); - assertTrue("hadoop Build version doesn't match: " + hadoopBuildVersion, - hadoopBuildVersion.matches(".*")); - String hadoopVersionBuiltOn = clusterinfo.get("hadoopVersionBuiltOn") - .toString(); - assertTrue( - "hadoop version built on doesn't match: " + hadoopVersionBuiltOn, - hadoopVersionBuiltOn - .matches("^\\w+\\s+\\w+\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+\\w+\\s+\\d\\d\\d\\d")); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject info = json.getJSONObject("clusterInfo"); + assertEquals("incorrect number of elements", 9, info.length()); + verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"), + info.getString("state"), info.getString("hadoopVersionBuiltOn"), + info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), + info.getString("resourceManagerVersionBuiltOn"), + info.getString("resourceManagerBuildVersion"), + info.getString("resourceManagerVersion")); + + } + + public void verifyClusterGeneric(long clusterid, long startedon, + String state, String hadoopVersionBuiltOn, String hadoopBuildVersion, + String hadoopVersion, String resourceManagerVersionBuiltOn, + String resourceManagerBuildVersion, String resourceManagerVersion) { + + assertEquals("clusterId doesn't match: ", ResourceManager.clusterTimeStamp, + clusterid); + assertEquals("startedOn doesn't match: ", ResourceManager.clusterTimeStamp, + startedon); + assertTrue("stated doesn't match: " + state, + state.matches(STATE.INITED.toString())); + + WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn", + VersionInfo.getDate(), hadoopVersionBuiltOn); + WebServicesTestUtils.checkStringMatch("hadoopBuildVersion", + VersionInfo.getBuildVersion(), hadoopBuildVersion); + WebServicesTestUtils.checkStringMatch("hadoopVersion", + VersionInfo.getVersion(), hadoopVersion); + + WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn", + YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn); + WebServicesTestUtils.checkStringMatch("resourceManagerBuildVersion", + YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion); + WebServicesTestUtils.checkStringMatch("resourceManagerVersion", + YarnVersionInfo.getVersion(), resourceManagerVersion); } @Test public void testClusterMetrics() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("metrics") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); - verifyClusterMetrics(json); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("metrics").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterMetricsJSON(json); } @Test public void testClusterMetricsSlash() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("metrics/") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); - verifyClusterMetrics(json); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("metrics/").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterMetricsJSON(json); } - public void verifyClusterMetrics(JSONObject json) throws JSONException, + @Test + public void testClusterMetricsDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("metrics").get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterMetricsJSON(json); + } + + @Test + public void testClusterMetricsXML() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("metrics").accept("application/xml").get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + verifyClusterMetricsXML(xml); + } + + public void verifyClusterMetricsXML(String xml) throws JSONException, Exception { - assertEquals("correct number of elements", 1, json.length()); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("clusterMetrics"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyClusterMetrics( + WebServicesTestUtils.getXmlInt(element, "appsSubmitted"), + WebServicesTestUtils.getXmlInt(element, "reservedMB"), + WebServicesTestUtils.getXmlInt(element, "availableMB"), + WebServicesTestUtils.getXmlInt(element, "allocatedMB"), + WebServicesTestUtils.getXmlInt(element, "containersAllocated"), + WebServicesTestUtils.getXmlInt(element, "totalMB"), + WebServicesTestUtils.getXmlInt(element, "totalNodes"), + WebServicesTestUtils.getXmlInt(element, "lostNodes"), + WebServicesTestUtils.getXmlInt(element, "unhealthyNodes"), + WebServicesTestUtils.getXmlInt(element, "decommissionedNodes"), + WebServicesTestUtils.getXmlInt(element, "rebootedNodes")); + } + } + + public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, + Exception { + assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("correct number of elements", 11, clusterinfo.length()); - assertEquals("appsSubmitted doesn't match", 0, - clusterinfo.getInt("appsSubmitted")); - assertEquals("reservedMB doesn't match", 0, - clusterinfo.getInt("reservedMB")); - assertEquals("availableMB doesn't match", 0, - clusterinfo.getInt("availableMB")); - assertEquals("allocatedMB doesn't match", 0, - clusterinfo.getInt("allocatedMB")); - assertEquals("containersAllocated doesn't match", 0, - clusterinfo.getInt("containersAllocated")); - assertEquals("totalMB doesn't match", 0, clusterinfo.getInt("totalMB")); - assertEquals("totalNodes doesn't match", 0, - clusterinfo.getInt("totalNodes")); - assertEquals("lostNodes doesn't match", 0, clusterinfo.getInt("lostNodes")); - assertEquals("unhealthyNodes doesn't match", 0, - clusterinfo.getInt("unhealthyNodes")); - assertEquals("decommissionedNodes doesn't match", 0, - clusterinfo.getInt("decommissionedNodes")); - assertEquals("rebootedNodes doesn't match", 0, + assertEquals("incorrect number of elements", 11, clusterinfo.length()); + verifyClusterMetrics(clusterinfo.getInt("appsSubmitted"), + clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), + clusterinfo.getInt("allocatedMB"), + clusterinfo.getInt("containersAllocated"), + clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"), + clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"), + clusterinfo.getInt("decommissionedNodes"), clusterinfo.getInt("rebootedNodes")); } + public void verifyClusterMetrics(int sub, int reservedMB, int availableMB, + int allocMB, int containersAlloc, int totalMB, int totalNodes, + int lostNodes, int unhealthyNodes, int decommissionedNodes, + int rebootedNodes) throws JSONException, Exception { + + ResourceScheduler rs = rm.getResourceScheduler(); + QueueMetrics metrics = rs.getRootQueueMetrics(); + ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); + final long MB_IN_GB = 1024; + + long totalMBExpect = (metrics.getReservedGB() * MB_IN_GB) + + (metrics.getAvailableGB() * MB_IN_GB) + + (metrics.getAllocatedGB() * MB_IN_GB); + + assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), sub); + assertEquals("reservedMB doesn't match", + metrics.getReservedGB() * MB_IN_GB, reservedMB); + assertEquals("availableMB doesn't match", metrics.getAvailableGB() + * MB_IN_GB, availableMB); + assertEquals("allocatedMB doesn't match", metrics.getAllocatedGB() + * MB_IN_GB, allocMB); + assertEquals("containersAllocated doesn't match", 0, containersAlloc); + assertEquals("totalMB doesn't match", totalMBExpect, totalMB); + assertEquals("totalNodes doesn't match", clusterMetrics.getNumNMs(), + totalNodes); + assertEquals("lostNodes doesn't match", clusterMetrics.getNumLostNMs(), + lostNodes); + assertEquals("unhealthyNodes doesn't match", + clusterMetrics.getUnhealthyNMs(), unhealthyNodes); + assertEquals("decommissionedNodes doesn't match", + clusterMetrics.getNumDecommisionedNMs(), decommissionedNodes); + assertEquals("rebootedNodes doesn't match", + clusterMetrics.getNumRebootedNMs(), rebootedNodes); + } + @Test public void testClusterSchedulerFifo() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("scheduler") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterSchedulerFifo(json); } @Test public void testClusterSchedulerFifoSlash() throws JSONException, Exception { WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster") + ClientResponse response = r.path("ws").path("v1").path("cluster") .path("scheduler/").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); verifyClusterSchedulerFifo(json); } + @Test + public void testClusterSchedulerFifoDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterSchedulerFifo(json); + } + + @Test + public void testClusterSchedulerFifoXML() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + verifySchedulerFifoXML(xml); + } + + public void verifySchedulerFifoXML(String xml) throws JSONException, + Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesSched = dom.getElementsByTagName("scheduler"); + assertEquals("incorrect number of elements", 1, nodesSched.getLength()); + NodeList nodes = dom.getElementsByTagName("schedulerInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyClusterSchedulerFifoGeneric( + WebServicesTestUtils.getXmlAttrString(element, "xsi:type"), + WebServicesTestUtils.getXmlString(element, "qstate"), + WebServicesTestUtils.getXmlFloat(element, "capacity"), + WebServicesTestUtils.getXmlFloat(element, "usedCapacity"), + WebServicesTestUtils.getXmlInt(element, "minQueueMemoryCapacity"), + WebServicesTestUtils.getXmlInt(element, "maxQueueMemoryCapacity"), + WebServicesTestUtils.getXmlInt(element, "numNodes"), + WebServicesTestUtils.getXmlInt(element, "usedNodeCapacity"), + WebServicesTestUtils.getXmlInt(element, "availNodeCapacity"), + WebServicesTestUtils.getXmlInt(element, "totalNodeCapacity"), + WebServicesTestUtils.getXmlInt(element, "numContainers")); + } + } + public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException, Exception { - assertEquals("correct number of elements", 1, json.length()); + assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("correct number of elements", 1, info.length()); + assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("correct number of elements", 11, info.length()); - assertEquals("type doesn't match", "fifoScheduler", info.getString("type")); - assertEquals("qstate doesn't match", QueueState.RUNNING.toString(), - info.getString("qstate")); - assertEquals("capacity doesn't match", 1.0, info.getDouble("capacity"), 0.0); - assertEquals("usedCapacity doesn't match", Float.NaN, - info.getDouble("usedCapacity"), 0.0); - assertEquals("minQueueMemoryCapacity doesn't match", 1024, - info.getInt("minQueueMemoryCapacity")); - assertEquals("maxQueueMemoryCapacity doesn't match", 10240, - info.getInt("maxQueueMemoryCapacity")); - assertEquals("maxQueueMemoryCapacity doesn't match", 10240, - info.getInt("maxQueueMemoryCapacity")); + assertEquals("incorrect number of elements", 11, info.length()); + + verifyClusterSchedulerFifoGeneric(info.getString("type"), + info.getString("qstate"), (float) info.getDouble("capacity"), + (float) info.getDouble("usedCapacity"), + info.getInt("minQueueMemoryCapacity"), + info.getInt("maxQueueMemoryCapacity"), info.getInt("numNodes"), + info.getInt("usedNodeCapacity"), info.getInt("availNodeCapacity"), + info.getInt("totalNodeCapacity"), info.getInt("numContainers")); } - @Test - public void testNodes() throws JSONException, Exception { - testNodesHelper("nodes"); - } - - @Test - public void testNodesSlash() throws JSONException, Exception { - testNodesHelper("nodes/"); - } - - @Test - public void testNodesQueryState() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); - rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); - - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("state", RMNodeState.RUNNING.toString()) - .accept("application/json").get(JSONObject.class); - - assertEquals("correct number of elements", 1, json.length()); - JSONObject nodes = json.getJSONObject("nodes"); - assertEquals("correct number of elements", 1, nodes.length()); - JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("correct number of elements", 1, nodeArray.length()); - JSONObject info = nodeArray.getJSONObject(0); - - verifyNodeInfo(info, nm1, RMNodeState.RUNNING); - } - - @Test - public void testNodesQueryStateNone() throws JSONException, Exception { - WebResource r = resource(); - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("state", RMNodeState.DECOMMISSIONED.toString()) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); - } - - @Test - public void testNodesQueryStateInvalid() throws JSONException, Exception { - WebResource r = resource(); - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - - try { - r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("state", "BOGUSSTATE").accept("application/json") - .get(JSONObject.class); - - fail("should have thrown exception querying invalid state"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch( - "exception message", - "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState.BOGUSSTATE", - message); - checkStringMatch("exception type", "IllegalArgumentException", type); - checkStringMatch("exception classname", - "java.lang.IllegalArgumentException", classname); - - } finally { - rm.stop(); - } - } - - @Test - public void testNodesQueryHealthy() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); - rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "true").accept("application/json") - .get(JSONObject.class); - - assertEquals("correct number of elements", 1, json.length()); - JSONObject nodes = json.getJSONObject("nodes"); - assertEquals("correct number of elements", 1, nodes.length()); - JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("correct number of elements", 2, nodeArray.length()); - } - - @Test - public void testNodesQueryHealthyCase() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); - rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "TRUe").accept("application/json") - .get(JSONObject.class); - - assertEquals("correct number of elements", 1, json.length()); - JSONObject nodes = json.getJSONObject("nodes"); - assertEquals("correct number of elements", 1, nodes.length()); - JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("correct number of elements", 2, nodeArray.length()); - - } - - @Test - public void testNodesQueryHealthyAndState() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); - RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes() - .get(nm1.getNodeId()); - NodeHealthStatus nodeHealth = node.getNodeHealthStatus(); - nodeHealth.setHealthReport("test health report"); - nodeHealth.setIsNodeHealthy(false); - node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth, - new ArrayList(), null, null)); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.UNHEALTHY); - - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "true") - .queryParam("state", RMNodeState.RUNNING.toString()) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); - } - - @Test - public void testNodesQueryHealthyFalse() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); - rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "false").accept("application/json") - .get(JSONObject.class); - - assertEquals("correct number of elements", 1, json.length()); - assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); - } - - @Test - public void testNodesQueryHealthyInvalid() throws JSONException, Exception { - WebResource r = resource(); - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - - try { - r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "tr").accept("application/json") - .get(JSONObject.class); - fail("should have thrown exception querying invalid healthy string"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch( - "exception message", - "java.lang.Exception: Error: You must specify either true or false to query on health", - message); - checkStringMatch("exception type", "BadRequestException", type); - checkStringMatch("exception classname", - "org.apache.hadoop.yarn.webapp.BadRequestException", classname); - - } finally { - rm.stop(); - } - } - - public void testNodesHelper(String path) throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - JSONObject json = r.path("ws").path("v1").path("cluster").path(path) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject nodes = json.getJSONObject("nodes"); - assertEquals("correct number of elements", 1, nodes.length()); - JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("correct number of elements", 2, nodeArray.length()); - JSONObject info = nodeArray.getJSONObject(0); - String id = info.get("id").toString(); - - if (id.matches("h1:1234")) { - verifyNodeInfo(info, nm1, RMNodeState.NEW); - verifyNodeInfo(nodeArray.getJSONObject(1), nm2, RMNodeState.NEW); - } else { - verifyNodeInfo(info, nm2, RMNodeState.NEW); - verifyNodeInfo(nodeArray.getJSONObject(1), nm1, RMNodeState.NEW); - } - } - - @Test - public void testSingleNode() throws JSONException, Exception { - rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - testSingleNodeHelper("h2:1235", nm2); - } - - @Test - public void testSingleNodeSlash() throws JSONException, Exception { - MockNM nm1 = rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - testSingleNodeHelper("h1:1234/", nm1); - } - - public void testSingleNodeHelper(String nodeid, MockNM nm) + public void verifyClusterSchedulerFifoGeneric(String type, String state, + float capacity, float usedCapacity, int minQueueCapacity, + int maxQueueCapacity, int numNodes, int usedNodeCapacity, + int availNodeCapacity, int totalNodeCapacity, int numContainers) throws JSONException, Exception { - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("nodes") - .path(nodeid).accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("node"); - verifyNodeInfo(info, nm, RMNodeState.NEW); - } - @Test - public void testNonexistNode() throws JSONException, Exception { - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - WebResource r = resource(); - try { - r.path("ws").path("v1").path("cluster").path("nodes") - .path("node_invalid:99").accept("application/json") - .get(JSONObject.class); + assertEquals("type doesn't match", "fifoScheduler", type); + assertEquals("qstate doesn't match", QueueState.RUNNING.toString(), state); + assertEquals("capacity doesn't match", 1.0, capacity, 0.0); + assertEquals("usedCapacity doesn't match", Float.NaN, usedCapacity, 0.0); + assertEquals("minQueueMemoryCapacity doesn't match", 1024, minQueueCapacity); + assertEquals("maxQueueMemoryCapacity doesn't match", 10240, + maxQueueCapacity); + assertEquals("numNodes doesn't match", 0, numNodes); + assertEquals("usedNodeCapacity doesn't match", 0, usedNodeCapacity); + assertEquals("availNodeCapacity doesn't match", 0, availNodeCapacity); + assertEquals("totalNodeCapacity doesn't match", 0, totalNodeCapacity); + assertEquals("numContainers doesn't match", 0, numContainers); - fail("should have thrown exception on non-existent nodeid"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); - - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch("exception message", - "java.lang.Exception: nodeId, node_invalid:99, is not found", message); - checkStringMatch("exception type", "NotFoundException", type); - checkStringMatch("exception classname", - "org.apache.hadoop.yarn.webapp.NotFoundException", classname); - - } finally { - rm.stop(); - } - } - - @Test - public void testInvalidNode() throws JSONException, Exception { - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - - WebResource r = resource(); - try { - r.path("ws").path("v1").path("cluster").path("nodes") - .path("node_invalid_foo").accept("application/json") - .get(JSONObject.class); - - fail("should have thrown exception on non-existent nodeid"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch("exception message", - "Invalid NodeId \\[node_invalid_foo\\]. Expected host:port", message); - checkStringMatch("exception type", "IllegalArgumentException", type); - checkStringMatch("exception classname", - "java.lang.IllegalArgumentException", classname); - } finally { - rm.stop(); - } - } - - public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm, - RMNodeState expectedState) throws JSONException, Exception { - assertEquals("correct number of elements", 11, nodeInfo.length()); - String state = nodeInfo.get("state").toString(); - assertTrue("stated doesn't match: " + state, - state.matches(expectedState.toString())); - String rack = nodeInfo.get("rack").toString(); - assertTrue("rack doesn't match: " + rack, rack.matches("/default-rack")); - String healthStatus = nodeInfo.get("healthStatus").toString(); - assertTrue("healthStatus doesn't match: " + healthStatus, - healthStatus.matches("Healthy")); - String id = nodeInfo.get("id").toString(); - assertTrue("id doesn't match, got: " + id + " expected: " - + nm.getNodeId().toString(), id.matches(nm.getNodeId().toString())); - String nodeHostName = nodeInfo.get("nodeHostName").toString(); - assertTrue("hostname doesn't match, got: " + nodeHostName + " expected: " - + nm.getNodeId().getHost(), - nodeHostName.matches(nm.getNodeId().getHost())); - - String nodeHTTPAddress = nodeInfo.get("nodeHTTPAddress").toString(); - String expectedHttpAddress = nm.getNodeId().getHost() + ":" - + nm.getHttpPort(); - assertTrue("nodeHTTPAddress doesn't match, got: " + nodeHTTPAddress - + " expected: " + expectedHttpAddress, - nodeHTTPAddress.matches(expectedHttpAddress)); - // could use this for other checks - RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); - long lastHealthUpdate = nodeInfo.getLong("lastHealthUpdate"); - long expectedHealthUpdate = node.getNodeHealthStatus() - .getLastHealthReportTime(); - assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate - + " expected: " + expectedHealthUpdate, expectedHealthUpdate, - lastHealthUpdate); - String healthReport = nodeInfo.get("healthReport").toString(); - assertTrue("healthReport doesn't match: " + healthReport, - healthReport.matches("Healthy")); - - int numContainers = nodeInfo.getInt("numContainers"); - assertEquals("numContainers doesn't match: " + numContainers, 0, - numContainers); - - long usedMemoryMB = nodeInfo.getLong("usedMemoryMB"); - assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, 0, usedMemoryMB); - - long availMemoryMB = nodeInfo.getLong("availMemoryMB"); - assertEquals("availMemoryMB doesn't match: " + availMemoryMB, 0, - availMemoryMB); - } - - @Test - public void testApps() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - testAppsHelper("apps", app1); - rm.stop(); - - } - - @Test - public void testAppsSlash() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - testAppsHelper("apps/", app1); - rm.stop(); - - } - - public void testAppsHelper(String path, RMApp app) throws JSONException, - Exception { - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path(path) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app); - - } - - @Test - public void testAppsQueryState() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("state", RMAppState.ACCEPTED.toString()) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1); - rm.stop(); - } - - @Test - public void testAppsQueryStateNone() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("state", RMAppState.RUNNING.toString()) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); - rm.stop(); - } - - @Test - public void testAppsQueryStateInvalid() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - try { - r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("state", "INVALID_test").accept("application/json") - .get(JSONObject.class); - fail("should have thrown exception on invalid state query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch( - "exception message", - "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test", - message); - checkStringMatch("exception type", "IllegalArgumentException", type); - checkStringMatch("exception classname", - "java.lang.IllegalArgumentException", classname); - - } finally { - rm.stop(); - } - } - - @Test - public void testAppsQueryUser() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - JSONObject json = r - .path("ws") - .path("v1") - .path("cluster") - .path("apps") - .queryParam("user", - UserGroupInformation.getCurrentUser().getShortUserName()) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 2, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryQueue() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("queue", "default").accept("application/json") - .get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 2, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryLimit() throws JSONException, Exception { - rm.start(); - rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("limit", "2").accept("application/json") - .get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 2, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryStartBegin() throws JSONException, Exception { - rm.start(); - long start = System.currentTimeMillis(); - Thread.sleep(1); - rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("startedTimeBegin", String.valueOf(start)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 3, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryStartBeginSome() throws JSONException, Exception { - rm.start(); - rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - long start = System.currentTimeMillis(); - Thread.sleep(1); - rm.submitApp(1024); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("startedTimeBegin", String.valueOf(start)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 1, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryStartEnd() throws JSONException, Exception { - rm.start(); - rm.registerNode("amNM:1234", 2048); - long end = System.currentTimeMillis(); - Thread.sleep(1); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("startedTimeEnd", String.valueOf(end)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); - rm.stop(); - } - - @Test - public void testAppsQueryStartBeginEnd() throws JSONException, Exception { - rm.start(); - rm.registerNode("amNM:1234", 2048); - long start = System.currentTimeMillis(); - Thread.sleep(1); - rm.submitApp(1024); - rm.submitApp(1024); - long end = System.currentTimeMillis(); - Thread.sleep(1); - rm.submitApp(1024); - - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("startedTimeBegin", String.valueOf(start)) - .queryParam("startedTimeEnd", String.valueOf(end)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 2, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryFinishBegin() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - long start = System.currentTimeMillis(); - Thread.sleep(1); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - rm.submitApp(1024); - rm.submitApp(1024); - - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("finishedTimeBegin", String.valueOf(start)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 1, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryFinishEnd() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - - rm.submitApp(1024); - rm.submitApp(1024); - long end = System.currentTimeMillis(); - - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("finishedTimeEnd", String.valueOf(end)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 3, array.length()); - rm.stop(); - } - - @Test - public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - long start = System.currentTimeMillis(); - Thread.sleep(1); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - - rm.submitApp(1024); - rm.submitApp(1024); - long end = System.currentTimeMillis(); - - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("finishedTimeBegin", String.valueOf(start)) - .queryParam("finishedTimeEnd", String.valueOf(end)) - .accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("correct number of elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("correct number of elements", 1, array.length()); - rm.stop(); - } - - @Test - public void testSingleApp() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - testSingleAppsHelper(app1.getApplicationId().toString(), app1); - rm.stop(); - } - - @Test - public void testSingleAppsSlash() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - RMApp app1 = rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1); - rm.stop(); - } - - @Test - public void testInvalidApp() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_invalid_12").accept("application/json") - .get(JSONObject.class); - fail("should have thrown exception on invalid appid"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch("exception message", "For input string: \"invalid\"", - message); - checkStringMatch("exception type", "NumberFormatException", type); - checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); - - } finally { - rm.stop(); - } - } - - @Test - public void testNonexistApp() throws JSONException, Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); - rm.submitApp(1024); - amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_00000_0099").accept("application/json") - .get(JSONObject.class); - fail("should have thrown exception on invalid appid"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - - assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("correct number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - checkStringMatch("exception message", - "java.lang.Exception: app with id: application_00000_0099 not found", - message); - checkStringMatch("exception type", "NotFoundException", type); - checkStringMatch("exception classname", - "org.apache.hadoop.yarn.webapp.NotFoundException", classname); - } finally { - rm.stop(); - } - } - - public void testSingleAppsHelper(String path, RMApp app) - throws JSONException, Exception { - WebResource r = resource(); - JSONObject json = r.path("ws").path("v1").path("cluster").path("apps") - .path(path).accept("application/json").get(JSONObject.class); - assertEquals("correct number of elements", 1, json.length()); - verifyAppInfo(json.getJSONObject("app"), app); - } - - public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, - Exception { - - // 15 because trackingUrl not assigned yet - assertEquals("correct number of elements", 15, info.length()); - String id = info.getString("id"); - String expectedId = app.getApplicationId().toString(); - checkStringMatch("id", expectedId, id); - - String user = info.getString("user"); - String expectedUser = app.getUser(); - checkStringMatch("user", expectedUser, user); - - checkStringMatch("name", "", info.getString("name")); - checkStringMatch("queue", "default", info.getString("queue")); - checkStringMatch("state", RMAppState.ACCEPTED.toString(), - info.getString("state")); - checkStringMatch("finalStatus", - FinalApplicationStatus.UNDEFINED.toString(), - info.getString("finalStatus")); - assertEquals("progress doesn't match", 0, info.getDouble("progress"), 0.0); - checkStringMatch("trackingUI", "UNASSIGNED", info.getString("trackingUI")); - checkStringMatch("diagnostics", "", info.getString("diagnostics")); - assertEquals("clusterId doesn't match", ResourceManager.clusterTimeStamp, - info.getLong("clusterId")); - assertEquals("startedTime doesn't match", app.getStartTime(), - info.getLong("startedTime")); - assertEquals("finishedTime doesn't match", app.getFinishTime(), - info.getLong("finishedTime")); - assertTrue("elapsed time not greater than 0", - info.getLong("elapsedTime") > 0); - checkStringMatch("amHostHttpAddress", app.getCurrentAppAttempt() - .getMasterContainer().getNodeHttpAddress(), - info.getString("amHostHttpAddress")); - assertTrue("amContainerLogs doesn't match", - info.getString("amContainerLogs").startsWith("http://")); - } - - private void checkStringMatch(String print, String expected, String got) { - assertTrue( - print + " doesn't match, got: " + got + " expected: " + expected, - got.matches(expected)); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java new file mode 100644 index 00000000000..61012aa483f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -0,0 +1,756 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.StringReader; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesApps extends JerseyTest { + + private static MockRM rm; + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + rm = new MockRM(new Configuration()); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + bind(ApplicationACLsManager.class).toInstance( + rm.getApplicationACLsManager()); + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + public TestRMWebServicesApps() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testApps() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testAppsHelper("apps", app1, MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testAppsSlash() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testAppsDefault() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testAppsHelper("apps/", app1, ""); + rm.stop(); + } + + @Test + public void testAppsXML() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesApps = dom.getElementsByTagName("apps"); + assertEquals("incorrect number of elements", 1, nodesApps.getLength()); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyAppsXML(nodes, app1); + rm.stop(); + } + + @Test + public void testAppsXMLMulti() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024, "testwordcount", "user1"); + rm.submitApp(2048, "testwordcount2", "user1"); + + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesApps = dom.getElementsByTagName("apps"); + assertEquals("incorrect number of elements", 1, nodesApps.getLength()); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 2, nodes.getLength()); + rm.stop(); + } + + public void testAppsHelper(String path, RMApp app, String media) + throws JSONException, Exception { + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path(path).accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + verifyAppInfo(array.getJSONObject(0), app); + + } + + @Test + public void testAppsQueryState() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("state", RMAppState.ACCEPTED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + verifyAppInfo(array.getJSONObject(0), app1); + rm.stop(); + } + + @Test + public void testAppsQueryStateNone() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("state", RMAppState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + rm.stop(); + } + + @Test + public void testAppsQueryStateInvalid() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("state", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid state query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + + } finally { + rm.stop(); + } + } + + @Test + public void testAppsQueryUser() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + rm.submitApp(1024); + + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r + .path("ws") + .path("v1") + .path("cluster") + .path("apps") + .queryParam("user", + UserGroupInformation.getCurrentUser().getShortUserName()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryQueue() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + rm.submitApp(1024); + + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("queue", "default") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryLimit() throws JSONException, Exception { + rm.start(); + rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + rm.submitApp(1024); + rm.submitApp(1024); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("limit", "2") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryStartBegin() throws JSONException, Exception { + rm.start(); + long start = System.currentTimeMillis(); + Thread.sleep(1); + rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + rm.submitApp(1024); + rm.submitApp(1024); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryStartBeginSome() throws JSONException, Exception { + rm.start(); + rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + rm.submitApp(1024); + long start = System.currentTimeMillis(); + Thread.sleep(1); + rm.submitApp(1024); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryStartEnd() throws JSONException, Exception { + rm.start(); + rm.registerNode("amNM:1234", 2048); + long end = System.currentTimeMillis(); + Thread.sleep(1); + rm.submitApp(1024); + rm.submitApp(1024); + rm.submitApp(1024); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("startedTimeEnd", String.valueOf(end)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + rm.stop(); + } + + @Test + public void testAppsQueryStartBeginEnd() throws JSONException, Exception { + rm.start(); + rm.registerNode("amNM:1234", 2048); + long start = System.currentTimeMillis(); + Thread.sleep(1); + rm.submitApp(1024); + rm.submitApp(1024); + long end = System.currentTimeMillis(); + Thread.sleep(1); + rm.submitApp(1024); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + .queryParam("startedTimeEnd", String.valueOf(end)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryFinishBegin() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + long start = System.currentTimeMillis(); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + rm.submitApp(1024); + rm.submitApp(1024); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("finishedTimeBegin", String.valueOf(start)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryFinishEnd() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + + rm.submitApp(1024); + rm.submitApp(1024); + long end = System.currentTimeMillis(); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("finishedTimeEnd", String.valueOf(end)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + rm.stop(); + } + + @Test + public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + long start = System.currentTimeMillis(); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + + rm.submitApp(1024); + rm.submitApp(1024); + long end = System.currentTimeMillis(); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("finishedTimeBegin", String.valueOf(start)) + .queryParam("finishedTimeEnd", String.valueOf(end)) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + rm.stop(); + } + + @Test + public void testSingleApp() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + testSingleAppsHelper(app1.getApplicationId().toString(), app1, + MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testSingleAppsSlash() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, + MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testSingleAppsDefault() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, ""); + rm.stop(); + } + + @Test + public void testInvalidApp() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid appid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "For input string: \"invalid\"", message); + WebServicesTestUtils.checkStringMatch("exception type", + "NumberFormatException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.NumberFormatException", classname); + + } finally { + rm.stop(); + } + } + + @Test + public void testNonexistApp() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .path("application_00000_0099").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid appid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "java.lang.Exception: app with id: application_00000_0099 not found", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "NotFoundException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); + } finally { + rm.stop(); + } + } + + public void testSingleAppsHelper(String path, RMApp app, String media) + throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").path(path).accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + assertEquals("incorrect number of elements", 1, json.length()); + verifyAppInfo(json.getJSONObject("app"), app); + } + + @Test + public void testSingleAppsXML() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").path(app1.getApplicationId().toString()) + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyAppsXML(nodes, app1); + rm.stop(); + } + + public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException, + Exception { + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyAppInfoGeneric(app, + WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "user"), + WebServicesTestUtils.getXmlString(element, "name"), + WebServicesTestUtils.getXmlString(element, "queue"), + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "finalStatus"), + WebServicesTestUtils.getXmlFloat(element, "progress"), + WebServicesTestUtils.getXmlString(element, "trackingUI"), + WebServicesTestUtils.getXmlString(element, "diagnostics"), + WebServicesTestUtils.getXmlLong(element, "clusterId"), + WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "finishedTime"), + WebServicesTestUtils.getXmlLong(element, "elapsedTime"), + WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), + WebServicesTestUtils.getXmlString(element, "amContainerLogs")); + } + } + + public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, + Exception { + + // 15 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 15, info.length()); + + verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), + info.getString("name"), info.getString("queue"), + info.getString("state"), info.getString("finalStatus"), + (float) info.getDouble("progress"), info.getString("trackingUI"), + info.getString("diagnostics"), info.getLong("clusterId"), + info.getLong("startedTime"), info.getLong("finishedTime"), + info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), + info.getString("amContainerLogs")); + } + + public void verifyAppInfoGeneric(RMApp app, String id, String user, + String name, String queue, String state, String finalStatus, + float progress, String trackingUI, String diagnostics, long clusterId, + long startedTime, long finishedTime, long elapsedTime, + String amHostHttpAddress, String amContainerLogs) throws JSONException, + Exception { + + WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() + .toString(), id); + WebServicesTestUtils.checkStringMatch("user", app.getUser(), user); + WebServicesTestUtils.checkStringMatch("name", app.getName(), name); + WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue); + WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), + state); + WebServicesTestUtils.checkStringMatch("finalStatus", app + .getFinalApplicationStatus().toString(), finalStatus); + assertEquals("progress doesn't match", 0, progress, 0.0); + WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", + trackingUI); + WebServicesTestUtils.checkStringMatch("diagnostics", app.getDiagnostics() + .toString(), diagnostics); + assertEquals("clusterId doesn't match", ResourceManager.clusterTimeStamp, + clusterId); + assertEquals("startedTime doesn't match", app.getStartTime(), startedTime); + assertEquals("finishedTime doesn't match", app.getFinishTime(), + finishedTime); + assertTrue("elapsed time not greater than 0", elapsedTime > 0); + WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app + .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), + amHostHttpAddress); + assertTrue("amContainerLogs doesn't match", + amContainerLogs.startsWith("http://")); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java new file mode 100644 index 00000000000..0ffe8a98e2f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -0,0 +1,316 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.StringReader; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesCapacitySched extends JerseyTest { + + private static MockRM rm; + private CapacitySchedulerConfiguration csConf; + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + csConf = new CapacitySchedulerConfiguration(); + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + setupQueueConfiguration(csConf); + rm = new MockRM(csConf); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + bind(ApplicationACLsManager.class).toInstance( + rm.getApplicationACLsManager()); + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + private static void setupQueueConfiguration( + CapacitySchedulerConfiguration conf) { + + // Define top-level queues + conf.setQueues(CapacityScheduler.ROOT, new String[] { "a", "b" }); + conf.setCapacity(CapacityScheduler.ROOT, 100); + + final String A = CapacityScheduler.ROOT + ".a"; + conf.setCapacity(A, 10); + conf.setMaximumCapacity(A, 50); + + final String B = CapacityScheduler.ROOT + ".b"; + conf.setCapacity(B, 90); + + // Define 2nd-level queues + final String A1 = A + ".a1"; + final String A2 = A + ".a2"; + conf.setQueues(A, new String[] { "a1", "a2" }); + conf.setCapacity(A1, 30); + conf.setMaximumCapacity(A1, 50); + + conf.setUserLimitFactor(A1, 100.0f); + conf.setCapacity(A2, 70); + conf.setUserLimitFactor(A2, 100.0f); + + final String B1 = B + ".b1"; + final String B2 = B + ".b2"; + final String B3 = B + ".b3"; + conf.setQueues(B, new String[] { "b1", "b2", "b3" }); + conf.setCapacity(B1, 50); + conf.setUserLimitFactor(B1, 100.0f); + conf.setCapacity(B2, 30); + conf.setUserLimitFactor(B2, 100.0f); + conf.setCapacity(B3, 20); + conf.setUserLimitFactor(B3, 100.0f); + + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + public TestRMWebServicesCapacitySched() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testClusterScheduler() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterScheduler(json); + } + + @Test + public void testClusterSchedulerSlash() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler/").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterScheduler(json); + } + + @Test + public void testClusterSchedulerDefault() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterScheduler(json); + } + + @Test + public void testClusterSchedulerXML() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler/").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList scheduler = dom.getElementsByTagName("scheduler"); + assertEquals("incorrect number of elements", 1, scheduler.getLength()); + NodeList schedulerInfo = dom.getElementsByTagName("schedulerInfo"); + assertEquals("incorrect number of elements", 1, schedulerInfo.getLength()); + verifyClusterSchedulerXML(schedulerInfo); + } + + public void verifyClusterSchedulerXML(NodeList nodes) throws Exception { + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyClusterSchedulerGeneric( + WebServicesTestUtils.getXmlAttrString(element, "xsi:type"), + WebServicesTestUtils.getXmlFloat(element, "usedCapacity"), + WebServicesTestUtils.getXmlFloat(element, "capacity"), + WebServicesTestUtils.getXmlFloat(element, "maxCapacity"), + WebServicesTestUtils.getXmlString(element, "queueName")); + + NodeList queues = element.getElementsByTagName("queues"); + for (int j = 0; j < queues.getLength(); j++) { + Element qElem = (Element) queues.item(j); + String qName = WebServicesTestUtils.getXmlString(qElem, "queueName"); + String q = CapacityScheduler.ROOT + "." + qName; + verifySubQueueXML(qElem, q); + } + } + } + + public void verifySubQueueXML(Element qElem, String q) throws Exception { + + verifySubQueueGeneric(q, + WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity"), + WebServicesTestUtils.getXmlFloat(qElem, "capacity"), + WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity"), + WebServicesTestUtils.getXmlString(qElem, "queueName"), + WebServicesTestUtils.getXmlString(qElem, "state")); + + NodeList queues = qElem.getElementsByTagName("subQueues"); + if (queues != null) { + for (int j = 0; j < queues.getLength(); j++) { + Element subqElem = (Element) queues.item(j); + String qName = WebServicesTestUtils.getXmlString(subqElem, "queueName"); + String q2 = q + "." + qName; + verifySubQueueXML(subqElem, q2); + } + } + } + + private void verifyClusterScheduler(JSONObject json) throws JSONException, + Exception { + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject info = json.getJSONObject("scheduler"); + assertEquals("incorrect number of elements", 1, info.length()); + info = info.getJSONObject("schedulerInfo"); + assertEquals("incorrect number of elements", 6, info.length()); + verifyClusterSchedulerGeneric(info.getString("type"), + (float) info.getDouble("usedCapacity"), + (float) info.getDouble("capacity"), + (float) info.getDouble("maxCapacity"), info.getString("queueName")); + + JSONArray arr = info.getJSONArray("queues"); + assertEquals("incorrect number of elements", 2, arr.length()); + + // test subqueues + for (int i = 0; i < arr.length(); i++) { + JSONObject obj = arr.getJSONObject(i); + String q = CapacityScheduler.ROOT + "." + obj.getString("queueName"); + verifySubQueue(obj, q); + } + } + + private void verifyClusterSchedulerGeneric(String type, float usedCapacity, + float capacity, float maxCapacity, String queueName) throws Exception { + + assertTrue("type doesn't match", "capacityScheduler".matches(type)); + assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f); + assertEquals("capacity doesn't match", 100, capacity, 1e-3f); + assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f); + assertTrue("queueName doesn't match", "root".matches(queueName)); + } + + private void verifySubQueue(JSONObject info, String q) throws JSONException, + Exception { + if (info.has("subQueues")) { + assertEquals("incorrect number of elements", 6, info.length()); + } else { + assertEquals("incorrect number of elements", 5, info.length()); + } + verifySubQueueGeneric(q, (float) info.getDouble("usedCapacity"), + (float) info.getDouble("capacity"), + (float) info.getDouble("maxCapacity"), info.getString("queueName"), + info.getString("state")); + + if (info.has("subQueues")) { + JSONArray arr = info.getJSONArray("subQueues"); + // test subqueues + for (int i = 0; i < arr.length(); i++) { + JSONObject obj = arr.getJSONObject(i); + String q2 = q + "." + obj.getString("queueName"); + verifySubQueue(obj, q2); + } + } + } + + private void verifySubQueueGeneric(String q, float usedCapacity, + float capacity, float maxCapacity, String qname, String state) + throws Exception { + String[] qArr = q.split("\\."); + assertTrue("q name invalid: " + q, qArr.length > 1); + String qshortName = qArr[qArr.length - 1]; + + assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f); + assertEquals("capacity doesn't match", csConf.getCapacity(q), capacity, + 1e-3f); + float expectCapacity = csConf.getMaximumCapacity(q); + if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) { + expectCapacity = 100; + } + assertEquals("maxCapacity doesn't match", expectCapacity, maxCapacity, + 1e-3f); + assertTrue("queueName doesn't match, got: " + qname + " expected: " + q, + qshortName.matches(qname)); + assertTrue("state doesn't match", + (csConf.getState(q).toString()).matches(state)); + + } +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java new file mode 100644 index 00000000000..2c2e9fd8cfd --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -0,0 +1,601 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.StringReader; +import java.util.ArrayList; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesNodes extends JerseyTest { + + private static MockRM rm; + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + rm = new MockRM(new Configuration()); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + bind(ApplicationACLsManager.class).toInstance( + rm.getApplicationACLsManager()); + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + public TestRMWebServicesNodes() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testNodes() throws JSONException, Exception { + testNodesHelper("nodes", MediaType.APPLICATION_JSON); + } + + @Test + public void testNodesSlash() throws JSONException, Exception { + testNodesHelper("nodes/", MediaType.APPLICATION_JSON); + } + + @Test + public void testNodesDefault() throws JSONException, Exception { + testNodesHelper("nodes/", ""); + } + + @Test + public void testNodesQueryState() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); + rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("state", RMNodeState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 1, nodeArray.length()); + JSONObject info = nodeArray.getJSONObject(0); + + verifyNodeInfo(info, nm1); + } + + @Test + public void testNodesQueryStateNone() throws JSONException, Exception { + WebResource r = resource(); + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes") + .queryParam("state", RMNodeState.DECOMMISSIONED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); + } + + @Test + public void testNodesQueryStateInvalid() throws JSONException, Exception { + WebResource r = resource(); + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + + try { + r.path("ws").path("v1").path("cluster").path("nodes") + .queryParam("state", "BOGUSSTATE").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + + fail("should have thrown exception querying invalid state"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState.BOGUSSTATE", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + + } finally { + rm.stop(); + } + } + + @Test + public void testNodesQueryHealthy() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); + rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("healthy", "true") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 2, nodeArray.length()); + } + + @Test + public void testNodesQueryHealthyCase() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); + rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("healthy", "TRUe") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 2, nodeArray.length()); + + } + + @Test + public void testNodesQueryHealthyAndState() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); + RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes() + .get(nm1.getNodeId()); + NodeHealthStatus nodeHealth = node.getNodeHealthStatus(); + nodeHealth.setHealthReport("test health report"); + nodeHealth.setIsNodeHealthy(false); + node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth, + new ArrayList(), null, null)); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.UNHEALTHY); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("healthy", "true") + .queryParam("state", RMNodeState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); + } + + @Test + public void testNodesQueryHealthyFalse() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm1.getNodeId(), RMNodeState.RUNNING); + rm.NMwaitForState(nm2.getNodeId(), RMNodeState.NEW); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("healthy", "false") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); + } + + @Test + public void testNodesQueryHealthyInvalid() throws JSONException, Exception { + WebResource r = resource(); + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + + try { + r.path("ws").path("v1").path("cluster").path("nodes") + .queryParam("healthy", "tr").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception querying invalid healthy string"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "java.lang.Exception: Error: You must specify either true or false to query on health", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + + } finally { + rm.stop(); + } + } + + public void testNodesHelper(String path, String media) throws JSONException, + Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path(path).accept(media).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 2, nodeArray.length()); + JSONObject info = nodeArray.getJSONObject(0); + String id = info.get("id").toString(); + + if (id.matches("h1:1234")) { + verifyNodeInfo(info, nm1); + verifyNodeInfo(nodeArray.getJSONObject(1), nm2); + } else { + verifyNodeInfo(info, nm2); + verifyNodeInfo(nodeArray.getJSONObject(1), nm1); + } + } + + @Test + public void testSingleNode() throws JSONException, Exception { + rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + testSingleNodeHelper("h2:1235", nm2, MediaType.APPLICATION_JSON); + } + + @Test + public void testSingleNodeSlash() throws JSONException, Exception { + MockNM nm1 = rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + testSingleNodeHelper("h1:1234/", nm1, MediaType.APPLICATION_JSON); + } + + @Test + public void testSingleNodeDefault() throws JSONException, Exception { + MockNM nm1 = rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + testSingleNodeHelper("h1:1234/", nm1, ""); + } + + public void testSingleNodeHelper(String nodeid, MockNM nm, String media) + throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").path(nodeid).accept(media).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject info = json.getJSONObject("node"); + verifyNodeInfo(info, nm); + } + + @Test + public void testNonexistNode() throws JSONException, Exception { + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + WebResource r = resource(); + try { + r.path("ws").path("v1").path("cluster").path("nodes") + .path("node_invalid:99").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + + fail("should have thrown exception on non-existent nodeid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch("exception message", + "java.lang.Exception: nodeId, node_invalid:99, is not found", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "NotFoundException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); + + } finally { + rm.stop(); + } + } + + @Test + public void testInvalidNode() throws JSONException, Exception { + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + + WebResource r = resource(); + try { + r.path("ws").path("v1").path("cluster").path("nodes") + .path("node_invalid_foo").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + + fail("should have thrown exception on non-existent nodeid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "Invalid NodeId \\[node_invalid_foo\\]. Expected host:port", message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + } finally { + rm.stop(); + } + } + + @Test + public void testNodesXML() throws JSONException, Exception { + rm.start(); + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + // MockNM nm2 = rm.registerNode("h2:1235", 5121); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesApps = dom.getElementsByTagName("nodes"); + assertEquals("incorrect number of elements", 1, nodesApps.getLength()); + NodeList nodes = dom.getElementsByTagName("node"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyNodesXML(nodes, nm1); + rm.stop(); + } + + @Test + public void testSingleNodesXML() throws JSONException, Exception { + rm.start(); + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + // MockNM nm2 = rm.registerNode("h2:1235", 5121); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").path("h1:1234").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("node"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyNodesXML(nodes, nm1); + rm.stop(); + } + + @Test + public void testNodes2XML() throws JSONException, Exception { + rm.start(); + WebResource r = resource(); + rm.registerNode("h1:1234", 5120); + rm.registerNode("h2:1235", 5121); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesApps = dom.getElementsByTagName("nodes"); + assertEquals("incorrect number of elements", 1, nodesApps.getLength()); + NodeList nodes = dom.getElementsByTagName("node"); + assertEquals("incorrect number of elements", 2, nodes.getLength()); + rm.stop(); + } + + public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException, + Exception { + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + verifyNodeInfoGeneric(nm, + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "rack"), + WebServicesTestUtils.getXmlString(element, "healthStatus"), + WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "nodeHostName"), + WebServicesTestUtils.getXmlString(element, "nodeHTTPAddress"), + WebServicesTestUtils.getXmlLong(element, "lastHealthUpdate"), + WebServicesTestUtils.getXmlString(element, "healthReport"), + WebServicesTestUtils.getXmlInt(element, "numContainers"), + WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"), + WebServicesTestUtils.getXmlLong(element, "availMemoryMB")); + } + } + + public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm) + throws JSONException, Exception { + assertEquals("incorrect number of elements", 11, nodeInfo.length()); + + verifyNodeInfoGeneric(nm, nodeInfo.getString("state"), + nodeInfo.getString("rack"), nodeInfo.getString("healthStatus"), + nodeInfo.getString("id"), nodeInfo.getString("nodeHostName"), + nodeInfo.getString("nodeHTTPAddress"), + nodeInfo.getLong("lastHealthUpdate"), + nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"), + nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB")); + + } + + public void verifyNodeInfoGeneric(MockNM nm, String state, String rack, + String healthStatus, String id, String nodeHostName, + String nodeHTTPAddress, long lastHealthUpdate, String healthReport, + int numContainers, long usedMemoryMB, long availMemoryMB) + throws JSONException, Exception { + + RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); + NodeHealthStatus health = node.getNodeHealthStatus(); + ResourceScheduler sched = rm.getResourceScheduler(); + SchedulerNodeReport report = sched.getNodeReport(nm.getNodeId()); + + WebServicesTestUtils.checkStringMatch("state", node.getState().toString(), + state); + WebServicesTestUtils.checkStringMatch("rack", node.getRackName(), rack); + WebServicesTestUtils.checkStringMatch("healthStatus", "Healthy", + healthStatus); + WebServicesTestUtils.checkStringMatch("id", nm.getNodeId().toString(), id); + WebServicesTestUtils.checkStringMatch("nodeHostName", nm.getNodeId() + .getHost(), nodeHostName); + WebServicesTestUtils.checkStringMatch("healthReport", + String.valueOf(health.getHealthReport()), healthReport); + String expectedHttpAddress = nm.getNodeId().getHost() + ":" + + nm.getHttpPort(); + WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", + expectedHttpAddress, nodeHTTPAddress); + + long expectedHealthUpdate = health.getLastHealthReportTime(); + assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + + " expected: " + expectedHealthUpdate, expectedHealthUpdate, + lastHealthUpdate); + + if (report != null) { + assertEquals("numContainers doesn't match: " + numContainers, + report.getNumContainers(), numContainers); + assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report + .getUsedResource().getMemory(), usedMemoryMB); + assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report + .getAvailableResource().getMemory(), availMemoryMB); + } + } + +}