From c71d86cd7c343f7d12de254030c5169ba3ed6a85 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Sun, 16 Jun 2013 22:20:45 +0000 Subject: [PATCH] MAPREDUCE-5171. Expose blacklisted nodes from the MR AM REST API. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1493603 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapreduce/v2/app/AppContext.java | 3 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 7 +++ .../v2/app/rm/RMContainerRequestor.java | 4 ++ .../v2/app/webapp/AMWebServices.java | 9 +++ .../app/webapp/dao/BlacklistedNodesInfo.java | 42 +++++++++++++ .../mapreduce/v2/app/MockAppContext.java | 13 +++- .../v2/app/TestRuntimeEstimators.java | 6 ++ .../v2/app/webapp/TestAMWebServices.java | 59 ++++++++++++++++++- .../hadoop/mapreduce/v2/hs/JobHistory.java | 7 +++ 9 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/BlacklistedNodesInfo.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java index 075f3f546a0..8ff47e6423c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.v2.app; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -56,4 +57,6 @@ public interface AppContext { Clock getClock(); ClusterInfo getClusterInfo(); + + Set getBlacklistedNodes(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index a0e3a98e90b..6c912cdc0c8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -29,6 +29,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.io.IOUtils; @@ -95,6 +96,7 @@ import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent; import org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator; import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator; +import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor; import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.app.speculate.DefaultSpeculator; import org.apache.hadoop.mapreduce.v2.app.speculate.Speculator; @@ -937,6 +939,11 @@ public class MRAppMaster extends CompositeService { public ClusterInfo getClusterInfo() { return this.clusterInfo; } + + @Override + public Set getBlacklistedNodes() { + return ((RMContainerRequestor) containerAllocator).getBlacklistedNodes(); + } } @SuppressWarnings("unchecked") diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 43d3abf3d08..cc34eb0923f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -424,4 +424,8 @@ public abstract class RMContainerRequestor extends RMCommunicator { hosts, orig.racks, orig.priority); return newReq; } + + public Set getBlacklistedNodes() { + return blacklistedNodes; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java index 1a35027b681..d02eb1827f9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java @@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptInfo; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptsInfo; +import org.apache.hadoop.mapreduce.v2.app.webapp.dao.BlacklistedNodesInfo; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfInfo; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobCounterInfo; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobInfo; @@ -216,6 +217,14 @@ public class AMWebServices { init(); return new AppInfo(this.app, this.app.context); } + + @GET + @Path("/blacklistednodes") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public BlacklistedNodesInfo getBlacklistedNodes() { + init(); + return new BlacklistedNodesInfo(this.app.context); + } @GET @Path("/jobs") diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/BlacklistedNodesInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/BlacklistedNodesInfo.java new file mode 100644 index 00000000000..a67bef04928 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/BlacklistedNodesInfo.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.v2.app.webapp.dao; + +import java.util.Set; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.mapreduce.v2.app.AppContext; + +@XmlRootElement(name = "blacklistednodesinfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class BlacklistedNodesInfo { + private Set blacklistedNodes; + + public BlacklistedNodesInfo() { } + + public BlacklistedNodesInfo(AppContext appContext) { + blacklistedNodes = appContext.getBlacklistedNodes(); + } + + public Set getBlacklistedNodes() { + return blacklistedNodes; + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java index 8cb1eac9a93..944bbe37462 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.v2.app; import java.util.Map; +import java.util.Set; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -37,7 +38,8 @@ public class MockAppContext implements AppContext { final String user = MockJobs.newUserName(); final Map jobs; final long startTime = System.currentTimeMillis(); - + Set blacklistedNodes; + public MockAppContext(int appid) { appID = MockJobs.newAppID(appid); appAttemptID = ApplicationAttemptId.newInstance(appID, 0); @@ -115,4 +117,13 @@ public class MockAppContext implements AppContext { return null; } + @Override + public Set getBlacklistedNodes() { + return blacklistedNodes; + } + + public void setBlacklistedNodes(Set blacklistedNodes) { + this.blacklistedNodes = blacklistedNodes; + } + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index f7cdd4fedfc..1daacf8c368 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -850,5 +851,10 @@ public class TestRuntimeEstimators { public ClusterInfo getClusterInfo() { return new ClusterInfo(); } + + @Override + public Set getBlacklistedNodes() { + return null; + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java index 477f537afd1..21a2611daac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.StringReader; +import java.util.Set; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; @@ -33,6 +34,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.MockAppContext; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Before; @@ -42,6 +44,7 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; +import com.google.common.collect.Sets; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.servlet.GuiceServletContextListener; @@ -64,13 +67,15 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestAMWebServices extends JerseyTest { private static Configuration conf = new Configuration(); - private static AppContext appContext; + private static MockAppContext appContext; private Injector injector = Guice.createInjector(new ServletModule() { @Override protected void configureServlets() { appContext = new MockAppContext(0, 1, 1, 1); + appContext.setBlacklistedNodes(Sets.newHashSet("badnode1", "badnode2")); + bind(JAXBContextResolver.class); bind(AMWebServices.class); bind(GenericExceptionHandler.class); @@ -240,6 +245,29 @@ public class TestAMWebServices extends JerseyTest { "error string exists and shouldn't", "", responseStr); } } + + @Test + public void testBlacklistedNodes() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("mapreduce") + .path("blacklistednodes").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + verifyBlacklistedNodesInfo(json, appContext); + } + + @Test + public void testBlacklistedNodesXML() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("mapreduce") + .path("blacklistednodes").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + verifyBlacklistedNodesInfoXML(xml, appContext); + } public void verifyAMInfo(JSONObject info, AppContext ctx) throws JSONException { @@ -285,4 +313,33 @@ public class TestAMWebServices extends JerseyTest { assertTrue("elapsedTime not greater then 0", (elapsedTime > 0)); } + + public void verifyBlacklistedNodesInfo(JSONObject blacklist, AppContext ctx) + throws JSONException, Exception{ + JSONArray array = blacklist.getJSONArray("blacklistedNodes"); + assertEquals(array.length(), ctx.getBlacklistedNodes().size()); + for (int i = 0; i < array.length(); i++) { + assertTrue(ctx.getBlacklistedNodes().contains(array.getString(i))); + } + } + + public void verifyBlacklistedNodesInfoXML(String xml, AppContext ctx) + throws JSONException, Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList infonodes = dom.getElementsByTagName("blacklistednodesinfo"); + assertEquals("incorrect number of elements", 1, infonodes.getLength()); + NodeList nodes = dom.getElementsByTagName("blacklistedNodes"); + Set blacklistedNodes = ctx.getBlacklistedNodes(); + assertEquals("incorrect number of elements", blacklistedNodes.size(), + nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + assertTrue( + blacklistedNodes.contains(element.getFirstChild().getNodeValue())); + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java index fb0856ee3eb..feb6a7c61ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java @@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.hs; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -301,4 +302,10 @@ public class JobHistory extends AbstractService implements HistoryContext { public ClusterInfo getClusterInfo() { return null; } + + // TODO AppContext - Not Required + @Override + public Set getBlacklistedNodes() { + return null; + } }