From 70d6537d398f4e5524a428b0d206555dd0b54c88 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sun, 25 Sep 2011 09:51:09 +0000 Subject: [PATCH] Merge -r 1175356:1175357 from trunk to branch-0.23 to fix MAPREDUCE-3053. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1175359 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 6 ++++ .../yarn/ipc/ProtoOverHadoopRpcEngine.java | 6 ++++ .../java/org/apache/hadoop/yarn/TestRPC.java | 32 +++++++++++++++++++ .../resourcemanager/rmnode/RMNodeImpl.java | 23 +++++++++++-- .../resourcemanager/TestResourceManager.java | 21 ++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dc795fe17cd..85d073f7de2 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1377,6 +1377,12 @@ Release 0.23.0 - Unreleased MAPREDUCE-2691. Increase threadpool size for launching containers in MapReduce ApplicationMaster. (vinodkv via acmurthy) + MAPREDUCE-2990. Fixed display of NodeHealthStatus. (Subroto Sanyal via + acmurthy) + + MAPREDUCE-3053. Better diagnostic message for unknown methods in ProtoBuf + RPCs. (vinodkv via acmurthy) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java index 399a275e5d2..c94a31871ad 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java @@ -329,6 +329,12 @@ public class ProtoOverHadoopRpcEngine implements RpcEngine { + methodName); MethodDescriptor methodDescriptor = service.getDescriptorForType() .findMethodByName(methodName); + if (methodDescriptor == null) { + String msg = "Unknown method " + methodName + " called on " + + protocol + " protocol."; + LOG.warn(msg); + return handleException(new IOException(msg)); + } Message prototype = service.getRequestPrototype(methodDescriptor); Message param = prototype.newBuilderForType() .mergeFrom(rpcRequest.getRequestProto()).build(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 58efcc42307..332d044158b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -25,9 +25,11 @@ import junit.framework.Assert; import org.apache.avro.ipc.Server; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; @@ -47,6 +49,7 @@ import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvid import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.Records; import org.junit.Test; public class TestRPC { @@ -65,6 +68,35 @@ public class TestRPC { // test(HadoopYarnRPC.class.getName()); // } + @Test + public void testUnknownCall() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class + .getName()); + YarnRPC rpc = YarnRPC.create(conf); + String bindAddr = "localhost:0"; + InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); + Server server = rpc.getServer(ContainerManager.class, + new DummyContainerManager(), addr, conf, null, 1); + server.start(); + + // Any unrelated protocol would do + ClientRMProtocol proxy = (ClientRMProtocol) rpc.getProxy( + ClientRMProtocol.class, NetUtils.createSocketAddr("localhost:" + + server.getPort()), conf); + + try { + proxy.getNewApplicationId(Records + .newRecord(GetNewApplicationIdRequest.class)); + Assert.fail("Excepted RPC call to fail with unknown method."); + } catch (YarnRemoteException e) { + Assert.assertEquals("Unknown method getNewApplicationId called on " + + "org.apache.hadoop.yarn.proto.ClientRMProtocol" + + "$ClientRMProtocolService$BlockingInterface protocol.", e + .getMessage()); + } + } + @Test public void testHadoopProtoRPC() throws Exception { test(HadoopYarnProtoRPC.class.getName()); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index dd8d7f840f7..81de047bc04 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -147,6 +147,7 @@ public class RMNodeImpl implements RMNode, EventHandler { this.httpAddress = hostName + ":" + httpPort;; this.node = node; this.nodeHealthStatus.setIsNodeHealthy(true); + this.nodeHealthStatus.setHealthReport("Healthy"); this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis()); this.latestHeartBeatResponse.setResponseId(0); @@ -222,6 +223,18 @@ public class RMNodeImpl implements RMNode, EventHandler { } } + private void setNodeHealthStatus(NodeHealthStatus status) + { + this.writeLock.lock(); + try { + this.nodeHealthStatus.setHealthReport(status.getHealthReport()); + this.nodeHealthStatus.setIsNodeHealthy(status.getIsNodeHealthy()); + this.nodeHealthStatus.setLastHealthReportTime(status.getLastHealthReportTime()); + } finally { + this.writeLock.unlock(); + } + } + @Override public RMNodeState getState() { this.readLock.lock(); @@ -345,7 +358,10 @@ public class RMNodeImpl implements RMNode, EventHandler { // Switch the last heartbeatresponse. rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse(); - if (!statusEvent.getNodeHealthStatus().getIsNodeHealthy()) { + NodeHealthStatus remoteNodeHealthStatus = + statusEvent.getNodeHealthStatus(); + rmNode.setNodeHealthStatus(remoteNodeHealthStatus); + if (!remoteNodeHealthStatus.getIsNodeHealthy()) { // Inform the scheduler rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); @@ -392,8 +408,9 @@ public class RMNodeImpl implements RMNode, EventHandler { // Switch the last heartbeatresponse. rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse(); - - if (statusEvent.getNodeHealthStatus().getIsNodeHealthy()) { + NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus(); + rmNode.setNodeHealthStatus(remoteNodeHealthStatus); + if (remoteNodeHealthStatus.getIsNodeHealthy()) { rmNode.context.getDispatcher().getEventHandler().handle( new NodeAddedSchedulerEvent(rmNode)); return RMNodeState.RUNNING; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 60a227bc6d4..536aa672d79 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -18,12 +18,16 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; +import java.util.Collection; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; @@ -153,6 +157,23 @@ public class TestResourceManager { LOG.info("--- END: testResourceAllocation ---"); } + + @Test + public void testNodeHealthReportIsNotNull() throws Exception{ + String host1 = "host1"; + final int memory = 4 * 1024; + org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 = + registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory); + nm1.heartbeat(); + nm1.heartbeat(); + Collection values = resourceManager.getRMContext().getRMNodes().values(); + for (RMNode ni : values) + { + NodeHealthStatus nodeHealthStatus = ni.getNodeHealthStatus(); + String healthReport = nodeHealthStatus.getHealthReport(); + assertNotNull(healthReport); + } + } private void checkResourceUsage( org.apache.hadoop.yarn.server.resourcemanager.NodeManager... nodes ) {