From c7d64769c1a72f2aab9958e33a7cf28d02612306 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Sat, 4 Feb 2012 20:03:37 +0000 Subject: [PATCH] MAPREDUCE-3759. ClassCastException thrown in -list-active-trackers when there are a few unhealthy nodes (vinodkv via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1240598 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../resourcemanager/ClientRMService.java | 9 +- .../resourcemanager/ResourceManager.java | 2 +- .../resourcemanager/TestClientRMService.java | 82 +++++++++++++++++++ 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ef296700493..6e7550562ca 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -696,6 +696,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3795. "job -status" command line output is malformed. (vinodkv via mahadev) + MAPREDUCE-3759. ClassCastException thrown in -list-active-trackers when + there are a few unhealthy nodes (vinodkv via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index f02ae5148bf..40fdfc02fe0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.Text; @@ -75,7 +76,6 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; -import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -157,6 +157,11 @@ public void start() { super.start(); } + @Private + public InetSocketAddress getBindAddress() { + return clientBindAddress; + } + /** * check if the calling user has the access to application information. * @param callerUGI @@ -412,7 +417,7 @@ private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); - Resource used = Resources.none(); + Resource used = BuilderUtils.newResource(0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index cda23f7a5bb..dc7d29cdcbc 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -110,7 +110,7 @@ public class ResourceManager extends CompositeService implements Recoverable { protected ApplicationACLsManager applicationACLsManager; protected RMDelegationTokenSecretManager rmDTSecretManager; private WebApp webApp; - private RMContext rmContext; + protected RMContext rmContext; private final Store store; protected ResourceTrackerService resourceTracker; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java new file mode 100644 index 00000000000..5ea88814c17 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.net.InetSocketAddress; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.Records; +import org.junit.Test; + +public class TestClientRMService { + + private static final Log LOG = LogFactory.getLog(TestClientRMService.class); + + @Test + public void testGetClusterNodes() throws Exception { + MockRM rm = new MockRM() { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, + this.rmAppManager, this.applicationACLsManager, + this.rmDTSecretManager); + }; + }; + rm.start(); + + // Add a healthy node + MockNM node = rm.registerNode("host:1234", 1024); + node.nodeHeartbeat(true); + + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ClientRMProtocol client = + (ClientRMProtocol) rpc + .getProxy(ClientRMProtocol.class, rmAddress, conf); + + // Make call + GetClusterNodesRequest request = + Records.newRecord(GetClusterNodesRequest.class); + List nodeReports = + client.getClusterNodes(request).getNodeReports(); + Assert.assertEquals(1, nodeReports.size()); + Assert.assertTrue("Node is expected to be healthy!", nodeReports.get(0) + .getNodeHealthStatus().getIsNodeHealthy()); + + // Now make the node unhealthy. + node.nodeHeartbeat(false); + + // Call again + nodeReports = client.getClusterNodes(request).getNodeReports(); + Assert.assertEquals(1, nodeReports.size()); + Assert.assertFalse("Node is expected to be unhealthy!", nodeReports.get(0) + .getNodeHealthStatus().getIsNodeHealthy()); + } +}