From ab850368e897b9634f95a6fb6341fcceb22c18bf Mon Sep 17 00:00:00 2001 From: Luke Lu Date: Thu, 4 Apr 2013 07:07:59 +0000 Subject: [PATCH] HADOOP-9451. Fault single-layer config if node group topology is enabled. (Junping Du via llu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1464301 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../net/NetworkTopologyWithNodeGroup.java | 7 +- .../net/TestNetworkTopologyWithNodeGroup.java | 200 ++++++++++++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3329baf1506..41964745aef 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -165,6 +165,9 @@ Trunk (Unreleased) BUG FIXES + HADOOP-9451. Fault single-layer config if node group topology is enabled. + (Junping Du via llu) + HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang) HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java index 1bc21450e60..86920f08b83 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java @@ -191,7 +191,12 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology { } rack = getNode(nodeGroup.getNetworkLocation()); - if (rack != null && !(rack instanceof InnerNode)) { + // rack should be an innerNode and with parent. + // note: rack's null parent case is: node's topology only has one layer, + // so rack is recognized as "/" and no parent. + // This will be recognized as a node with fault topology. + if (rack != null && + (!(rack instanceof InnerNode) || rack.getParent() == null)) { throw new IllegalArgumentException("Unexpected data node " + node.toString() + " at an illegal network location"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java new file mode 100644 index 00000000000..2b6ce622cf8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +public class TestNetworkTopologyWithNodeGroup { + private final static NetworkTopologyWithNodeGroup cluster = new + NetworkTopologyWithNodeGroup(); + + private final static NodeBase dataNodes[] = new NodeBase[] { + new NodeBase("h1", "/d1/r1/s1"), + new NodeBase("h2", "/d1/r1/s1"), + new NodeBase("h3", "/d1/r1/s2"), + new NodeBase("h4", "/d1/r2/s3"), + new NodeBase("h5", "/d1/r2/s3"), + new NodeBase("h6", "/d1/r2/s4"), + new NodeBase("h7", "/d2/r3/s5"), + new NodeBase("h8", "/d2/r3/s6") + }; + + private final static NodeBase computeNode = new NodeBase("/d1/r1/s1/h9"); + + private final static NodeBase rackOnlyNode = new NodeBase("h10", "/r2"); + + static { + for(int i=0; i pickNodesAtRandom(int numNodes, + String excludedScope) { + Map frequency = new HashMap(); + for (NodeBase dnd : dataNodes) { + frequency.put(dnd, 0); + } + + for (int j = 0; j < numNodes; j++) { + Node random = cluster.chooseRandom(excludedScope); + frequency.put(random, frequency.get(random) + 1); + } + return frequency; + } + + /** + * This test checks that chooseRandom works for an excluded node. + */ + /** + * Test replica placement policy in case last node is invalid. + * We create 6 nodes but the last node is in fault topology (with rack info), + * so cannot be added to cluster. We should test proper exception is thrown in + * adding node but shouldn't affect the cluster. + */ + @Test + public void testChooseRandomExcludedNode() { + String scope = "~" + NodeBase.getPath(dataNodes[0]); + Map frequency = pickNodesAtRandom(100, scope); + + for (Node key : dataNodes) { + // all nodes except the first should be more than zero + assertTrue(frequency.get(key) > 0 || key == dataNodes[0]); + } + } + + /** + * This test checks that adding a node with invalid topology will be failed + * with an exception to show topology is invalid. + */ + @Test + public void testAddNodeWithInvalidTopology() { + // The last node is a node with invalid topology + try { + cluster.add(rackOnlyNode); + fail("Exception should be thrown, so we should not have reached here."); + } catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + fail("Expecting IllegalArgumentException, but caught:" + e); + } + assertTrue(e.getMessage().contains("illegal network location")); + } + } + +}