Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15098. TestClusterTopology#testChooseRandom fails intermittently. Contributed by Zsolt Venczel.

This commit is contained in:
Sean Mackrory 2017-12-07 10:52:02 -07:00
parent 67662e2ac9
commit acb92904d0
1 changed files with 35 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
View File

@ -139,12 +139,18 @@ public class TestClusterTopology extends Assert {
NodeElement node4 = getNewNode("node4", "/d1/r3"); NodeElement node4 = getNewNode("node4", "/d1/r3");
cluster.add(node4); cluster.add(node4);
// Number of test runs
int numTestRuns = 3;
int chiSquareTestRejectedCounter = 0;
// Number of iterations to do the test // Number of iterations to do the test
int numIterations = 100; int numIterations = 100;
for (int testRun = 0; testRun < numTestRuns; ++testRun) {
// Pick random nodes // Pick random nodes
HashMap<String,Integer> histogram = new HashMap<String,Integer>(); HashMap<String, Integer> histogram = new HashMap<String, Integer>();
for (int i=0; i<numIterations; i++) { for (int i = 0; i < numIterations; i++) {
String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName(); String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName();
if (!histogram.containsKey(randomNode)) { if (!histogram.containsKey(randomNode)) {
histogram.put(randomNode, 0); histogram.put(randomNode, 0);
@ -153,11 +159,11 @@ public class TestClusterTopology extends Assert {
} }
assertEquals("Random is not selecting all nodes", 4, histogram.size()); assertEquals("Random is not selecting all nodes", 4, histogram.size());
// Check with 99% confidence (alpha=0.01 as confidence = (100 * (1 - alpha) // Check with 99% confidence alpha=0.01 as confidence = 100 * (1 - alpha)
ChiSquareTest chiSquareTest = new ChiSquareTest(); ChiSquareTest chiSquareTest = new ChiSquareTest();
double[] expected = new double[histogram.size()]; double[] expected = new double[histogram.size()];
long[] observed = new long[histogram.size()]; long[] observed = new long[histogram.size()];
int j=0; int j = 0;
for (Integer occurrence : histogram.values()) { for (Integer occurrence : histogram.values()) {
expected[j] = 1.0 * numIterations / histogram.size(); expected[j] = 1.0 * numIterations / histogram.size();
observed[j] = occurrence; observed[j] = occurrence;
@ -166,11 +172,17 @@ public class TestClusterTopology extends Assert {
boolean chiSquareTestRejected = boolean chiSquareTestRejected =
chiSquareTest.chiSquareTest(expected, observed, 0.01); chiSquareTest.chiSquareTest(expected, observed, 0.01);
if (chiSquareTestRejected) {
++chiSquareTestRejectedCounter;
}
}
// Check that they have the proper distribution // Check that they have the proper distribution
assertFalse("Not choosing nodes randomly", chiSquareTestRejected); assertFalse("Random not choosing nodes with proper distribution",
chiSquareTestRejectedCounter==3);
// Pick random nodes excluding the 2 nodes in /d1/r3 // Pick random nodes excluding the 2 nodes in /d1/r3
histogram = new HashMap<String,Integer>(); HashMap<String, Integer> histogram = new HashMap<String, Integer>();
for (int i=0; i<numIterations; i++) { for (int i=0; i<numIterations; i++) {
String randomNode = cluster.chooseRandom("~/d1/r3").getName(); String randomNode = cluster.chooseRandom("~/d1/r3").getName();
if (!histogram.containsKey(randomNode)) { if (!histogram.containsKey(randomNode)) {