HDFS-11419. Performance analysis of new DFSNetworkTopology#chooseRandom. Contributed by Chen Liang.
This commit is contained in:
Arpit Agarwal
parent
ca6bcc3c76
commit
d0f346af26
|
|
@ -0,0 +1,524 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs.net;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.StorageType;
|
||||||
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
|
||||||
|
import org.apache.hadoop.net.NetworkTopology;
|
||||||
|
import org.apache.hadoop.net.Node;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Ignore;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.text.NumberFormat;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertNotNull;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performance test of the new DFSNetworkTopology chooseRandom.
|
||||||
|
*
|
||||||
|
* NOTE that the tests are not for correctness but for performance comparison,
|
||||||
|
* so the tests are printing and writing down values rather than doing assertion
|
||||||
|
* checks or timeout checks. Therefore, it is pointless to run these
|
||||||
|
* tests without something reading the value. So disabled the tests to for now,
|
||||||
|
* anyone interested in looking at the numbers can enable them.
|
||||||
|
*/
|
||||||
|
@Ignore
|
||||||
|
public class TestDFSNetworkTopologyPerformance {
|
||||||
|
public static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(TestDFSNetworkTopologyPerformance.class);
|
||||||
|
private static NetworkTopology cluster;
|
||||||
|
private static DFSNetworkTopology dfscluster;
|
||||||
|
private DatanodeDescriptor[] dataNodes;
|
||||||
|
|
||||||
|
private final static int NODE_NUM = 2000;
|
||||||
|
private final static int OP_NUM = 20000;
|
||||||
|
|
||||||
|
private final static int L1_NUM = 5;
|
||||||
|
private final static int L2_NUM = 10;
|
||||||
|
private final static int L3_NUM = 10;
|
||||||
|
|
||||||
|
private final static float NS_TO_MS = 1000000;
|
||||||
|
|
||||||
|
private final static Random RANDOM = new Random();
|
||||||
|
|
||||||
|
private Node node;
|
||||||
|
private long totalStart;
|
||||||
|
private long totalEnd;
|
||||||
|
private int totalTrials;
|
||||||
|
private float totalMs;
|
||||||
|
private Set<Node> excluded;
|
||||||
|
private static String[] racks;
|
||||||
|
private static String[] hosts;
|
||||||
|
private static StorageType[] types;
|
||||||
|
|
||||||
|
private static long[] records;
|
||||||
|
private long localStart;
|
||||||
|
private long localEnd;
|
||||||
|
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
racks = new String[NODE_NUM];
|
||||||
|
hosts = new String[NODE_NUM];
|
||||||
|
types = new StorageType[NODE_NUM];
|
||||||
|
records = new long[OP_NUM];
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
racks[i] = getRandLocation();
|
||||||
|
hosts[i] = "host" + i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws Exception {
|
||||||
|
cluster = NetworkTopology.getInstance(new Configuration());
|
||||||
|
dfscluster = DFSNetworkTopology.getInstance(new Configuration());
|
||||||
|
excluded = new HashSet<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In this test, all the node types are uniformly distributed. i.e. a node
|
||||||
|
* has equal of being DISK, RAM_DISK, SSD and ARCHIVE. This test did two tests
|
||||||
|
* first test runs the old chooseRandom approach, the second test runs the
|
||||||
|
* new chooseRandom approach.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testUniformStorageType() throws Exception {
|
||||||
|
EnumSet<StorageType> missingTypes = EnumSet.allOf(StorageType.class);
|
||||||
|
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
types[i] = getRandType();
|
||||||
|
missingTypes.remove(types);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (missingTypes.size() != 0) {
|
||||||
|
// it is statistically very, very rare that missingTypes is not length 0
|
||||||
|
// at this point. But if it happened, we do the trick by inserting 1 per
|
||||||
|
// remaining types randomly into the array, such that it is guaranteed all
|
||||||
|
// types exist
|
||||||
|
Set<Integer> usedIdx = new HashSet<>();
|
||||||
|
int idx;
|
||||||
|
for (StorageType type : missingTypes) {
|
||||||
|
do {
|
||||||
|
idx = RANDOM.nextInt(NODE_NUM);
|
||||||
|
} while (usedIdx.contains(idx));
|
||||||
|
usedIdx.add(idx);
|
||||||
|
types[idx] = type;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addNodeByTypes(types);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("before test1");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
totalTrials = 0;
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
StorageType type = StorageType.values()[i%StorageType.values().length];
|
||||||
|
// mimic the behaviour of current code:
|
||||||
|
// 1. chooseRandom on NetworkTopology
|
||||||
|
// 2. if it satisfies, we are good, break
|
||||||
|
// 3. if not, add to excluded, try again
|
||||||
|
// The reason of doing simulated behaviour is that in NetworkTopology,
|
||||||
|
// the code that does this is a wrapper method that is fairly complex
|
||||||
|
// and does something more than just chooseRandom. So it would be unfair
|
||||||
|
// to the current code if we compare the wrapper with the new chooseRandom
|
||||||
|
// because we should only compare the two chooseRandom methods.
|
||||||
|
// However the way current chooseRandom works involves checking storage
|
||||||
|
// type of return nodes. We still need to simulate this behaviour.
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
do {
|
||||||
|
totalTrials += 1;
|
||||||
|
node = cluster.chooseRandom("", excluded);
|
||||||
|
assertNotNull(node);
|
||||||
|
if (isType(node, type)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
excluded.add(node);
|
||||||
|
} while (true);
|
||||||
|
excluded.clear();
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
// 4 trials on average.
|
||||||
|
LOG.info("total time: {} avg time: {} avg trials: {}",
|
||||||
|
totalMs, totalMs / OP_NUM, (float)totalTrials / OP_NUM);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("after test1 before test2");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
StorageType type = StorageType.values()[i%StorageType.values().length];
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
node = dfscluster.chooseRandomWithStorageType("", excluded, type);
|
||||||
|
assertNotNull(node);
|
||||||
|
// with dfs cluster, the returned is always already the required type;
|
||||||
|
// add assertion mainly to make a more fair comparison
|
||||||
|
assertTrue(isType(node, type));
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
LOG.info("total time: {} avg time: {}", totalMs, totalMs / OP_NUM);
|
||||||
|
|
||||||
|
printMemUsage("after test2");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* There are two storage types DISK and ARCHIVE. DISK compose the majority and
|
||||||
|
* ARCHIVE only compose 1/20 nodes, uniformly distributed. This test also runs
|
||||||
|
* two tests, one with old approach and one with the new approach. Try to
|
||||||
|
* search for ARCHIVE type devices. This test show how new approach can
|
||||||
|
* outperform the old one.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testUnbalancedStorageType() throws Exception {
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
types[i] = StorageType.DISK;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < NODE_NUM/20; i++) {
|
||||||
|
types[RANDOM.nextInt(NODE_NUM)] = StorageType.ARCHIVE;
|
||||||
|
}
|
||||||
|
addNodeByTypes(types);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("before test1");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
totalTrials = 0;
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
// mimic the behaviour of current code:
|
||||||
|
// 1. chooseRandom on NetworkTopology
|
||||||
|
// 2. if it satisfies, we are good, break
|
||||||
|
// 3. if not, add to excluded, try again
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
do {
|
||||||
|
totalTrials += 1;
|
||||||
|
node = cluster.chooseRandom("", excluded);
|
||||||
|
assertNotNull(node);
|
||||||
|
if (isType(node, StorageType.ARCHIVE)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
excluded.add(node);
|
||||||
|
} while (true);
|
||||||
|
excluded.clear();
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
// on average it takes 20 trials
|
||||||
|
LOG.info("total time: {} avg time: {} avg trials: {}",
|
||||||
|
totalMs, totalMs / OP_NUM, (float)totalTrials / OP_NUM);
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("after test1 before test2");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
node = dfscluster.chooseRandomWithStorageType("", excluded,
|
||||||
|
StorageType.ARCHIVE);
|
||||||
|
assertNotNull(node);
|
||||||
|
// with dfs cluster, the returned is always already the required type;
|
||||||
|
// add assertion mainly to make a more fair comparison
|
||||||
|
assertTrue(isType(node, StorageType.ARCHIVE));
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
LOG.info("total time: {} avg time: {}", totalMs, totalMs/OP_NUM);
|
||||||
|
|
||||||
|
printMemUsage("after test2");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* There is only one storage type (DISK) in the cluster. And tries to select
|
||||||
|
* a DISK devices every time also. One approach will always succeed in one
|
||||||
|
* call. This test shows how bad the new approach is compared to the optimal
|
||||||
|
* situation.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testSameStorageType() throws Exception {
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
types[i] = StorageType.DISK;
|
||||||
|
}
|
||||||
|
addNodeByTypes(types);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("before test1");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
totalTrials = 0;
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
// mimic the behaviour of current code:
|
||||||
|
// 1. chooseRandom on NetworkTopology
|
||||||
|
// 2. if it satisfies, we are good, break
|
||||||
|
// 3. if not, add to excluded, try again
|
||||||
|
do {
|
||||||
|
totalTrials += 1;
|
||||||
|
node = cluster.chooseRandom("", excluded);
|
||||||
|
assertNotNull(node);
|
||||||
|
if (isType(node, StorageType.DISK)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
excluded.add(node);
|
||||||
|
} while (true);
|
||||||
|
excluded.clear();
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
// on average it takes 20 trials
|
||||||
|
LOG.info("total time: {} avg time: {} avg trials: {}",
|
||||||
|
totalMs, totalMs / OP_NUM, (float)totalTrials / OP_NUM);
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("after test1 before test2");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
node = dfscluster.chooseRandomWithStorageType("", excluded,
|
||||||
|
StorageType.DISK);
|
||||||
|
assertNotNull(node);
|
||||||
|
// with dfs cluster, the returned is always already the required type;
|
||||||
|
// add assertion mainly to make a more fair comparison
|
||||||
|
assertTrue(isType(node, StorageType.DISK));
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart) / NS_TO_MS;
|
||||||
|
LOG.info("total time: {} avg time: {}", totalMs, totalMs / OP_NUM);
|
||||||
|
|
||||||
|
printMemUsage("after test2");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean coinFlip(double chance) {
|
||||||
|
return RANDOM.nextInt(NODE_NUM) <= NODE_NUM*chance;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is a helper test, can be changed to different distribution each run.
|
||||||
|
* Changing the value percentage = X where X is between 0.0 to 1.0 will result
|
||||||
|
* in different outcomes. This is to help understanding what is the boundary
|
||||||
|
* that makes the new approach better than the old one. The lower X is, the
|
||||||
|
* less likely the old approach will succeed in one call, in which case the
|
||||||
|
* new approach is more likely to be better.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testPercentageStorageType() throws Exception {
|
||||||
|
double percentage = 0.9;
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
if (coinFlip(percentage)) {
|
||||||
|
types[i] = StorageType.ARCHIVE;
|
||||||
|
} else {
|
||||||
|
types[i] = StorageType.DISK;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
addNodeByTypes(types);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("before test1");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
totalTrials = 0;
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
// mimic the behaviour of current code:
|
||||||
|
// 1. chooseRandom on NetworkTopology
|
||||||
|
// 2. if it satisfies, we are good, break
|
||||||
|
// 3. if not, add to excluded, try again
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
do {
|
||||||
|
totalTrials += 1;
|
||||||
|
node = cluster.chooseRandom("", excluded);
|
||||||
|
assertNotNull(node);
|
||||||
|
if (isType(node, StorageType.ARCHIVE)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
excluded.add(node);
|
||||||
|
} while (true);
|
||||||
|
excluded.clear();
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart) / NS_TO_MS;
|
||||||
|
// on average it takes 20 trials
|
||||||
|
LOG.info("total time: {} avg time: {} avg trials: {}",
|
||||||
|
totalMs, totalMs / OP_NUM, (float)totalTrials / OP_NUM);
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("after test1 before test2");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
node = dfscluster.chooseRandomWithStorageType("", excluded,
|
||||||
|
StorageType.ARCHIVE);
|
||||||
|
assertNotNull(node);
|
||||||
|
// with dfs cluster, the returned is always already the required type;
|
||||||
|
// add assertion mainly to make a more fair comparison
|
||||||
|
assertTrue(isType(node, StorageType.ARCHIVE));
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart) / NS_TO_MS;
|
||||||
|
LOG.info("total time: {} avg time: {}", totalMs, totalMs / OP_NUM);
|
||||||
|
|
||||||
|
printMemUsage("after test2");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Similar to the previous test, change the percentage value to understand
|
||||||
|
* the performance of the mixed approach. More specifically, this test takes
|
||||||
|
* the approach that, it uses old approach for the first try, only if the
|
||||||
|
* old approach failed in the first try, it makes another call with the
|
||||||
|
* new approach. There is no comparison within this test.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testPercentageStorageTypeWithMixedTopology() throws Exception {
|
||||||
|
double percentage = 0.9;
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
if (coinFlip(percentage)) {
|
||||||
|
types[i] = StorageType.ARCHIVE;
|
||||||
|
} else {
|
||||||
|
types[i] = StorageType.DISK;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
addNodeByTypes(types);
|
||||||
|
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("before test1");
|
||||||
|
|
||||||
|
totalStart = System.nanoTime();
|
||||||
|
totalTrials = 0;
|
||||||
|
for (int i = 0; i < OP_NUM; i++) {
|
||||||
|
// mimic the behavior of current code:
|
||||||
|
// 1. chooseRandom on NetworkTopology
|
||||||
|
// 2. if it satisfies, we are good, do the next operation
|
||||||
|
// 3. if not, chooseRandom on DFSNetworkTopology
|
||||||
|
localStart = System.nanoTime();
|
||||||
|
|
||||||
|
totalTrials += 1;
|
||||||
|
node = cluster.chooseRandom("", excluded);
|
||||||
|
assertNotNull(node);
|
||||||
|
if (!isType(node, StorageType.ARCHIVE)) {
|
||||||
|
totalTrials += 1;
|
||||||
|
excluded.add(node);
|
||||||
|
node = dfscluster.chooseRandomWithStorageType("", excluded,
|
||||||
|
StorageType.ARCHIVE);
|
||||||
|
}
|
||||||
|
assertTrue(isType(node, StorageType.ARCHIVE));
|
||||||
|
|
||||||
|
excluded.clear();
|
||||||
|
localEnd = System.nanoTime();
|
||||||
|
records[i] = localEnd - localStart;
|
||||||
|
}
|
||||||
|
totalEnd = System.nanoTime();
|
||||||
|
totalMs = (totalEnd - totalStart)/NS_TO_MS;
|
||||||
|
LOG.info("total time: {} avg time: {} avg trials: {}",
|
||||||
|
totalMs, totalMs / OP_NUM, (float)totalTrials / OP_NUM);
|
||||||
|
// wait a bit for things to become stable
|
||||||
|
Thread.sleep(1000);
|
||||||
|
printMemUsage("test StorageType with mixed topology.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Print out the memory usage statistics. Note that this is an estimation by
|
||||||
|
* Java Runtime. Should not take the actual value too serious but more focus
|
||||||
|
* on the relative changes.
|
||||||
|
* @param message a prefix message
|
||||||
|
* @throws Exception throws exception
|
||||||
|
*/
|
||||||
|
private void printMemUsage(String message) throws Exception {
|
||||||
|
Runtime runtime = Runtime.getRuntime();
|
||||||
|
NumberFormat format = NumberFormat.getInstance();
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append(message);
|
||||||
|
|
||||||
|
long maxMemory = runtime.maxMemory();
|
||||||
|
long allocatedMemory = runtime.totalMemory();
|
||||||
|
long freeMemory = runtime.freeMemory();
|
||||||
|
|
||||||
|
sb.append("\nfree memory: " + format.format(freeMemory / 1024));
|
||||||
|
sb.append("\nallocated memory: " + format.format(allocatedMemory / 1024));
|
||||||
|
sb.append("\nmax memory: " + format.format(maxMemory / 1024));
|
||||||
|
sb.append("\ntotal free memory: " + format.format(
|
||||||
|
(freeMemory + (maxMemory - allocatedMemory)) / 1024));
|
||||||
|
LOG.info(sb.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addNodeByTypes(StorageType[] allTypes) {
|
||||||
|
DatanodeStorageInfo[] storages =
|
||||||
|
DFSTestUtil.createDatanodeStorageInfos(
|
||||||
|
NODE_NUM, racks, hosts, allTypes);
|
||||||
|
dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
|
||||||
|
for (int i = 0; i < NODE_NUM; i++) {
|
||||||
|
cluster.add(dataNodes[i]);
|
||||||
|
dfscluster.add(dataNodes[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getRandLocation() {
|
||||||
|
int l1 = RANDOM.nextInt(L1_NUM) + 1;
|
||||||
|
int l2 = RANDOM.nextInt(L2_NUM) + 1;
|
||||||
|
int l3 = RANDOM.nextInt(L3_NUM) + 1;
|
||||||
|
return String.format("/%d/%d/%d", l1, l2, l3);
|
||||||
|
}
|
||||||
|
|
||||||
|
private StorageType getRandType() {
|
||||||
|
int len = StorageType.values().length;
|
||||||
|
return StorageType.values()[RANDOM.nextInt(len)];
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isType(Node n, StorageType type) {
|
||||||
|
// no need to check n == null, because it's done by the caller
|
||||||
|
if (!(n instanceof DatanodeDescriptor)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
DatanodeDescriptor dnDescriptor = (DatanodeDescriptor)n;
|
||||||
|
return dnDescriptor.hasStorageType(type);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue