Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819) 

Reviewed-by: Viraj Jasani <vjasani@apache.org>
Reviewed-by: Akira Ajisaka <aajisaka@apache.org>
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
(cherry picked from commit 900682e712)
(cherry picked from commit 4aa9698ee1e9a10a0f882229036bbb4db1ede939)
This commit is contained in:
jianghuazhu 2022-04-18 11:21:51 +08:00 committed by Wei-Chiu Chuang
parent 248c548616
commit ed83a278c2
No known key found for this signature in database
GPG Key ID: B362E1C021854B9D
2 changed files with 32 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
View File

@ -77,6 +77,10 @@ public class FileNameGenerator {
return fn; return fn;
} }
public long getFileCount() {
return fileCount;
}
private synchronized void reset() { private synchronized void reset() {
Arrays.fill(pathIndecies, -1); Arrays.fill(pathIndecies, -1);
fileCount = 0L; fileCount = 0L;

40
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -570,12 +570,20 @@ public class NNThroughputBenchmark implements Tool {
// int generatedFileIdx = 0; // int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName()); LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
fileNames = new String[numThreads][]; fileNames = new String[numThreads][];
for(int idx=0; idx < numThreads; idx++) { try {
int threadOps = opsPerThread[idx]; for(int idx=0; idx < numThreads; idx++) {
fileNames[idx] = new String[threadOps]; int threadOps = opsPerThread[idx];
for(int jdx=0; jdx < threadOps; jdx++) fileNames[idx] = new String[threadOps];
fileNames[idx][jdx] = nameGenerator. for(int jdx=0; jdx < threadOps; jdx++) {
getNextFileName("ThroughputBench"); fileNames[idx][jdx] = nameGenerator.
getNextFileName("ThroughputBench");
}
}
} catch (ArrayIndexOutOfBoundsException e) {
LOG.error("The current environment allows {} files to be created. " +
"If you want to test more files, please update the -filesPerDir parameter.",
nameGenerator.getFileCount());
throw e;
} }
} }
@ -669,12 +677,20 @@ public class NNThroughputBenchmark implements Tool {
false); false);
LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName()); LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
dirPaths = new String[numThreads][]; dirPaths = new String[numThreads][];
for(int idx=0; idx < numThreads; idx++) { try {
int threadOps = opsPerThread[idx]; for(int idx=0; idx < numThreads; idx++) {
dirPaths[idx] = new String[threadOps]; int threadOps = opsPerThread[idx];
for(int jdx=0; jdx < threadOps; jdx++) dirPaths[idx] = new String[threadOps];
dirPaths[idx][jdx] = nameGenerator. for(int jdx=0; jdx < threadOps; jdx++) {
getNextFileName("ThroughputBench"); dirPaths[idx][jdx] = nameGenerator.
getNextFileName("ThroughputBench");
}
}
} catch (ArrayIndexOutOfBoundsException e) {
LOG.error("The current environment allows {} directories to be created. " +
"If you want to test more directories, please update the -dirsPerDir parameter.",
nameGenerator.getFileCount());
throw e;
} }
} }