From ed83a278c2860b35e537533c37a4d1b801ea3dd5 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 18 Apr 2022 11:21:51 +0800 Subject: [PATCH] HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819) Reviewed-by: Viraj Jasani Reviewed-by: Akira Ajisaka Signed-off-by: Wei-Chiu Chuang (cherry picked from commit 900682e7120e7af250ebf0b4bafb40b412d3e11c) (cherry picked from commit 4aa9698ee1e9a10a0f882229036bbb4db1ede939) --- .../server/namenode/FileNameGenerator.java | 4 ++ .../namenode/NNThroughputBenchmark.java | 40 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java index 6f7087b9215..ed18696c232 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java @@ -77,6 +77,10 @@ public class FileNameGenerator { return fn; } + public long getFileCount() { + return fileCount; + } + private synchronized void reset() { Arrays.fill(pathIndecies, -1); fileCount = 0L; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index f192d9ae2fb..f71ef71d99e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -570,12 +570,20 @@ public class NNThroughputBenchmark implements Tool { // int generatedFileIdx = 0; LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName()); fileNames = new String[numThreads][]; - for(int idx=0; idx < numThreads; idx++) { - int threadOps = opsPerThread[idx]; - fileNames[idx] = new String[threadOps]; - for(int jdx=0; jdx < threadOps; jdx++) - fileNames[idx][jdx] = nameGenerator. - getNextFileName("ThroughputBench"); + try { + for(int idx=0; idx < numThreads; idx++) { + int threadOps = opsPerThread[idx]; + fileNames[idx] = new String[threadOps]; + for(int jdx=0; jdx < threadOps; jdx++) { + fileNames[idx][jdx] = nameGenerator. + getNextFileName("ThroughputBench"); + } + } + } catch (ArrayIndexOutOfBoundsException e) { + LOG.error("The current environment allows {} files to be created. " + + "If you want to test more files, please update the -filesPerDir parameter.", + nameGenerator.getFileCount()); + throw e; } } @@ -669,12 +677,20 @@ public class NNThroughputBenchmark implements Tool { false); LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName()); dirPaths = new String[numThreads][]; - for(int idx=0; idx < numThreads; idx++) { - int threadOps = opsPerThread[idx]; - dirPaths[idx] = new String[threadOps]; - for(int jdx=0; jdx < threadOps; jdx++) - dirPaths[idx][jdx] = nameGenerator. - getNextFileName("ThroughputBench"); + try { + for(int idx=0; idx < numThreads; idx++) { + int threadOps = opsPerThread[idx]; + dirPaths[idx] = new String[threadOps]; + for(int jdx=0; jdx < threadOps; jdx++) { + dirPaths[idx][jdx] = nameGenerator. + getNextFileName("ThroughputBench"); + } + } + } catch (ArrayIndexOutOfBoundsException e) { + LOG.error("The current environment allows {} directories to be created. " + + "If you want to test more directories, please update the -dirsPerDir parameter.", + nameGenerator.getFileCount()); + throw e; } }