Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15652. Make block size from NNThroughputBenchmark configurable (#2416) 

(cherry picked from commit 8a6d5b9151)
This commit is contained in:
Hui Fei 2020-10-28 09:13:25 +08:00 committed by Konstantin V Shvachko
parent 7b3a6e96d9
commit 77d9c6d0f7
2 changed files with 36 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -168,6 +168,7 @@ public class NNThroughputBenchmark implements Tool {
protected final String baseDir; protected final String baseDir;
protected short replication; protected short replication;
protected int blockSize;
protected int numThreads = 0; // number of threads protected int numThreads = 0; // number of threads
protected int numOpsRequired = 0; // number of operations requested protected int numOpsRequired = 0; // number of operations requested
protected int numOpsExecuted = 0; // number of operations executed protected int numOpsExecuted = 0; // number of operations executed
@ -229,6 +230,7 @@ public class NNThroughputBenchmark implements Tool {
OperationStatsBase() { OperationStatsBase() {
baseDir = BASE_DIR_NAME + "/" + getOpName(); baseDir = BASE_DIR_NAME + "/" + getOpName();
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3); replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
blockSize = config.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
numOpsRequired = 10; numOpsRequired = 10;
numThreads = 3; numThreads = 3;
logLevel = Level.ERROR; logLevel = Level.ERROR;
@ -516,7 +518,8 @@ public class NNThroughputBenchmark implements Tool {
// Operation types // Operation types
static final String OP_CREATE_NAME = "create"; static final String OP_CREATE_NAME = "create";
static final String OP_CREATE_USAGE = static final String OP_CREATE_USAGE =
"-op create [-threads T] [-files N] [-filesPerDir P] [-close]"; "-op create [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-close]";
protected FileNameGenerator nameGenerator; protected FileNameGenerator nameGenerator;
protected String[][] fileNames; protected String[][] fileNames;
@ -541,6 +544,9 @@ public class NNThroughputBenchmark implements Tool {
if(args.get(i).equals("-files")) { if(args.get(i).equals("-files")) {
if(i+1 == args.size()) printUsage(); if(i+1 == args.size()) printUsage();
numOpsRequired = Integer.parseInt(args.get(++i)); numOpsRequired = Integer.parseInt(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-threads")) { } else if(args.get(i).equals("-threads")) {
if(i+1 == args.size()) printUsage(); if(i+1 == args.size()) printUsage();
numThreads = Integer.parseInt(args.get(++i)); numThreads = Integer.parseInt(args.get(++i));
@ -596,7 +602,7 @@ public class NNThroughputBenchmark implements Tool {
clientProto.create(fileNames[daemonId][inputIdx], FsPermission.getDefault(), clientProto.create(fileNames[daemonId][inputIdx], FsPermission.getDefault(),
clientName, new EnumSetWritable<CreateFlag>(EnumSet clientName, new EnumSetWritable<CreateFlag>(EnumSet
.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, .of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true,
replication, BLOCK_SIZE, CryptoProtocolVersion.supported()); replication, blockSize, CryptoProtocolVersion.supported());
long end = Time.now(); long end = Time.now();
for(boolean written = !closeUponCreate; !written; for(boolean written = !closeUponCreate; !written;
written = clientProto.complete(fileNames[daemonId][inputIdx], written = clientProto.complete(fileNames[daemonId][inputIdx],
@ -716,7 +722,8 @@ public class NNThroughputBenchmark implements Tool {
// Operation types // Operation types
static final String OP_OPEN_NAME = "open"; static final String OP_OPEN_NAME = "open";
static final String OP_USAGE_ARGS = static final String OP_USAGE_ARGS =
" [-threads T] [-files N] [-filesPerDir P] [-useExisting]"; " [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-useExisting]";
static final String OP_OPEN_USAGE = static final String OP_OPEN_USAGE =
"-op " + OP_OPEN_NAME + OP_USAGE_ARGS; "-op " + OP_OPEN_NAME + OP_USAGE_ARGS;
@ -748,6 +755,7 @@ public class NNThroughputBenchmark implements Tool {
"-op", "create", "-op", "create",
"-threads", String.valueOf(this.numThreads), "-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired), "-files", String.valueOf(numOpsRequired),
"-blockSize", String.valueOf(blockSize),
"-filesPerDir", "-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()), String.valueOf(nameGenerator.getFilesPerDirectory()),
"-close"}; "-close"};
@ -778,7 +786,8 @@ public class NNThroughputBenchmark implements Tool {
long executeOp(int daemonId, int inputIdx, String ignore) long executeOp(int daemonId, int inputIdx, String ignore)
throws IOException { throws IOException {
long start = Time.now(); long start = Time.now();
clientProto.getBlockLocations(fileNames[daemonId][inputIdx], 0L, BLOCK_SIZE); clientProto.getBlockLocations(fileNames[daemonId][inputIdx], 0L,
blockSize);
long end = Time.now(); long end = Time.now();
return end-start; return end-start;
} }
@ -1068,7 +1077,7 @@ public class NNThroughputBenchmark implements Tool {
static final String OP_BLOCK_REPORT_NAME = "blockReport"; static final String OP_BLOCK_REPORT_NAME = "blockReport";
static final String OP_BLOCK_REPORT_USAGE = static final String OP_BLOCK_REPORT_USAGE =
"-op blockReport [-datanodes T] [-reports N] " + "-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F]"; "[-blocksPerReport B] [-blocksPerFile F] [-blockSize S]";
private int blocksPerReport; private int blocksPerReport;
private int blocksPerFile; private int blocksPerFile;
@ -1115,6 +1124,9 @@ public class NNThroughputBenchmark implements Tool {
} else if(args.get(i).equals("-blocksPerFile")) { } else if(args.get(i).equals("-blocksPerFile")) {
if(i+1 == args.size()) printUsage(); if(i+1 == args.size()) printUsage();
blocksPerFile = Integer.parseInt(args.get(++i)); blocksPerFile = Integer.parseInt(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(!ignoreUnrelatedOptions) } else if(!ignoreUnrelatedOptions)
printUsage(); printUsage();
} }
@ -1145,7 +1157,7 @@ public class NNThroughputBenchmark implements Tool {
String fileName = nameGenerator.getNextFileName("ThroughputBench"); String fileName = nameGenerator.getNextFileName("ThroughputBench");
clientProto.create(fileName, FsPermission.getDefault(), clientName, clientProto.create(fileName, FsPermission.getDefault(), clientName,
new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, replication, new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, replication,
BLOCK_SIZE, CryptoProtocolVersion.supported()); blockSize, CryptoProtocolVersion.supported());
ExtendedBlock lastBlock = addBlocks(fileName, clientName); ExtendedBlock lastBlock = addBlocks(fileName, clientName);
clientProto.complete(fileName, clientName, lastBlock, HdfsConstants.GRANDFATHER_INODE_ID); clientProto.complete(fileName, clientName, lastBlock, HdfsConstants.GRANDFATHER_INODE_ID);
} }
@ -1256,8 +1268,9 @@ public class NNThroughputBenchmark implements Tool {
class ReplicationStats extends OperationStatsBase { class ReplicationStats extends OperationStatsBase {
static final String OP_REPLICATION_NAME = "replication"; static final String OP_REPLICATION_NAME = "replication";
static final String OP_REPLICATION_USAGE = static final String OP_REPLICATION_USAGE =
"-op replication [-datanodes T] [-nodesToDecommission D] " + "-op replication [-datanodes T] [-nodesToDecommission D] " +
"[-nodeReplicationLimit C] [-totalBlocks B] [-replication R]"; "[-nodeReplicationLimit C] [-totalBlocks B] [-blockSize S] "
+ "[-replication R]";
private final BlockReportStats blockReportObject; private final BlockReportStats blockReportObject;
private int numDatanodes; private int numDatanodes;
@ -1282,10 +1295,11 @@ public class NNThroughputBenchmark implements Tool {
/ (numDatanodes*numDatanodes); / (numDatanodes*numDatanodes);
String[] blkReportArgs = { String[] blkReportArgs = {
"-op", "blockReport", "-op", "blockReport",
"-datanodes", String.valueOf(numDatanodes), "-datanodes", String.valueOf(numDatanodes),
"-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes), "-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes),
"-blocksPerFile", String.valueOf(numDatanodes)}; "-blocksPerFile", String.valueOf(numDatanodes),
"-blockSize", String.valueOf(blockSize)};
blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs)); blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs));
numDecommissionedBlocks = 0; numDecommissionedBlocks = 0;
numPendingBlocks = 0; numPendingBlocks = 0;
@ -1315,6 +1329,9 @@ public class NNThroughputBenchmark implements Tool {
} else if(args.get(i).equals("-replication")) { } else if(args.get(i).equals("-replication")) {
if(i+1 == args.size()) printUsage(); if(i+1 == args.size()) printUsage();
replication = Short.parseShort(args.get(++i)); replication = Short.parseShort(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(!ignoreUnrelatedOptions) } else if(!ignoreUnrelatedOptions)
printUsage(); printUsage();
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View File

@ -49,6 +49,7 @@ public class TestNNThroughputBenchmark {
@Test @Test
public void testNNThroughput() throws Exception { public void testNNThroughput() throws Exception {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
nameDir.getAbsolutePath()); nameDir.getAbsolutePath());
@ -63,6 +64,7 @@ public class TestNNThroughputBenchmark {
@Test(timeout = 120000) @Test(timeout = 120000)
public void testNNThroughputWithFsOption() throws Exception { public void testNNThroughputWithFsOption() throws Exception {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
nameDir.getAbsolutePath()); nameDir.getAbsolutePath());
@ -84,6 +86,7 @@ public class TestNNThroughputBenchmark {
cluster.waitActive(); cluster.waitActive();
final Configuration benchConf = new HdfsConfiguration(); final Configuration benchConf = new HdfsConfiguration();
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
FileSystem.setDefaultUri(benchConf, cluster.getURI()); FileSystem.setDefaultUri(benchConf, cluster.getURI());
NNThroughputBenchmark.runBenchmark(benchConf, new String[]{"-op", "all"}); NNThroughputBenchmark.runBenchmark(benchConf, new String[]{"-op", "all"});
} finally { } finally {
@ -101,12 +104,15 @@ public class TestNNThroughputBenchmark {
public void testNNThroughputRemoteAgainstNNWithFsOption() throws Exception { public void testNNThroughputRemoteAgainstNNWithFsOption() throws Exception {
final Configuration conf = new HdfsConfiguration(); final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
MiniDFSCluster cluster = null; MiniDFSCluster cluster = null;
try { try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive(); cluster.waitActive();
NNThroughputBenchmark.runBenchmark(new HdfsConfiguration(), final Configuration benchConf = new HdfsConfiguration();
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
NNThroughputBenchmark.runBenchmark(benchConf,
new String[]{"-fs", cluster.getURI().toString(), "-op", "all"}); new String[]{"-fs", cluster.getURI().toString(), "-op", "all"});
} finally { } finally {
if (cluster != null) { if (cluster != null) {