HBASE-5808 Remove TestHLogBench, HLogPerformanceEvaluation is more comprehensive
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1327143 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
ea65619d35
commit
1e7ca13fda
|
|
@ -1,346 +0,0 @@
|
||||||
/**
|
|
||||||
* Copyright 2010 The Apache Software Foundation
|
|
||||||
*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hbase.regionserver.wal;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Random;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.impl.Log4JLogger;
|
|
||||||
import org.apache.hadoop.hbase.*;
|
|
||||||
import org.apache.log4j.Level;
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
|
|
||||||
import org.apache.hadoop.util.Tool;
|
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.conf.Configured;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
|
||||||
import org.apache.hadoop.hbase.ipc.HBaseRPC;
|
|
||||||
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.experimental.categories.Category;
|
|
||||||
|
|
||||||
@Category(MediumTests.class)
|
|
||||||
public class TestHLogBench extends Configured implements Tool {
|
|
||||||
|
|
||||||
static final Log LOG = LogFactory.getLog(TestHLogBench.class);
|
|
||||||
private static final Random r = new Random();
|
|
||||||
|
|
||||||
private static final byte [] FAMILY = Bytes.toBytes("hlogbenchFamily");
|
|
||||||
|
|
||||||
// accumulate time here
|
|
||||||
private static int totalTime = 0;
|
|
||||||
private static Object lock = new Object();
|
|
||||||
|
|
||||||
// the file system where to create the Hlog file
|
|
||||||
protected FileSystem fs;
|
|
||||||
|
|
||||||
// the number of threads and the number of iterations per thread
|
|
||||||
private int numThreads = 300;
|
|
||||||
private int numIterationsPerThread = 10000;
|
|
||||||
|
|
||||||
private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
|
||||||
private Path regionRootDir =TEST_UTIL.getDataTestDir("TestHLogBench") ;
|
|
||||||
|
|
||||||
private boolean appendNoSync = false;
|
|
||||||
|
|
||||||
public TestHLogBench() {
|
|
||||||
this(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestHLogBench(Configuration conf) {
|
|
||||||
super(conf);
|
|
||||||
fs = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize file system object
|
|
||||||
*/
|
|
||||||
public void init() throws IOException {
|
|
||||||
getConf().setQuietMode(true);
|
|
||||||
if (this.fs == null) {
|
|
||||||
this.fs = FileSystem.get(getConf());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Close down file system
|
|
||||||
*/
|
|
||||||
public void close() throws IOException {
|
|
||||||
if (fs != null) {
|
|
||||||
fs.close();
|
|
||||||
fs = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The main run method of TestHLogBench
|
|
||||||
*/
|
|
||||||
public int run(String argv[]) throws Exception {
|
|
||||||
|
|
||||||
int exitCode = -1;
|
|
||||||
int i = 0;
|
|
||||||
|
|
||||||
// verify that we have enough command line parameters
|
|
||||||
if (argv.length < 4) {
|
|
||||||
printUsage("");
|
|
||||||
return exitCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
// initialize LogBench
|
|
||||||
try {
|
|
||||||
init();
|
|
||||||
} catch (HBaseRPC.VersionMismatch v) {
|
|
||||||
LOG.warn("Version Mismatch between client and server" +
|
|
||||||
"... command aborted.");
|
|
||||||
return exitCode;
|
|
||||||
} catch (IOException e) {
|
|
||||||
LOG.warn("Bad connection to FS. command aborted.");
|
|
||||||
return exitCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (; i < argv.length; i++) {
|
|
||||||
if ("-numThreads".equals(argv[i])) {
|
|
||||||
i++;
|
|
||||||
this.numThreads = Integer.parseInt(argv[i]);
|
|
||||||
} else if ("-numIterationsPerThread".equals(argv[i])) {
|
|
||||||
i++;
|
|
||||||
this.numIterationsPerThread = Integer.parseInt(argv[i]);
|
|
||||||
} else if ("-path".equals(argv[i])) {
|
|
||||||
// get an absolute path using the default file system
|
|
||||||
i++;
|
|
||||||
this.regionRootDir = new Path(argv[i]);
|
|
||||||
this.regionRootDir = regionRootDir.makeQualified(this.fs);
|
|
||||||
} else if ("-nosync".equals(argv[i])) {
|
|
||||||
this.appendNoSync = true;
|
|
||||||
} else {
|
|
||||||
printUsage(argv[i]);
|
|
||||||
return exitCode;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (NumberFormatException nfe) {
|
|
||||||
LOG.warn("Illegal numThreads or numIterationsPerThread, " +
|
|
||||||
" a positive integer expected");
|
|
||||||
throw nfe;
|
|
||||||
}
|
|
||||||
go();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void go() throws IOException, InterruptedException {
|
|
||||||
|
|
||||||
long start = System.currentTimeMillis();
|
|
||||||
log("Running TestHLogBench with " + numThreads + " threads each doing " +
|
|
||||||
numIterationsPerThread + " HLog appends " +
|
|
||||||
(appendNoSync ? "nosync" : "sync") +
|
|
||||||
" at rootDir " + regionRootDir);
|
|
||||||
|
|
||||||
// Mock an HRegion
|
|
||||||
byte [] tableName = Bytes.toBytes("table");
|
|
||||||
byte [][] familyNames = new byte [][] { FAMILY };
|
|
||||||
HTableDescriptor htd = new HTableDescriptor();
|
|
||||||
htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1")));
|
|
||||||
HRegion region = mockRegion(tableName, familyNames, regionRootDir);
|
|
||||||
HLog hlog = region.getLog();
|
|
||||||
|
|
||||||
// Spin up N threads to each perform M log operations
|
|
||||||
LogWriter [] incrementors = new LogWriter[numThreads];
|
|
||||||
for (int i=0; i<numThreads; i++) {
|
|
||||||
incrementors[i] = new LogWriter(region, tableName, hlog, i,
|
|
||||||
numIterationsPerThread,
|
|
||||||
appendNoSync);
|
|
||||||
incrementors[i].start();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for threads to finish
|
|
||||||
for (int i=0; i<numThreads; i++) {
|
|
||||||
//log("Waiting for #" + i + " to finish");
|
|
||||||
incrementors[i].join();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output statistics
|
|
||||||
long totalOps = numThreads * numIterationsPerThread;
|
|
||||||
log("Operations per second " + ((totalOps * 1000L)/totalTime));
|
|
||||||
log("Average latency in ms " + ((totalTime * 1000L)/totalOps));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Displays format of commands.
|
|
||||||
*/
|
|
||||||
private static void printUsage(String cmd) {
|
|
||||||
String prefix = "Usage: java " + TestHLogBench.class.getSimpleName();
|
|
||||||
System.err.println(prefix + cmd +
|
|
||||||
" [-numThreads <number>] " +
|
|
||||||
" [-numIterationsPerThread <number>] " +
|
|
||||||
" [-path <path where region's root directory is created>]" +
|
|
||||||
" [-nosync]");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A thread that writes data to an HLog
|
|
||||||
*/
|
|
||||||
public static class LogWriter extends Thread {
|
|
||||||
|
|
||||||
private final HRegion region;
|
|
||||||
private final int threadNumber;
|
|
||||||
private final int numIncrements;
|
|
||||||
private final HLog hlog;
|
|
||||||
private boolean appendNoSync;
|
|
||||||
private byte[] tableName;
|
|
||||||
|
|
||||||
private int count;
|
|
||||||
|
|
||||||
public LogWriter(HRegion region, byte[] tableName,
|
|
||||||
HLog log, int threadNumber,
|
|
||||||
int numIncrements, boolean appendNoSync) {
|
|
||||||
this.region = region;
|
|
||||||
this.threadNumber = threadNumber;
|
|
||||||
this.numIncrements = numIncrements;
|
|
||||||
this.hlog = log;
|
|
||||||
this.count = 0;
|
|
||||||
this.appendNoSync = appendNoSync;
|
|
||||||
this.tableName = tableName;
|
|
||||||
setDaemon(true);
|
|
||||||
//log("LogWriter[" + threadNumber + "] instantiated");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
long now = System.currentTimeMillis();
|
|
||||||
byte [] key = Bytes.toBytes("thisisakey");
|
|
||||||
KeyValue kv = new KeyValue(key, now);
|
|
||||||
WALEdit walEdit = new WALEdit();
|
|
||||||
walEdit.add(kv);
|
|
||||||
HRegionInfo hri = region.getRegionInfo();
|
|
||||||
HTableDescriptor htd = new HTableDescriptor();
|
|
||||||
htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1")));
|
|
||||||
boolean isMetaRegion = false;
|
|
||||||
long start = System.currentTimeMillis();
|
|
||||||
for (int i=0; i<numIncrements; i++) {
|
|
||||||
try {
|
|
||||||
if (appendNoSync) {
|
|
||||||
hlog.appendNoSync(hri, tableName, walEdit,
|
|
||||||
HConstants.DEFAULT_CLUSTER_ID, now, htd);
|
|
||||||
} else {
|
|
||||||
hlog.append(hri, tableName, walEdit, now, htd);
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
log("Fatal exception: " + e);
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
long tot = System.currentTimeMillis() - start;
|
|
||||||
synchronized (lock) {
|
|
||||||
totalTime += tot; // update global statistics
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void log(String string) {
|
|
||||||
LOG.info(string);
|
|
||||||
}
|
|
||||||
|
|
||||||
private byte[][] makeBytes(int numArrays, int arraySize) {
|
|
||||||
byte [][] bytes = new byte[numArrays][];
|
|
||||||
for (int i=0; i<numArrays; i++) {
|
|
||||||
bytes[i] = new byte[arraySize];
|
|
||||||
r.nextBytes(bytes[i]);
|
|
||||||
}
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a dummy region
|
|
||||||
*/
|
|
||||||
private HRegion mockRegion(byte[] tableName, byte[][] familyNames,
|
|
||||||
Path rootDir) throws IOException {
|
|
||||||
|
|
||||||
HBaseTestingUtility htu = new HBaseTestingUtility();
|
|
||||||
Configuration conf = htu.getConfiguration();
|
|
||||||
conf.setBoolean("hbase.rs.cacheblocksonwrite", true);
|
|
||||||
conf.setBoolean("hbase.hregion.use.incrementnew", true);
|
|
||||||
conf.setBoolean("dfs.support.append", true);
|
|
||||||
FileSystem fs = FileSystem.get(conf);
|
|
||||||
int numQualifiers = 10;
|
|
||||||
byte [][] qualifiers = new byte [numQualifiers][];
|
|
||||||
for (int i=0; i<numQualifiers; i++) qualifiers[i] = Bytes.toBytes("qf" + i);
|
|
||||||
int numRows = 10;
|
|
||||||
byte [][] rows = new byte [numRows][];
|
|
||||||
for (int i=0; i<numRows; i++) rows[i] = Bytes.toBytes("r" + i);
|
|
||||||
|
|
||||||
// switch off debug message from Region server
|
|
||||||
((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.WARN);
|
|
||||||
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(tableName);
|
|
||||||
for (byte [] family : familyNames)
|
|
||||||
htd.addFamily(new HColumnDescriptor(family));
|
|
||||||
|
|
||||||
HRegionInfo hri = new HRegionInfo(tableName, Bytes.toBytes(0L),
|
|
||||||
Bytes.toBytes(0xffffffffL));
|
|
||||||
if (fs.exists(rootDir)) {
|
|
||||||
if (!fs.delete(rootDir, true)) {
|
|
||||||
throw new IOException("Failed delete of " + rootDir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return HRegion.createHRegion(hri, rootDir, conf, htd);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testLogPerformance() throws Exception {
|
|
||||||
TestHLogBench bench = new TestHLogBench();
|
|
||||||
int res;
|
|
||||||
String[] argv = new String[7];
|
|
||||||
argv[0] = "-numThreads";
|
|
||||||
argv[1] = Integer.toString(100);
|
|
||||||
argv[2] = "-numIterationsPerThread";
|
|
||||||
argv[3] = Integer.toString(1000);
|
|
||||||
argv[4] = "-path";
|
|
||||||
argv[5] = TEST_UTIL.getDataTestDir() + "/HlogPerformance";
|
|
||||||
argv[6] = "-nosync";
|
|
||||||
try {
|
|
||||||
res = ToolRunner.run(bench, argv);
|
|
||||||
} finally {
|
|
||||||
bench.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void main(String[] argv) throws Exception {
|
|
||||||
TestHLogBench bench = new TestHLogBench();
|
|
||||||
int res;
|
|
||||||
try {
|
|
||||||
res = ToolRunner.run(bench, argv);
|
|
||||||
} finally {
|
|
||||||
bench.close();
|
|
||||||
}
|
|
||||||
System.exit(res);
|
|
||||||
}
|
|
||||||
|
|
||||||
@org.junit.Rule
|
|
||||||
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
|
|
||||||
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
|
|
||||||
}
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue