Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11000 Add autoflush option to PerformanceEvaluation 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1588069 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
nkeywal 2014-04-16 20:51:05 +00:00
parent 16c6a00ac0
commit b4ad3d27d2
1 changed files with 29 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
View File

@ -43,7 +43,6 @@ import com.yammer.metrics.core.MetricsRegistry;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -117,7 +116,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
private static final MathContext CXT = MathContext.DECIMAL64; private static final MathContext CXT = MathContext.DECIMAL64;
private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000); private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024); private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
private static final TestOptions DEFAULT_OPTS = new TestOptions();
protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>(); protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
@ -237,7 +235,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Configuration conf = HBaseConfiguration.create(context.getConfiguration()); Configuration conf = HBaseConfiguration.create(context.getConfiguration());
// Evaluation task // Evaluation task
long elapsedTime = this.pe.runOneClient(this.cmd, conf, opts, status); long elapsedTime = runOneClient(this.cmd, conf, opts, status);
// Collect how much time the thing took. Report as map output and // Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter. // to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@ -353,8 +351,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ Arrays.toString(timings)); + Arrays.toString(timings));
Arrays.sort(timings); Arrays.sort(timings);
long total = 0; long total = 0;
for (int i = 0; i < timings.length; i++) { for (long timing : timings) {
total += timings[i]; total += timing;
} }
LOG.info("[" + test + "]" LOG.info("[" + test + "]"
+ "\tMin: " + timings[0] + "ms" + "\tMin: " + timings[0] + "ms"
@ -493,6 +491,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.tableName = that.tableName; this.tableName = that.tableName;
this.flushCommits = that.flushCommits; this.flushCommits = that.flushCommits;
this.writeToWAL = that.writeToWAL; this.writeToWAL = that.writeToWAL;
this.autoFlush = that.autoFlush;
this.useTags = that.useTags; this.useTags = that.useTags;
this.noOfTags = that.noOfTags; this.noOfTags = that.noOfTags;
this.reportLatency = that.reportLatency; this.reportLatency = that.reportLatency;
@ -513,6 +512,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
public String tableName = TABLE_NAME; public String tableName = TABLE_NAME;
public boolean flushCommits = true; public boolean flushCommits = true;
public boolean writeToWAL = true; public boolean writeToWAL = true;
public boolean autoFlush = false;
public boolean useTags = false; public boolean useTags = false;
public int noOfTags = 1; public int noOfTags = 1;
public boolean reportLatency = false; public boolean reportLatency = false;
@ -574,7 +574,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testSetup() throws IOException { void testSetup() throws IOException {
this.connection = HConnectionManager.createConnection(conf); this.connection = HConnectionManager.createConnection(conf);
this.table = connection.getTable(opts.tableName); this.table = connection.getTable(opts.tableName);
this.table.setAutoFlush(false, true); this.table.setAutoFlush(opts.autoFlush, true);
String metricName = String metricName =
testName + "-Client-" + Thread.currentThread().getName() + "-testRowTime"; testName + "-Client-" + Thread.currentThread().getName() + "-testRowTime";
latency = latency =
@ -615,9 +615,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
// Report on completion of 1/10th of total. // Report on completion of 1/10th of total.
for (int i = opts.startRow; i < lastRow; i++) { for (int i = opts.startRow; i < lastRow; i++) {
if (i % everyN != 0) continue; if (i % everyN != 0) continue;
long startTime = System.currentTimeMillis(); long startTime = System.nanoTime();
testRow(i); testRow(i);
latency.update(System.currentTimeMillis() - startTime); latency.update((System.nanoTime() - startTime) / 100000);
if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
status.setStatus(generateStatus(opts.startRow, i, lastRow)); status.setStatus(generateStatus(opts.startRow, i, lastRow));
} }
@ -629,14 +629,16 @@ public class PerformanceEvaluation extends Configured implements Tool {
*/ */
private void reportLatency() throws IOException { private void reportLatency() throws IOException {
status.setStatus(testName + " latency log (ms), on " + latency.count() + " measures"); status.setStatus(testName + " latency log (ms), on " + latency.count() + " measures");
status.setStatus(testName + " Min = " + latency.min()); status.setStatus(testName + " Min = " + latency.min());
status.setStatus(testName + " Avg = " + latency.mean()); status.setStatus(testName + " Avg = " + latency.mean());
status.setStatus(testName + " StdDev = " + latency.stdDev()); status.setStatus(testName + " StdDev = " + latency.stdDev());
status.setStatus(testName + " 50th = " + latency.getSnapshot().getMedian()); status.setStatus(testName + " 50th = " + latency.getSnapshot().getMedian());
status.setStatus(testName + " 95th = " + latency.getSnapshot().get95thPercentile()); status.setStatus(testName + " 95th = " + latency.getSnapshot().get95thPercentile());
status.setStatus(testName + " 99th = " + latency.getSnapshot().get99thPercentile()); status.setStatus(testName + " 99th = " + latency.getSnapshot().get99thPercentile());
status.setStatus(testName + " 99.9th = " + latency.getSnapshot().get999thPercentile()); status.setStatus(testName + " 99.9th = " + latency.getSnapshot().get999thPercentile());
status.setStatus(testName + " Max = " + latency.max()); status.setStatus(testName + " 99.99th = " + latency.getSnapshot().getValue(0.9999));
status.setStatus(testName + " 99.9999th= " + latency.getSnapshot().getValue(0.99999));
status.setStatus(testName + " Max = " + latency.max());
} }
/* /*
@ -684,7 +686,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
ResultScanner s = this.table.getScanner(scan); ResultScanner s = this.table.getScanner(scan);
int count = 0; int count = 0;
for (Result rr; (rr = s.next()) != null;) { while (s.next() != null) {
count++; count++;
} }
@ -758,7 +760,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class RandomReadTest extends Test { static class RandomReadTest extends Test {
private ArrayList<Get> gets; private ArrayList<Get> gets;
int idx = 0;
RandomReadTest(Configuration conf, TestOptions options, Status status) { RandomReadTest(Configuration conf, TestOptions options, Status status) {
super(conf, options, status); super(conf, options, status);
@ -973,7 +974,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
*/ */
public static byte[] generateData(final Random r, int length) { public static byte[] generateData(final Random r, int length) {
byte [] b = new byte [length]; byte [] b = new byte [length];
int i = 0; int i;
for(i = 0; i < (length-8); i += 8) { for(i = 0; i < (length-8); i += 8) {
b[i] = (byte) (65 + r.nextInt(26)); b[i] = (byte) (65 + r.nextInt(26));
@ -1002,7 +1003,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
throws IOException { throws IOException {
status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " + status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
opts.perClientRunRows + " rows"); opts.perClientRunRows + " rows");
long totalElapsedTime = 0; long totalElapsedTime;
final Test t; final Test t;
try { try {
@ -1066,6 +1067,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
System.err.println(" flushCommits Used to determine if the test should flush the table. " + System.err.println(" flushCommits Used to determine if the test should flush the table. " +
"Default: false"); "Default: false");
System.err.println(" writeToWAL Set writeToWAL on puts. Default: True"); System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
System.err.println(" autoFlush Set autoFlush on htable. Default: False");
System.err.println(" presplit Create presplit table. Recommended for accurate perf " + System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
"analysis (see guide). Default: disabled"); "analysis (see guide). Default: disabled");
System.err.println(" inmemory Tries to keep the HFiles of the CF " + System.err.println(" inmemory Tries to keep the HFiles of the CF " +
@ -1075,8 +1077,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"Default: false"); "Default: false");
System.err.println(" numoftags Specify the no of tags that would be needed. " + System.err.println(" numoftags Specify the no of tags that would be needed. " +
"This works only if usetags is true."); "This works only if usetags is true.");
System.err.println(" latency Set to report operation latencies. " + System.err.println(" latency Set to report operation latencies. Default: False");
"Currently only supported by randomRead test. Default: False");
System.err.println(); System.err.println();
System.err.println(" Note: -D properties will be applied to the conf used. "); System.err.println(" Note: -D properties will be applied to the conf used. ");
System.err.println(" For example: "); System.err.println(" For example: ");
@ -1183,6 +1184,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue; continue;
} }
final String autoFlush = "--autoFlush=";
if (cmd.startsWith(autoFlush)) {
opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length()));
continue;
}
final String presplit = "--presplit="; final String presplit = "--presplit=";
if (cmd.startsWith(presplit)) { if (cmd.startsWith(presplit)) {
opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length())); opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));