Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11390 PerformanceEvaluation: add an option to use a single connection

This commit is contained in:
Nicolas Liochon 2014-06-24 11:34:45 +02:00
parent b2914be9d3
commit e991fd8055
1 changed files with 50 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
View File

@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
@ -503,6 +504,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = that.flushCommits; this.flushCommits = that.flushCommits;
this.writeToWAL = that.writeToWAL; this.writeToWAL = that.writeToWAL;
this.autoFlush = that.autoFlush; this.autoFlush = that.autoFlush;
this.oneCon = that.oneCon;
this.useTags = that.useTags; this.useTags = that.useTags;
this.noOfTags = that.noOfTags; this.noOfTags = that.noOfTags;
this.reportLatency = that.reportLatency; this.reportLatency = that.reportLatency;
@ -531,6 +533,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
public boolean flushCommits = true; public boolean flushCommits = true;
public boolean writeToWAL = true; public boolean writeToWAL = true;
public boolean autoFlush = false; public boolean autoFlush = false;
public boolean oneCon = false;
public boolean useTags = false; public boolean useTags = false;
public int noOfTags = 1; public int noOfTags = 1;
public boolean reportLatency = false; public boolean reportLatency = false;
@ -577,8 +580,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
* Note that all subclasses of this class must provide a public contructor * Note that all subclasses of this class must provide a public contructor
* that has the exact same list of arguments. * that has the exact same list of arguments.
*/ */
Test(final Configuration conf, final TestOptions options, final Status status) { Test(final HConnection con, final TestOptions options, final Status status) {
this.conf = conf; this.connection = con;
this.conf = con.getConfiguration();
this.opts = options; this.opts = options;
this.status = status; this.status = status;
this.testName = this.getClass().getSimpleName(); this.testName = this.getClass().getSimpleName();
@ -631,9 +635,11 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
void testSetup() throws IOException { void testSetup() throws IOException {
if (!opts.oneCon) {
this.connection = HConnectionManager.createConnection(conf); this.connection = HConnectionManager.createConnection(conf);
this.table = connection.getTable(opts.tableName); }
this.table.setAutoFlush(opts.autoFlush, true); this.table = new HTable(TableName.valueOf(opts.tableName), connection);
this.table.setAutoFlushTo(opts.autoFlush);
try { try {
Constructor<?> ctor = Constructor<?> ctor =
@ -654,7 +660,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.table.flushCommits(); this.table.flushCommits();
} }
table.close(); table.close();
if (!opts.oneCon) {
connection.close(); connection.close();
}
receiverHost.closeReceivers(); receiverHost.closeReceivers();
} }
@ -765,8 +773,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
@SuppressWarnings("unused") @SuppressWarnings("unused")
static class RandomSeekScanTest extends Test { static class RandomSeekScanTest extends Test {
RandomSeekScanTest(Configuration conf, TestOptions options, Status status) { RandomSeekScanTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -795,8 +803,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static abstract class RandomScanWithRangeTest extends Test { static abstract class RandomScanWithRangeTest extends Test {
RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) { RandomScanWithRangeTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -811,7 +819,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
int count = 0; int count = 0;
ResultScanner s = this.table.getScanner(scan); ResultScanner s = this.table.getScanner(scan);
for (; (r = s.next()) != null;) { for (; (r = s.next()) != null;) {
updateValueSize(r);; updateValueSize(r);
count++; count++;
} }
if (i % 100 == 0) { if (i % 100 == 0) {
@ -839,8 +847,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class RandomScanWithRange10Test extends RandomScanWithRangeTest { static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) { RandomScanWithRange10Test(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -850,8 +858,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class RandomScanWithRange100Test extends RandomScanWithRangeTest { static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) { RandomScanWithRange100Test(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -861,8 +869,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class RandomScanWithRange1000Test extends RandomScanWithRangeTest { static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) { RandomScanWithRange1000Test(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -872,8 +880,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class RandomScanWithRange10000Test extends RandomScanWithRangeTest { static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) { RandomScanWithRange10000Test(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -885,8 +893,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class RandomReadTest extends Test { static class RandomReadTest extends Test {
private ArrayList<Get> gets; private ArrayList<Get> gets;
RandomReadTest(Configuration conf, TestOptions options, Status status) { RandomReadTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
if (opts.multiGet > 0) { if (opts.multiGet > 0) {
LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + "."); LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
this.gets = new ArrayList<Get>(opts.multiGet); this.gets = new ArrayList<Get>(opts.multiGet);
@ -930,8 +938,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class RandomWriteTest extends Test { static class RandomWriteTest extends Test {
RandomWriteTest(Configuration conf, TestOptions options, Status status) { RandomWriteTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -962,8 +970,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class ScanTest extends Test { static class ScanTest extends Test {
private ResultScanner testScanner; private ResultScanner testScanner;
ScanTest(Configuration conf, TestOptions options, Status status) { ScanTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -993,8 +1001,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class SequentialReadTest extends Test { static class SequentialReadTest extends Test {
SequentialReadTest(Configuration conf, TestOptions options, Status status) { SequentialReadTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -1009,8 +1017,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
} }
static class SequentialWriteTest extends Test { static class SequentialWriteTest extends Test {
SequentialWriteTest(Configuration conf, TestOptions options, Status status) { SequentialWriteTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -1041,8 +1049,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class FilteredScanTest extends Test { static class FilteredScanTest extends Test {
protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName()); protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
FilteredScanTest(Configuration conf, TestOptions options, Status status) { FilteredScanTest(HConnection con, TestOptions options, Status status) {
super(conf, options, status); super(con, options, status);
} }
@Override @Override
@ -1159,10 +1167,11 @@ public class PerformanceEvaluation extends Configured implements Tool {
long totalElapsedTime; long totalElapsedTime;
final Test t; final Test t;
HConnection con = HConnectionManager.createConnection(conf);
try { try {
Constructor<? extends Test> constructor = Constructor<? extends Test> constructor =
cmd.getDeclaredConstructor(Configuration.class, TestOptions.class, Status.class); cmd.getDeclaredConstructor(HConnection.class, TestOptions.class, Status.class);
t = constructor.newInstance(conf, opts, status); t = constructor.newInstance(con, opts, status);
} catch (NoSuchMethodException e) { } catch (NoSuchMethodException e) {
throw new IllegalArgumentException("Invalid command class: " + throw new IllegalArgumentException("Invalid command class: " +
cmd.getName() + ". It does not provide a constructor as described by " + cmd.getName() + ". It does not provide a constructor as described by " +
@ -1177,6 +1186,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
" (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
getAverageValueLength(opts)) + ")"); getAverageValueLength(opts)) + ")");
con.close();
return totalElapsedTime; return totalElapsedTime;
} }
@ -1229,6 +1239,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"Default: false"); "Default: false");
System.err.println(" writeToWAL Set writeToWAL on puts. Default: True"); System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
System.err.println(" autoFlush Set autoFlush on htable. Default: False"); System.err.println(" autoFlush Set autoFlush on htable. Default: False");
System.err.println(" oneCon all the threads share the same connection. Default: False");
System.err.println(" presplit Create presplit table. Recommended for accurate perf " + System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
"analysis (see guide). Default: disabled"); "analysis (see guide). Default: disabled");
System.err.println(" inmemory Tries to keep the HFiles of the CF " + System.err.println(" inmemory Tries to keep the HFiles of the CF " +
@ -1366,6 +1377,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue; continue;
} }
final String onceCon = "--oneCon=";
if (cmd.startsWith(onceCon)) {
opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length()));
continue;
}
final String presplit = "--presplit="; final String presplit = "--presplit=";
if (cmd.startsWith(presplit)) { if (cmd.startsWith(presplit)) {
opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length())); opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));