Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10440 Integration tests fail due to nonce collisions (Sergey Shelukhin) 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1562675 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Kyle Purtell 2014-01-30 03:10:06 +00:00
parent 1bc0a5b6c7
commit 3f9f4e00b6
3 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java
View File

@ -129,7 +129,7 @@ public class IntegrationTestIngest extends IntegrationTestBase {
Assert.fail(errorMsg);
}
ret = loadTool.run(getArgsForLoadTestTool("-update", String.format("60:%d", writeThreads),
ret = loadTool.run(getArgsForLoadTestTool("-update", String.format("60:%d:1", writeThreads),
startKey, numKeys));
if (0 != ret) {
String errorMsg = "Update failed with error code " + ret;

11
hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
View File

@ -92,7 +92,8 @@ public class LoadTestTool extends AbstractHBaseTool {
/** Usage string for the update option */
protected static final String OPT_USAGE_UPDATE =
"<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
"<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
+ ">][:<#whether to ignore nonce collisions=0>]";
protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
Arrays.toString(BloomType.values());
@ -168,6 +169,7 @@ public class LoadTestTool extends AbstractHBaseTool {
// Updater options
protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
protected int updatePercent;
protected boolean ignoreConflicts = false;
protected boolean isBatchUpdate;
// Reader options
@ -353,18 +355,22 @@ public class LoadTestTool extends AbstractHBaseTool {
}
if (isUpdate) {
String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 2);
String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
int colIndex = 0;
updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
if (colIndex < mutateOpts.length) {
numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
}
if (colIndex < mutateOpts.length) {
ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
}
isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
System.out.println("Batch updates: " + isBatchUpdate);
System.out.println("Percent of keys to update: " + updatePercent);
System.out.println("Updater threads: " + numUpdaterThreads);
System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
}
if (isRead) {
@ -499,6 +505,7 @@ public class LoadTestTool extends AbstractHBaseTool {
updaterThreads = new MultiThreadedUpdater(dataGen, conf, tableName, updatePercent);
}
updaterThreads.setBatchUpdate(isBatchUpdate);
updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
}
if (isRead) {

15
hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.exceptions.OperationConflictException;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
import org.apache.hadoop.util.StringUtils;
@ -60,6 +61,7 @@ public class MultiThreadedUpdater extends MultiThreadedWriterBase {
private MultiThreadedWriterBase writer = null;
private boolean isBatchUpdate = false;
private boolean ignoreNonceConflicts = false;
private final double updatePercent;
public MultiThreadedUpdater(LoadTestDataGenerator dataGen, Configuration conf,
@ -294,16 +296,17 @@ public class MultiThreadedUpdater extends MultiThreadedWriterBase {
}
totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
} catch (IOException e) {
if (ignoreNonceConflicts && (e instanceof OperationConflictException)) {
LOG.info("Detected nonce conflict, ignoring: " + e.getMessage());
totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
return;
}
failedKeySet.add(keyBase);
String exceptionInfo;
if (e instanceof RetriesExhaustedWithDetailsException) {
RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException) e;
exceptionInfo = aggEx.getExhaustiveDescription();
} else {
StringWriter stackWriter = new StringWriter();
PrintWriter pw = new PrintWriter(stackWriter);
e.printStackTrace(pw);
pw.flush();
exceptionInfo = StringUtils.stringifyException(e);
}
LOG.error("Failed to mutate: " + keyBase + " after " +
@ -363,4 +366,8 @@ public class MultiThreadedUpdater extends MultiThreadedWriterBase {
+ exceptionInfo);
}
}
public void setIgnoreNonceConflicts(boolean value) {
this.ignoreNonceConflicts = value;
}
}