Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14541 TestHFileOutputFormat.testMRIncrementalLoadWithSplit failed due to too many splits and few retries

Browse Source
This commit is contained in:
Matteo Bertozzi 2015-10-19 09:22:59 -07:00
parent 61b11e0765
commit d01063c9a3
2 changed files with 28 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
View File

@ -428,6 +428,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
} }
int maxRetries = getConf().getInt("hbase.bulkload.retries.number", 10); int maxRetries = getConf().getInt("hbase.bulkload.retries.number", 10);
maxRetries = Math.max(maxRetries, startEndKeys.getFirst().length + 1);
if (maxRetries != 0 && count >= maxRetries) { if (maxRetries != 0 && count >= maxRetries) {
throw new IOException("Retry attempted " + count + throw new IOException("Retry attempted " + count +
" times without completing, bailing out"); " times without completing, bailing out");
@ -611,7 +612,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
// We use a '_' prefix which is ignored when walking directory trees // We use a '_' prefix which is ignored when walking directory trees
// above. // above.
final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); final String TMP_DIR = "_tmp";
Path tmpDir = item.hfilePath.getParent();
if (!tmpDir.getName().equals(TMP_DIR)) {
tmpDir = new Path(tmpDir, TMP_DIR);
}
LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + LOG.info("HFile at " + hfilePath + " no longer fits inside a single " +
"region. Splitting..."); "region. Splitting...");

22
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
View File

@ -191,6 +191,28 @@ public class TestLoadIncrementalHFiles {
testRegionCrossingHFileSplit(BloomType.ROWCOL); testRegionCrossingHFileSplit(BloomType.ROWCOL);
} }
@Test
public void testSplitALot() throws Exception {
runTest("testSplitALot", BloomType.NONE,
new byte[][] {
Bytes.toBytes("aaaa"), Bytes.toBytes("bbb"),
Bytes.toBytes("ccc"), Bytes.toBytes("ddd"),
Bytes.toBytes("eee"), Bytes.toBytes("fff"),
Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
Bytes.toBytes("iii"), Bytes.toBytes("lll"),
Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
Bytes.toBytes("ooo"), Bytes.toBytes("ppp"),
Bytes.toBytes("qqq"), Bytes.toBytes("rrr"),
Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
Bytes.toBytes("uuu"), Bytes.toBytes("vvv"),
Bytes.toBytes("zzz"),
},
new byte[][][] {
new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("zzz") },
}
);
}
private void testRegionCrossingHFileSplit(BloomType bloomType) throws Exception { private void testRegionCrossingHFileSplit(BloomType bloomType) throws Exception {
runTest("testHFileSplit" + bloomType + "Bloom", bloomType, runTest("testHFileSplit" + bloomType + "Bloom", bloomType,
new byte[][] { new byte[][] {