From 29ec882a5e0269ce453cf806727551a4f28020a5 Mon Sep 17 00:00:00 2001 From: tedyu Date: Wed, 7 Jan 2015 19:32:17 -0800 Subject: [PATCH] HBASE-12817 Data missing while scanning using PREFIX_TREE data block encoding (Duo Zhang) --- .../codec/prefixtree/PrefixTreeSeeker.java | 9 +- .../PrefixTreeArrayReversibleScanner.java | 1 + .../decode/PrefixTreeArrayScanner.java | 21 +- .../hbase/io/encoding/TestPrefixTree.java | 256 +++++++++--------- 4 files changed, 134 insertions(+), 153 deletions(-) diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index 2db0f29aa18..b95055ccb8f 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -130,14 +130,9 @@ public class PrefixTreeSeeker implements EncodedSeeker { @Override public boolean next() { - boolean advance = ptSearcher.advance(); - if (ptSearcher.hasMovedToPreviousAsPartOfSeek()) { - ptSearcher.setMovedToPreviousAsPartOfSeek(false); - } - return advance; + return ptSearcher.advance(); } -// @Override public boolean advance() { return ptSearcher.advance(); } @@ -222,7 +217,6 @@ public class PrefixTreeSeeker implements EncodedSeeker { if (CellScannerPosition.AT == position) { if (seekBefore) { - // We need not set movedToPrevious because the intention is to seekBefore ptSearcher.previous(); return 1; } @@ -233,7 +227,6 @@ public class PrefixTreeSeeker implements EncodedSeeker { if (CellScannerPosition.AFTER == position) { if (!ptSearcher.isBeforeFirst()) { ptSearcher.previous(); - ptSearcher.setMovedToPreviousAsPartOfSeek(true); } return 1; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java index 584a8fb3151..5a817cb1e1d 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java @@ -115,6 +115,7 @@ public class PrefixTreeArrayReversibleScanner extends PrefixTreeArrayScanner imp } } if (currentRowNode.hasOccurrences()) {// escape clause + currentRowNode.resetFanIndex(); return true;// found some values } } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java index a68f8f8d6fc..cb7eeea7e7d 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java @@ -60,7 +60,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne protected boolean nubCellsRemain; protected int currentCellIndex; - protected boolean movedToPrevious; /*********************** construct ******************************/ @@ -261,12 +260,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne return true; } } else { - if (movedToPrevious && currentRowNode.hasOccurrences() - && currentRowNode.getFanIndex() == getNextFanIndex()) { - followFan(getNextFanIndex()); - } else { - discardCurrentRowNode(true); - } + discardCurrentRowNode(true); } } return false;// went past the end @@ -303,10 +297,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne followFan(currentFanPosition); } - protected int getNextFanIndex() { - return rowNodes[rowNodeStackIndex + 1].getFanIndex(); - } - protected void followNextFan() { int nextFanPosition = currentRowNode.getFanIndex() + 1; followFan(nextFanPosition); @@ -537,13 +527,4 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne public int getTagBufferLength() { return tagsBuffer.length; } - - public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) { - this.movedToPrevious = movedToPrevious; - } - - public boolean hasMovedToPreviousAsPartOfSeek() { - return this.movedToPrevious; - } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java index 5245cc153b0..2aab66270b7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java @@ -18,33 +18,37 @@ package org.apache.hadoop.hbase.io.encoding; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -import org.apache.hadoop.conf.Configuration; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -@Category(MediumTests.class) +@Category(SmallTests.class) public class TestPrefixTree { + private static final String row4 = "a-b-B-2-1402397300-1402416535"; private static final byte[] row4_bytes = Bytes.toBytes(row4); private static final String row3 = "a-b-A-1-1402397227-1402415999"; @@ -53,131 +57,133 @@ public class TestPrefixTree { private static final byte[] row2_bytes = Bytes.toBytes(row2); private static final String row1 = "a-b-A-1"; private static final byte[] row1_bytes = Bytes.toBytes(row1); - public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static byte[] fam = Bytes.toBytes("cf_1"); private final static byte[] qual1 = Bytes.toBytes("qf_1"); private final static byte[] qual2 = Bytes.toBytes("qf_2"); - public static Configuration conf; - @Rule - public final TestName TEST_NAME = new TestName(); + private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); - @BeforeClass - public static void setupBeforeClass() throws Exception { - // setup configuration - conf = TEST_UTIL.getConfiguration(); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); - conf.setBoolean("hbase.online.schema.update.enable", true); - conf.setInt("hbase.client.scanner.timeout.period", 600000); - TEST_UTIL.startMiniCluster(2); + private HRegion region; + + @Before + public void setUp() throws Exception { + TableName tableName = TableName.valueOf(getClass().getSimpleName()); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(fam).setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE)); + HRegionInfo info = new HRegionInfo(tableName, null, null, false); + Path path = testUtil.getDataTestDir(getClass().getSimpleName()); + region = HRegion.createHRegion(info, path, testUtil.getConfiguration(), htd); } - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); + @After + public void tearDown() throws Exception { + region.close(true); + testUtil.cleanupTestDir(); } @Test public void testHBASE11728() throws Exception { - TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); - Table table = null; - try { - Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); - HColumnDescriptor colDesc = new HColumnDescriptor(fam); - HTableDescriptor desc = new HTableDescriptor(tableName); - colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE); - colDesc.setTimeToLive(15552000); - desc.addFamily(colDesc); - hBaseAdmin.createTable(desc); - table = new HTable(conf, tableName); - Put put = new Put(Bytes.toBytes("a-b-0-0")); - put.add(fam, qual1, Bytes.toBytes("c1-value")); - table.put(put); - put = new Put(row1_bytes); - put.add(fam, qual1, Bytes.toBytes("c1-value")); - table.put(put); - put = new Put(row2_bytes); - put.add(fam, qual2, Bytes.toBytes("c2-value")); - table.put(put); - put = new Put(row3_bytes); - put.add(fam, qual2, Bytes.toBytes("c2-value-2")); - table.put(put); - put = new Put(row4_bytes); - put.add(fam, qual2, Bytes.toBytes("c2-value-3")); - table.put(put); - hBaseAdmin.flush(tableName); - String[] rows = new String[3]; - rows[0] = row1; - rows[1] = row2; - rows[2] = row3; - byte[][] val = new byte[3][]; - val[0] = Bytes.toBytes("c1-value"); - val[1] = Bytes.toBytes("c2-value"); - val[2] = Bytes.toBytes("c2-value-2"); - Scan scan = new Scan(); - scan.setStartRow(row1_bytes); - scan.setStopRow(Bytes.toBytes("a-b-A-1:")); - ResultScanner scanner = table.getScanner(scan); - Result[] next = scanner.next(10); - assertEquals(3, next.length); - int i = 0; - for (Result res : next) { - CellScanner cellScanner = res.cellScanner(); - while (cellScanner.advance()) { - assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner - .current().getRowOffset(), cellScanner.current().getRowLength())); - assertEquals(Bytes.toString(val[i]), Bytes.toString( - cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(), - cellScanner.current().getValueLength())); - } - i++; + Put put = new Put(Bytes.toBytes("a-b-0-0")); + put.add(fam, qual1, Bytes.toBytes("c1-value")); + region.put(put); + put = new Put(row1_bytes); + put.add(fam, qual1, Bytes.toBytes("c1-value")); + region.put(put); + put = new Put(row2_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value")); + region.put(put); + put = new Put(row3_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value-2")); + region.put(put); + put = new Put(row4_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value-3")); + region.put(put); + region.flushcache(); + String[] rows = new String[3]; + rows[0] = row1; + rows[1] = row2; + rows[2] = row3; + byte[][] val = new byte[3][]; + val[0] = Bytes.toBytes("c1-value"); + val[1] = Bytes.toBytes("c2-value"); + val[2] = Bytes.toBytes("c2-value-2"); + Scan scan = new Scan(); + scan.setStartRow(row1_bytes); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + + RegionScanner scanner = region.getScanner(scan); + List cells = new ArrayList(); + for (int i = 0; i < 3; i++) { + assertEquals(i < 2, scanner.next(cells)); + CellScanner cellScanner = Result.create(cells).cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(), + cellScanner.current().getValueOffset(), cellScanner.current().getValueLength())); } - scanner.close(); - // Add column - scan = new Scan(); - scan.addColumn(fam, qual2); - scan.setStartRow(row1_bytes); - scan.setStopRow(Bytes.toBytes("a-b-A-1:")); - scanner = table.getScanner(scan); - next = scanner.next(10); - assertEquals(2, next.length); - i = 1; - for (Result res : next) { - CellScanner cellScanner = res.cellScanner(); - while (cellScanner.advance()) { - assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner - .current().getRowOffset(), cellScanner.current().getRowLength())); - } - i++; - } - scanner.close(); - i = 1; - scan = new Scan(); - scan.addColumn(fam, qual2); - scan.setStartRow(Bytes.toBytes("a-b-A-1-")); - scan.setStopRow(Bytes.toBytes("a-b-A-1:")); - scanner = table.getScanner(scan); - next = scanner.next(10); - assertEquals(2, next.length); - for (Result res : next) { - CellScanner cellScanner = res.cellScanner(); - while (cellScanner.advance()) { - assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner - .current().getRowOffset(), cellScanner.current().getRowLength())); - } - i++; - } - scanner.close(); - scan = new Scan(); - scan.addColumn(fam, qual2); - scan.setStartRow(Bytes.toBytes("a-b-A-1-140239")); - scan.setStopRow(Bytes.toBytes("a-b-A-1:")); - scanner = table.getScanner(scan); - next = scanner.next(10); - assertEquals(1, next.length); - scanner.close(); - } finally { - table.close(); + cells.clear(); } + scanner.close(); + + // Add column + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(row1_bytes); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = region.getScanner(scan); + for (int i = 1; i < 3; i++) { + assertEquals(i < 2, scanner.next(cells)); + CellScanner cellScanner = Result.create(cells).cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + } + cells.clear(); + } + scanner.close(); + + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(Bytes.toBytes("a-b-A-1-")); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = region.getScanner(scan); + for (int i = 1; i < 3; i++) { + assertEquals(i < 2, scanner.next(cells)); + CellScanner cellScanner = Result.create(cells).cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + } + cells.clear(); + } + scanner.close(); + + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(Bytes.toBytes("a-b-A-1-140239")); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = region.getScanner(scan); + assertFalse(scanner.next(cells)); + assertFalse(cells.isEmpty()); + scanner.close(); + } + + @Test + public void testHBASE12817() throws IOException { + for (int i = 0; i < 100; i++) { + region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).add(fam, qual1, Bytes.toBytes(i))); + } + region.put(new Put(Bytes.toBytes("obj299")).add(fam, qual1, Bytes.toBytes("whatever"))); + region.put(new Put(Bytes.toBytes("obj29")).add(fam, qual1, Bytes.toBytes("whatever"))); + region.put(new Put(Bytes.toBytes("obj2")).add(fam, qual1, Bytes.toBytes("whatever"))); + region.put(new Put(Bytes.toBytes("obj3")).add(fam, qual1, Bytes.toBytes("whatever"))); + region.flushcache(); + Scan scan = new Scan(Bytes.toBytes("obj29995")); + RegionScanner scanner = region.getScanner(scan); + List cells = new ArrayList(); + assertFalse(scanner.next(cells)); + assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow()); } }