Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12817 Data missing while scanning using PREFIX_TREE data block encoding (Duo Zhang)

This commit is contained in:
tedyu 2015-01-07 18:45:09 -08:00
parent 7dce0d5c45
commit 645fbd7d87
4 changed files with 136 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
View File

@ -130,14 +130,9 @@ public class PrefixTreeSeeker implements EncodedSeeker {
@Override @Override
public boolean next() { public boolean next() {
boolean advance = ptSearcher.advance(); return ptSearcher.advance();
if (ptSearcher.hasMovedToPreviousAsPartOfSeek()) {
ptSearcher.setMovedToPreviousAsPartOfSeek(false);
}
return advance;
} }
// @Override
public boolean advance() { public boolean advance() {
return ptSearcher.advance(); return ptSearcher.advance();
} }
@ -222,7 +217,6 @@ public class PrefixTreeSeeker implements EncodedSeeker {
if (CellScannerPosition.AT == position) { if (CellScannerPosition.AT == position) {
if (seekBefore) { if (seekBefore) {
// We need not set movedToPrevious because the intention is to seekBefore
ptSearcher.previous(); ptSearcher.previous();
return 1; return 1;
} }
@ -233,7 +227,6 @@ public class PrefixTreeSeeker implements EncodedSeeker {
if (CellScannerPosition.AFTER == position) { if (CellScannerPosition.AFTER == position) {
if (!ptSearcher.isBeforeFirst()) { if (!ptSearcher.isBeforeFirst()) {
ptSearcher.previous(); ptSearcher.previous();
ptSearcher.setMovedToPreviousAsPartOfSeek(true);
} }
return 1; return 1;
} }

1
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
View File

@ -115,6 +115,7 @@ public class PrefixTreeArrayReversibleScanner extends PrefixTreeArrayScanner imp
} }
} }
if (currentRowNode.hasOccurrences()) {// escape clause if (currentRowNode.hasOccurrences()) {// escape clause
currentRowNode.resetFanIndex();
return true;// found some values return true;// found some values
} }
} }

19
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
View File

@ -60,7 +60,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
protected boolean nubCellsRemain; protected boolean nubCellsRemain;
protected int currentCellIndex; protected int currentCellIndex;
protected boolean movedToPrevious;
/*********************** construct ******************************/ /*********************** construct ******************************/
@ -260,15 +259,10 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
currentCellIndex = 0; currentCellIndex = 0;
return true; return true;
} }
} else {
if (movedToPrevious && currentRowNode.hasOccurrences()
&& currentRowNode.getFanIndex() == getNextFanIndex()) {
followFan(getNextFanIndex());
} else { } else {
discardCurrentRowNode(true); discardCurrentRowNode(true);
} }
} }
}
return false;// went past the end return false;// went past the end
} }
@ -303,10 +297,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
followFan(currentFanPosition); followFan(currentFanPosition);
} }
protected int getNextFanIndex() {
return rowNodes[rowNodeStackIndex + 1].getFanIndex();
}
protected void followNextFan() { protected void followNextFan() {
int nextFanPosition = currentRowNode.getFanIndex() + 1; int nextFanPosition = currentRowNode.getFanIndex() + 1;
followFan(nextFanPosition); followFan(nextFanPosition);
@ -537,13 +527,4 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
public int getTagBufferLength() { public int getTagBufferLength() {
return tagsBuffer.length; return tagsBuffer.length;
} }
public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) {
this.movedToPrevious = movedToPrevious;
}
public boolean hasMovedToPreviousAsPartOfSeek() {
return this.movedToPrevious;
}
} }

154
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
View File

@ -18,34 +18,38 @@
package org.apache.hadoop.hbase.io.encoding; package org.apache.hadoop.hbase.io.encoding;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import org.apache.hadoop.conf.Configuration; import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass; import org.junit.After;
import org.junit.BeforeClass; import org.junit.Before;
import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
@Category({IOTests.class, MediumTests.class}) @Category({ IOTests.class, SmallTests.class })
public class TestPrefixTree { public class TestPrefixTree {
private static final String row4 = "a-b-B-2-1402397300-1402416535"; private static final String row4 = "a-b-B-2-1402397300-1402416535";
private static final byte[] row4_bytes = Bytes.toBytes(row4); private static final byte[] row4_bytes = Bytes.toBytes(row4);
private static final String row3 = "a-b-A-1-1402397227-1402415999"; private static final String row3 = "a-b-A-1-1402397227-1402415999";
@ -54,59 +58,49 @@ public class TestPrefixTree {
private static final byte[] row2_bytes = Bytes.toBytes(row2); private static final byte[] row2_bytes = Bytes.toBytes(row2);
private static final String row1 = "a-b-A-1"; private static final String row1 = "a-b-A-1";
private static final byte[] row1_bytes = Bytes.toBytes(row1); private static final byte[] row1_bytes = Bytes.toBytes(row1);
public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static byte[] fam = Bytes.toBytes("cf_1"); private final static byte[] fam = Bytes.toBytes("cf_1");
private final static byte[] qual1 = Bytes.toBytes("qf_1"); private final static byte[] qual1 = Bytes.toBytes("qf_1");
private final static byte[] qual2 = Bytes.toBytes("qf_2"); private final static byte[] qual2 = Bytes.toBytes("qf_2");
public static Configuration conf;
@Rule private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
public final TestName TEST_NAME = new TestName();
@BeforeClass private HRegion region;
public static void setupBeforeClass() throws Exception {
// setup configuration @Before
conf = TEST_UTIL.getConfiguration(); public void setUp() throws Exception {
conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); TableName tableName = TableName.valueOf(getClass().getSimpleName());
conf.setBoolean("hbase.online.schema.update.enable", true); HTableDescriptor htd = new HTableDescriptor(tableName);
conf.setInt("hbase.client.scanner.timeout.period", 600000); htd.addFamily(new HColumnDescriptor(fam).setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE));
TEST_UTIL.startMiniCluster(2); HRegionInfo info = new HRegionInfo(tableName, null, null, false);
Path path = testUtil.getDataTestDir(getClass().getSimpleName());
region = HRegion.createHRegion(info, path, testUtil.getConfiguration(), htd);
} }
@AfterClass @After
public static void tearDownAfterClass() throws Exception { public void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster(); region.close(true);
testUtil.cleanupTestDir();
} }
@Test @Test
public void testHBASE11728() throws Exception { public void testHBASE11728() throws Exception {
TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Table table = null;
try {
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE);
colDesc.setTimeToLive(15552000);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
table = new HTable(conf, tableName);
Put put = new Put(Bytes.toBytes("a-b-0-0")); Put put = new Put(Bytes.toBytes("a-b-0-0"));
put.add(fam, qual1, Bytes.toBytes("c1-value")); put.add(fam, qual1, Bytes.toBytes("c1-value"));
table.put(put); region.put(put);
put = new Put(row1_bytes); put = new Put(row1_bytes);
put.add(fam, qual1, Bytes.toBytes("c1-value")); put.add(fam, qual1, Bytes.toBytes("c1-value"));
table.put(put); region.put(put);
put = new Put(row2_bytes); put = new Put(row2_bytes);
put.add(fam, qual2, Bytes.toBytes("c2-value")); put.add(fam, qual2, Bytes.toBytes("c2-value"));
table.put(put); region.put(put);
put = new Put(row3_bytes); put = new Put(row3_bytes);
put.add(fam, qual2, Bytes.toBytes("c2-value-2")); put.add(fam, qual2, Bytes.toBytes("c2-value-2"));
table.put(put); region.put(put);
put = new Put(row4_bytes); put = new Put(row4_bytes);
put.add(fam, qual2, Bytes.toBytes("c2-value-3")); put.add(fam, qual2, Bytes.toBytes("c2-value-3"));
table.put(put); region.put(put);
hBaseAdmin.flush(tableName); region.flushcache(true);
String[] rows = new String[3]; String[] rows = new String[3];
rows[0] = row1; rows[0] = row1;
rows[1] = row2; rows[1] = row2;
@ -118,67 +112,81 @@ public class TestPrefixTree {
Scan scan = new Scan(); Scan scan = new Scan();
scan.setStartRow(row1_bytes); scan.setStartRow(row1_bytes);
scan.setStopRow(Bytes.toBytes("a-b-A-1:")); scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
ResultScanner scanner = table.getScanner(scan);
Result[] next = scanner.next(10); RegionScanner scanner = region.getScanner(scan);
assertEquals(3, next.length); List<Cell> cells = new ArrayList<Cell>();
int i = 0; for (int i = 0; i < 3; i++) {
for (Result res : next) { assertEquals(i < 2, scanner.next(cells));
CellScanner cellScanner = res.cellScanner(); CellScanner cellScanner = Result.create(cells).cellScanner();
while (cellScanner.advance()) { while (cellScanner.advance()) {
assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
.current().getRowOffset(), cellScanner.current().getRowLength())); .current().getRowOffset(), cellScanner.current().getRowLength()));
assertEquals(Bytes.toString(val[i]), Bytes.toString( assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(),
cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(), cellScanner.current().getValueOffset(), cellScanner.current().getValueLength()));
cellScanner.current().getValueLength()));
} }
i++; cells.clear();
} }
scanner.close(); scanner.close();
// Add column // Add column
scan = new Scan(); scan = new Scan();
scan.addColumn(fam, qual2); scan.addColumn(fam, qual2);
scan.setStartRow(row1_bytes); scan.setStartRow(row1_bytes);
scan.setStopRow(Bytes.toBytes("a-b-A-1:")); scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
scanner = table.getScanner(scan); scanner = region.getScanner(scan);
next = scanner.next(10); for (int i = 1; i < 3; i++) {
assertEquals(2, next.length); // assertEquals(i < 2, scanner.next(cells));
i = 1; scanner.next(cells);
for (Result res : next) { System.out.println(Result.create(cells));
CellScanner cellScanner = res.cellScanner(); CellScanner cellScanner = Result.create(cells).cellScanner();
while (cellScanner.advance()) { while (cellScanner.advance()) {
assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
.current().getRowOffset(), cellScanner.current().getRowLength())); .current().getRowOffset(), cellScanner.current().getRowLength()));
} }
i++; cells.clear();
} }
scanner.close(); scanner.close();
i = 1;
scan = new Scan(); scan = new Scan();
scan.addColumn(fam, qual2); scan.addColumn(fam, qual2);
scan.setStartRow(Bytes.toBytes("a-b-A-1-")); scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
scan.setStopRow(Bytes.toBytes("a-b-A-1:")); scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
scanner = table.getScanner(scan); scanner = region.getScanner(scan);
next = scanner.next(10); for (int i = 1; i < 3; i++) {
assertEquals(2, next.length); assertEquals(i < 2, scanner.next(cells));
for (Result res : next) { CellScanner cellScanner = Result.create(cells).cellScanner();
CellScanner cellScanner = res.cellScanner();
while (cellScanner.advance()) { while (cellScanner.advance()) {
assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
.current().getRowOffset(), cellScanner.current().getRowLength())); .current().getRowOffset(), cellScanner.current().getRowLength()));
} }
i++; cells.clear();
} }
scanner.close(); scanner.close();
scan = new Scan(); scan = new Scan();
scan.addColumn(fam, qual2); scan.addColumn(fam, qual2);
scan.setStartRow(Bytes.toBytes("a-b-A-1-140239")); scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
scan.setStopRow(Bytes.toBytes("a-b-A-1:")); scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
scanner = table.getScanner(scan); scanner = region.getScanner(scan);
next = scanner.next(10); assertFalse(scanner.next(cells));
assertEquals(1, next.length); assertFalse(cells.isEmpty());
scanner.close(); scanner.close();
} finally {
table.close();
} }
@Test
public void testHBASE12817() throws IOException {
for (int i = 0; i < 100; i++) {
region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).add(fam, qual1, Bytes.toBytes(i)));
}
region.put(new Put(Bytes.toBytes("obj299")).add(fam, qual1, Bytes.toBytes("whatever")));
region.put(new Put(Bytes.toBytes("obj29")).add(fam, qual1, Bytes.toBytes("whatever")));
region.put(new Put(Bytes.toBytes("obj2")).add(fam, qual1, Bytes.toBytes("whatever")));
region.put(new Put(Bytes.toBytes("obj3")).add(fam, qual1, Bytes.toBytes("whatever")));
region.flushcache(true);
Scan scan = new Scan(Bytes.toBytes("obj29995"));
RegionScanner scanner = region.getScanner(scan);
List<Cell> cells = new ArrayList<Cell>();
assertFalse(scanner.next(cells));
assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow());
} }
} }