HBASE-13933 DBE's seekBefore with tags corrupts the tag's offset information thus leading to incorrect results (ramkrishna.s.vasudevan)
This commit is contained in:
Nick Dimiduk
parent
6b37f26280
commit
db08013ebe
|
|
@ -194,6 +194,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
|
|||
nextKvOffset = nextState.nextKvOffset;
|
||||
memstoreTS = nextState.memstoreTS;
|
||||
currentBuffer = nextState.currentBuffer;
|
||||
tagsOffset = nextState.tagsOffset;
|
||||
tagsLength = nextState.tagsLength;
|
||||
if (nextState.tagCompressionContext != null) {
|
||||
tagCompressionContext = nextState.tagCompressionContext;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,30 +19,71 @@
|
|||
package org.apache.hadoop.hbase.io.hfile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.Tag;
|
||||
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
||||
import org.apache.hadoop.hbase.testclassification.IOTests;
|
||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
/**
|
||||
* Test {@link HFileScanner#seekTo(byte[])} and its variants.
|
||||
*/
|
||||
@Category({IOTests.class, SmallTests.class})
|
||||
public class TestSeekTo extends HBaseTestCase {
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestSeekTo {
|
||||
|
||||
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private final DataBlockEncoding encoding;
|
||||
@Parameters
|
||||
public static Collection<Object[]> parameters() {
|
||||
List<Object[]> paramList = new ArrayList<Object[]>();
|
||||
for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
|
||||
// Remove after HBASE-13939
|
||||
if (encoding != DataBlockEncoding.PREFIX_TREE) {
|
||||
paramList.add(new Object[] { encoding });
|
||||
}
|
||||
}
|
||||
return paramList;
|
||||
}
|
||||
static boolean switchKVs = false;
|
||||
|
||||
public TestSeekTo(DataBlockEncoding encoding) {
|
||||
this.encoding = encoding;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
//reset
|
||||
switchKVs = false;
|
||||
}
|
||||
|
||||
static KeyValue toKV(String row, TagUsage tagUsage) {
|
||||
if (tagUsage == TagUsage.NO_TAG) {
|
||||
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
|
||||
|
|
@ -57,7 +98,7 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
if (!switchKVs) {
|
||||
switchKVs = true;
|
||||
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
|
||||
Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
|
||||
Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
|
||||
} else {
|
||||
switchKVs = false;
|
||||
Tag t = new Tag((byte) 1, "myTag1");
|
||||
|
|
@ -73,11 +114,13 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
}
|
||||
|
||||
Path makeNewFile(TagUsage tagUsage) throws IOException {
|
||||
Path ncTFile = new Path(testDir, "basic.hfile");
|
||||
FSDataOutputStream fout = this.fs.create(ncTFile);
|
||||
Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
|
||||
FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
|
||||
int blocksize = toKV("a", tagUsage).getLength() * 3;
|
||||
HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
|
||||
.withDataBlockEncoding(encoding)
|
||||
.withIncludesTags(true).build();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
|
||||
.withFileContext(context)
|
||||
.withComparator(CellComparator.COMPARATOR).create();
|
||||
|
|
@ -103,57 +146,77 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
|
||||
protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
|
||||
Path p = makeNewFile(tagUsage);
|
||||
FileSystem fs = TEST_UTIL.getTestFileSystem();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
|
||||
reader.loadFileInfo();
|
||||
HFileScanner scanner = reader.getScanner(false, true);
|
||||
assertEquals(false, scanner.seekBefore(toKV("a", tagUsage)));
|
||||
assertFalse(scanner.seekBefore(toKV("a", tagUsage)));
|
||||
|
||||
assertEquals(false, scanner.seekBefore(toKV("c", tagUsage)));
|
||||
assertFalse(scanner.seekBefore(toKV("c", tagUsage)));
|
||||
|
||||
assertEquals(true, scanner.seekBefore(toKV("d", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("d", tagUsage)));
|
||||
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
assertEquals(true, scanner.seekBefore(toKV("e", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("e", tagUsage)));
|
||||
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
assertEquals(true, scanner.seekBefore(toKV("f", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("f", tagUsage)));
|
||||
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
assertEquals(true, scanner.seekBefore(toKV("g", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("g", tagUsage)));
|
||||
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
assertEquals(true, scanner.seekBefore(toKV("h", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("h", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
assertEquals(true, scanner.seekBefore(toKV("i", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("i", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
assertEquals(true, scanner.seekBefore(toKV("j", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("j", tagUsage)));
|
||||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
assertEquals(true, scanner.seekBefore(toKV("k", tagUsage)));
|
||||
Cell cell = scanner.getKeyValue();
|
||||
if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
|
||||
Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
|
||||
cell.getTagsLength());
|
||||
while (tagsIterator.hasNext()) {
|
||||
Tag next = tagsIterator.next();
|
||||
assertEquals("myTag1", Bytes.toString(next.getValue()));
|
||||
}
|
||||
}
|
||||
assertTrue(scanner.seekBefore(toKV("k", tagUsage)));
|
||||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
assertEquals(true, scanner.seekBefore(toKV("l", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("l", tagUsage)));
|
||||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
reader.close();
|
||||
deleteTestDir(fs);
|
||||
}
|
||||
|
||||
protected void deleteTestDir(FileSystem fs) throws IOException {
|
||||
Path dataTestDir = TEST_UTIL.getDataTestDir();
|
||||
if(fs.exists(dataTestDir)) {
|
||||
fs.delete(dataTestDir, true);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSeekBeforeWithReSeekTo() throws Exception {
|
||||
testSeekBeforeInternals(TagUsage.NO_TAG);
|
||||
testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
|
||||
testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
|
||||
testSeekBeforeWithReSeekToInternals(TagUsage.PARTIAL_TAG);
|
||||
}
|
||||
|
||||
protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException {
|
||||
Path p = makeNewFile(tagUsage);
|
||||
FileSystem fs = TEST_UTIL.getTestFileSystem();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
|
||||
reader.loadFileInfo();
|
||||
HFileScanner scanner = reader.getScanner(false, true);
|
||||
assertEquals(false, scanner.seekBefore(toKV("a", tagUsage)));
|
||||
assertEquals(false, scanner.seekBefore(toKV("b", tagUsage)));
|
||||
assertEquals(false, scanner.seekBefore(toKV("c", tagUsage)));
|
||||
assertFalse(scanner.seekBefore(toKV("a", tagUsage)));
|
||||
assertFalse(scanner.seekBefore(toKV("b", tagUsage)));
|
||||
assertFalse(scanner.seekBefore(toKV("c", tagUsage)));
|
||||
|
||||
// seekBefore d, so the scanner points to c
|
||||
assertEquals(true, scanner.seekBefore(toKV("d", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("d", tagUsage)));
|
||||
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo e and g
|
||||
assertEquals(0, scanner.reseekTo(toKV("c", tagUsage)));
|
||||
|
|
@ -162,7 +225,7 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore e, so the scanner points to c
|
||||
assertEquals(true, scanner.seekBefore(toKV("e", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("e", tagUsage)));
|
||||
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo e and g
|
||||
assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
|
||||
|
|
@ -171,7 +234,7 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore f, so the scanner points to e
|
||||
assertEquals(true, scanner.seekBefore(toKV("f", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("f", tagUsage)));
|
||||
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo e and g
|
||||
assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
|
||||
|
|
@ -180,7 +243,7 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore g, so the scanner points to e
|
||||
assertEquals(true, scanner.seekBefore(toKV("g", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("g", tagUsage)));
|
||||
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo e and g again
|
||||
assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
|
||||
|
|
@ -189,28 +252,28 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore h, so the scanner points to g
|
||||
assertEquals(true, scanner.seekBefore(toKV("h", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("h", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo g
|
||||
assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore i, so the scanner points to g
|
||||
assertEquals(true, scanner.seekBefore(toKV("i", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("i", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo g
|
||||
assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
|
||||
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore j, so the scanner points to i
|
||||
assertEquals(true, scanner.seekBefore(toKV("j", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("j", tagUsage)));
|
||||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo i
|
||||
assertEquals(0, scanner.reseekTo(toKV("i", tagUsage)));
|
||||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore k, so the scanner points to i
|
||||
assertEquals(true, scanner.seekBefore(toKV("k", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("k", tagUsage)));
|
||||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo i and k
|
||||
assertEquals(0, scanner.reseekTo(toKV("i", tagUsage)));
|
||||
|
|
@ -219,22 +282,25 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
// seekBefore l, so the scanner points to k
|
||||
assertEquals(true, scanner.seekBefore(toKV("l", tagUsage)));
|
||||
assertTrue(scanner.seekBefore(toKV("l", tagUsage)));
|
||||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
// reseekTo k
|
||||
assertEquals(0, scanner.reseekTo(toKV("k", tagUsage)));
|
||||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
deleteTestDir(fs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSeekTo() throws Exception {
|
||||
testSeekBeforeInternals(TagUsage.NO_TAG);
|
||||
testSeekToInternals(TagUsage.NO_TAG);
|
||||
testSeekToInternals(TagUsage.ONLY_TAG);
|
||||
testSeekToInternals(TagUsage.PARTIAL_TAG);
|
||||
}
|
||||
|
||||
protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
|
||||
Path p = makeNewFile(tagUsage);
|
||||
FileSystem fs = TEST_UTIL.getTestFileSystem();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
|
||||
reader.loadFileInfo();
|
||||
assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
|
||||
|
|
@ -251,20 +317,28 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||
|
||||
assertEquals(1, scanner.seekTo(toKV("l", tagUsage)));
|
||||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
if (encoding == DataBlockEncoding.PREFIX_TREE) {
|
||||
// TODO : Fix this
|
||||
assertEquals(null, scanner.getKeyValue());
|
||||
} else {
|
||||
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||
}
|
||||
|
||||
reader.close();
|
||||
deleteTestDir(fs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlockContainingKey() throws Exception {
|
||||
testSeekBeforeInternals(TagUsage.NO_TAG);
|
||||
testBlockContainingKeyInternals(TagUsage.NO_TAG);
|
||||
testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
|
||||
testBlockContainingKeyInternals(TagUsage.PARTIAL_TAG);
|
||||
}
|
||||
|
||||
protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
|
||||
Path p = makeNewFile(tagUsage);
|
||||
FileSystem fs = TEST_UTIL.getTestFileSystem();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
|
||||
reader.loadFileInfo();
|
||||
HFileBlockIndex.BlockIndexReader blockIndexReader =
|
||||
|
|
@ -291,5 +365,6 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
assertEquals(1, blockIndexReader.rootBlockContainingKey(
|
||||
toKV("l", tagUsage)));
|
||||
reader.close();
|
||||
deleteTestDir(fs);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue