diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index 8406a1cd4d6..89f388cec4d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -194,6 +194,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { nextKvOffset = nextState.nextKvOffset; memstoreTS = nextState.memstoreTS; currentBuffer = nextState.currentBuffer; + tagsOffset = nextState.tagsOffset; + tagsLength = nextState.tagsLength; if (nextState.tagCompressionContext != null) { tagCompressionContext = nextState.tagCompressionContext; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index ca3afafd047..e9bf33eecce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -19,29 +19,70 @@ package org.apache.hadoop.hbase.io.hfile; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * Test {@link HFileScanner#seekTo(byte[])} and its variants. */ @Category({IOTests.class, SmallTests.class}) -public class TestSeekTo extends HBaseTestCase { +@RunWith(Parameterized.class) +public class TestSeekTo { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final DataBlockEncoding encoding; + @Parameters + public static Collection parameters() { + List paramList = new ArrayList(); + for (DataBlockEncoding encoding : DataBlockEncoding.values()) { + // Remove after HBASE-13939 + if (encoding != DataBlockEncoding.PREFIX_TREE) { + paramList.add(new Object[] { encoding }); + } + } + return paramList; + } static boolean switchKVs = false; + + public TestSeekTo(DataBlockEncoding encoding) { + this.encoding = encoding; + } + + @Before + public void setUp() { + //reset + switchKVs = false; + } static KeyValue toKV(String row, TagUsage tagUsage) { if (tagUsage == TagUsage.NO_TAG) { @@ -57,7 +98,7 @@ public class TestSeekTo extends HBaseTestCase { if (!switchKVs) { switchKVs = true; return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), - Bytes.toBytes("qualifier"), Bytes.toBytes("value")); + Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); } else { switchKVs = false; Tag t = new Tag((byte) 1, "myTag1"); @@ -73,11 +114,13 @@ public class TestSeekTo extends HBaseTestCase { } Path makeNewFile(TagUsage tagUsage) throws IOException { - Path ncTFile = new Path(testDir, "basic.hfile"); - FSDataOutputStream fout = this.fs.create(ncTFile); + Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); + FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) + .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparator.COMPARATOR).create(); @@ -103,57 +146,77 @@ public class TestSeekTo extends HBaseTestCase { protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); - assertEquals(false, scanner.seekBefore(toKV("a", tagUsage))); + assertFalse(scanner.seekBefore(toKV("a", tagUsage))); - assertEquals(false, scanner.seekBefore(toKV("c", tagUsage))); + assertFalse(scanner.seekBefore(toKV("c", tagUsage))); - assertEquals(true, scanner.seekBefore(toKV("d", tagUsage))); + assertTrue(scanner.seekBefore(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("e", tagUsage))); + assertTrue(scanner.seekBefore(toKV("e", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("f", tagUsage))); + assertTrue(scanner.seekBefore(toKV("f", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("g", tagUsage))); + assertTrue(scanner.seekBefore(toKV("g", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - - assertEquals(true, scanner.seekBefore(toKV("h", tagUsage))); + assertTrue(scanner.seekBefore(toKV("h", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("i", tagUsage))); + assertTrue(scanner.seekBefore(toKV("i", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("j", tagUsage))); + assertTrue(scanner.seekBefore(toKV("j", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("k", tagUsage))); + Cell cell = scanner.getKeyValue(); + if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) { + Iterator tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + while (tagsIterator.hasNext()) { + Tag next = tagsIterator.next(); + assertEquals("myTag1", Bytes.toString(next.getValue())); + } + } + assertTrue(scanner.seekBefore(toKV("k", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("l", tagUsage))); + assertTrue(scanner.seekBefore(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); reader.close(); + deleteTestDir(fs); + } + + protected void deleteTestDir(FileSystem fs) throws IOException { + Path dataTestDir = TEST_UTIL.getDataTestDir(); + if(fs.exists(dataTestDir)) { + fs.delete(dataTestDir, true); + } } @Test public void testSeekBeforeWithReSeekTo() throws Exception { - testSeekBeforeInternals(TagUsage.NO_TAG); + testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG); testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG); testSeekBeforeWithReSeekToInternals(TagUsage.PARTIAL_TAG); } protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); - assertEquals(false, scanner.seekBefore(toKV("a", tagUsage))); - assertEquals(false, scanner.seekBefore(toKV("b", tagUsage))); - assertEquals(false, scanner.seekBefore(toKV("c", tagUsage))); + assertFalse(scanner.seekBefore(toKV("a", tagUsage))); + assertFalse(scanner.seekBefore(toKV("b", tagUsage))); + assertFalse(scanner.seekBefore(toKV("c", tagUsage))); // seekBefore d, so the scanner points to c - assertEquals(true, scanner.seekBefore(toKV("d", tagUsage))); + assertTrue(scanner.seekBefore(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); // reseekTo e and g assertEquals(0, scanner.reseekTo(toKV("c", tagUsage))); @@ -162,7 +225,7 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore e, so the scanner points to c - assertEquals(true, scanner.seekBefore(toKV("e", tagUsage))); + assertTrue(scanner.seekBefore(toKV("e", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); // reseekTo e and g assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); @@ -171,7 +234,7 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore f, so the scanner points to e - assertEquals(true, scanner.seekBefore(toKV("f", tagUsage))); + assertTrue(scanner.seekBefore(toKV("f", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); // reseekTo e and g assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); @@ -180,7 +243,7 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore g, so the scanner points to e - assertEquals(true, scanner.seekBefore(toKV("g", tagUsage))); + assertTrue(scanner.seekBefore(toKV("g", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); // reseekTo e and g again assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); @@ -189,28 +252,28 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore h, so the scanner points to g - assertEquals(true, scanner.seekBefore(toKV("h", tagUsage))); + assertTrue(scanner.seekBefore(toKV("h", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // reseekTo g assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore i, so the scanner points to g - assertEquals(true, scanner.seekBefore(toKV("i", tagUsage))); + assertTrue(scanner.seekBefore(toKV("i", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // reseekTo g assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore j, so the scanner points to i - assertEquals(true, scanner.seekBefore(toKV("j", tagUsage))); + assertTrue(scanner.seekBefore(toKV("j", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // reseekTo i assertEquals(0, scanner.reseekTo(toKV("i", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // seekBefore k, so the scanner points to i - assertEquals(true, scanner.seekBefore(toKV("k", tagUsage))); + assertTrue(scanner.seekBefore(toKV("k", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // reseekTo i and k assertEquals(0, scanner.reseekTo(toKV("i", tagUsage))); @@ -219,22 +282,25 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("k", toRowStr(scanner.getKeyValue())); // seekBefore l, so the scanner points to k - assertEquals(true, scanner.seekBefore(toKV("l", tagUsage))); + assertTrue(scanner.seekBefore(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); // reseekTo k assertEquals(0, scanner.reseekTo(toKV("k", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); + deleteTestDir(fs); } @Test public void testSeekTo() throws Exception { - testSeekBeforeInternals(TagUsage.NO_TAG); + testSeekToInternals(TagUsage.NO_TAG); testSeekToInternals(TagUsage.ONLY_TAG); testSeekToInternals(TagUsage.PARTIAL_TAG); } protected void testSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); @@ -251,20 +317,28 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("i", toRowStr(scanner.getKeyValue())); assertEquals(1, scanner.seekTo(toKV("l", tagUsage))); - assertEquals("k", toRowStr(scanner.getKeyValue())); + if (encoding == DataBlockEncoding.PREFIX_TREE) { + // TODO : Fix this + assertEquals(null, scanner.getKeyValue()); + } else { + assertEquals("k", toRowStr(scanner.getKeyValue())); + } reader.close(); + deleteTestDir(fs); } @Test public void testBlockContainingKey() throws Exception { - testSeekBeforeInternals(TagUsage.NO_TAG); + testBlockContainingKeyInternals(TagUsage.NO_TAG); testBlockContainingKeyInternals(TagUsage.ONLY_TAG); testBlockContainingKeyInternals(TagUsage.PARTIAL_TAG); } protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileBlockIndex.BlockIndexReader blockIndexReader = @@ -291,5 +365,6 @@ public class TestSeekTo extends HBaseTestCase { assertEquals(1, blockIndexReader.rootBlockContainingKey( toKV("l", tagUsage))); reader.close(); + deleteTestDir(fs); } } \ No newline at end of file