From 92abf8ac5743849d4c32d6abf8fd33f161bab249 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 29 Dec 2015 17:41:04 -0800 Subject: [PATCH] HBASE-15060 Cull TestHFileV2 and HFileWriterFactory --- .../apache/hadoop/hbase/io/hfile/HFile.java | 10 +- .../hbase/io/hfile/HFileWriterFactory.java | 40 -- .../hbase/io/hfile/RandomKeyValueUtil.java | 104 ++++++ .../hbase/io/hfile/TestCacheOnWrite.java | 6 +- .../hbase/io/hfile/TestHFileBlockIndex.java | 10 +- .../TestHFileInlineToRootChunkConversion.java | 2 +- .../hbase/io/hfile/TestHFileWriterV2.java | 342 ------------------ .../hbase/io/hfile/TestHFileWriterV3.java | 9 +- .../hfile/TestLazyDataBlockDecompression.java | 6 +- .../hadoop/hbase/io/hfile/TestPrefetch.java | 4 +- .../hfile/TestSeekBeforeWithInlineBlocks.java | 6 +- .../TestCacheOnWriteInSchema.java | 6 +- .../regionserver/TestCompoundBloomFilter.java | 12 +- 13 files changed, 137 insertions(+), 420 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 35b232a6125..669ad92d728 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -241,7 +241,7 @@ public class HFile { * This variety of ways to construct writers is used throughout the code, and * we want to be able to swap writer implementations. */ - public static abstract class WriterFactory { + public static class WriterFactory { protected final Configuration conf; protected final CacheConfig cacheConf; protected FileSystem fs; @@ -309,12 +309,8 @@ public class HFile { else if (LOG.isDebugEnabled()) LOG.debug("Unable to set drop behind on " + path); } } - return createWriter(fs, path, ostream, - comparator, fileContext); + return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext); } - - protected abstract Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream, - CellComparator comparator, HFileContext fileContext) throws IOException; } /** The configuration key for HFile version to use for new files */ @@ -351,7 +347,7 @@ public class HFile { " hfiles only (but it can read v2 files without having to update hfile.format.version " + "in hbase-site.xml)"); case 3: - return new HFileWriterFactory(conf, cacheConf); + return new HFile.WriterFactory(conf, cacheConf); default: throw new IllegalArgumentException("Cannot create writer for HFile " + "format version " + version); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java deleted file mode 100644 index 92eb76088ae..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitationsME - * under the License. - */ -package org.apache.hadoop.hbase.io.hfile; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellComparator; - -public class HFileWriterFactory extends HFile.WriterFactory { - HFileWriterFactory(Configuration conf, CacheConfig cacheConf) { - super(conf, cacheConf); - } - - @Override - public HFile.Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream, - CellComparator comparator, HFileContext context) - throws IOException { - return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, context); - } -} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java new file mode 100644 index 00000000000..9507345f1d2 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import org.apache.hadoop.hbase.KeyValue; + +import java.util.Random; + +/** + * These helper methods generate random byte[]'s data for KeyValues + */ +public class RandomKeyValueUtil { + public static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_"; + private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64; + private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128; + + public static final char randomReadableChar(Random rand) { + int i = rand.nextInt(26 * 2 + 10 + 1); + if (i < 26) + return (char) ('A' + i); + i -= 26; + + if (i < 26) + return (char) ('a' + i); + i -= 26; + + if (i < 10) + return (char) ('0' + i); + i -= 10; + + assert i == 0; + return '_'; + } + + public static KeyValue randomKeyValue(Random rand) { + return new KeyValue(randomRowOrQualifier(rand), + COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand), + randomValue(rand)); + } + + public static byte[] randomRowOrQualifier(Random rand) { + StringBuilder field = new StringBuilder(); + int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH + + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH + - MIN_ROW_OR_QUALIFIER_LENGTH + 1); + for (int i = 0; i < fieldLen; ++i) + field.append(randomReadableChar(rand)); + return field.toString().getBytes(); + } + + public static byte[] randomValue(Random rand) { + StringBuilder v = new StringBuilder(); + for (int j = 0; j < 1 + rand.nextInt(2000); ++j) { + v.append((char) (32 + rand.nextInt(95))); + } + + byte[] valueBytes = v.toString().getBytes(); + return valueBytes; + } + + /** + * Generates a random key that is guaranteed to increase as the given index i + * increases. The result consists of a prefix, which is a deterministic + * increasing function of i, and a random suffix. + * + * @param rand + * random number generator to use + * @param i + * @return + */ + public static byte[] randomOrderedKey(Random rand, int i) { + StringBuilder k = new StringBuilder(); + + // The fixed-length lexicographically increasing part of the key. + for (int bitIndex = 31; bitIndex >= 0; --bitIndex) { + if ((i & (1 << bitIndex)) == 0) + k.append("a"); + else + k.append("b"); + } + + // A random-length random suffix of the key. + for (int j = 0; j < rand.nextInt(50); ++j) + k.append(randomReadableChar(rand)); + + byte[] keyBytes = k.toString().getBytes(); + return keyBytes; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index 2c957ef50d8..ccf59a4c6f6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -380,9 +380,9 @@ public class TestCacheOnWrite { .withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build(); byte[] cf = Bytes.toBytes("fam"); for (int i = 0; i < NUM_KV; ++i) { - byte[] row = TestHFileWriterV2.randomOrderedKey(rand, i); - byte[] qualifier = TestHFileWriterV2.randomRowOrQualifier(rand); - byte[] value = TestHFileWriterV2.randomValue(rand); + byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i); + byte[] qualifier = RandomKeyValueUtil.randomRowOrQualifier(rand); + byte[] value = RandomKeyValueUtil.randomValue(rand); KeyValue kv; if(useTags) { Tag t = new Tag((byte) 1, "visibility"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index 8a843a35d72..687d3cd2c72 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -280,7 +280,7 @@ public class TestHFileBlockIndex { byte[] qualifier = Bytes.toBytes("q"); for (int j = 0; j < 16; ++j) { byte[] k = - new KeyValue(TestHFileWriterV2.randomOrderedKey(rand, i * 16 + j), family, qualifier, + new KeyValue(RandomKeyValueUtil.randomOrderedKey(rand, i * 16 + j), family, qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey(); keys.add(k); if (j == 8) { @@ -348,7 +348,7 @@ public class TestHFileBlockIndex { int secondaryIndexEntries[] = new int[numTotalKeys]; for (int i = 0; i < numTotalKeys; ++i) { - byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i * 2); + byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i * 2); KeyValue cell = new KeyValue(k, Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("val")); //KeyValue cell = new KeyValue.KeyOnlyKeyValue(k, 0, k.length); @@ -473,7 +473,7 @@ public class TestHFileBlockIndex { c.writeRoot(dos); assertEquals(c.getRootSize(), dos.size()); - byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i); + byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i); numSubEntries += rand.nextInt(5) + 1; keys.add(k); c.add(k, getDummyFileOffset(i), getDummyOnDiskSize(i), numSubEntries); @@ -556,12 +556,12 @@ public class TestHFileBlockIndex { byte[] family = Bytes.toBytes("f"); byte[] qualifier = Bytes.toBytes("q"); for (int i = 0; i < NUM_KV; ++i) { - byte[] row = TestHFileWriterV2.randomOrderedKey(rand, i); + byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i); // Key will be interpreted by KeyValue.KEY_COMPARATOR KeyValue kv = new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), - TestHFileWriterV2.randomValue(rand)); + RandomKeyValueUtil.randomValue(rand)); byte[] k = kv.getKey(); writer.append(kv); keys[i] = k; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java index ab811f1db11..af4f2b86ad3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java @@ -55,7 +55,7 @@ public class TestHFileInlineToRootChunkConversion { CacheConfig cacheConf = new CacheConfig(conf); conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); HFileContext context = new HFileContextBuilder().withBlockSize(16).build(); - HFile.Writer hfw = new HFileWriterFactory(conf, cacheConf) + HFile.Writer hfw = new HFile.WriterFactory(conf, cacheConf) .withFileContext(context) .withPath(fs, hfPath).create(); List keys = new ArrayList(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java deleted file mode 100644 index 975073a0450..00000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java +++ /dev/null @@ -1,342 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io.hfile; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; -import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -import org.apache.hadoop.hbase.nio.ByteBuff; -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableUtils; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Testing writing a version 2 {@link HFile}. This is a low-level test written - * during the development of {@link HFileWriterImpl}. - */ -@Category({IOTests.class, SmallTests.class}) -public class TestHFileWriterV2 { - - private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class); - - private static final HBaseTestingUtility TEST_UTIL = - new HBaseTestingUtility(); - - private Configuration conf; - private FileSystem fs; - - @Before - public void setUp() throws IOException { - conf = TEST_UTIL.getConfiguration(); - fs = FileSystem.get(conf); - } - - @Test - public void testHFileFormatV2() throws IOException { - Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV2"); - final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ; - final int entryCount = 10000; - writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false); - } - - @Test - public void testMidKeyInHFile() throws IOException{ - Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), - "testMidKeyInHFile"); - Compression.Algorithm compressAlgo = Compression.Algorithm.NONE; - int entryCount = 50000; - writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true); - } - - private void writeDataAndReadFromHFile(Path hfilePath, - Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException { - - HFileContext context = new HFileContextBuilder() - .withBlockSize(4096) - .withCompression(compressAlgo) - .build(); - HFile.Writer writer = new HFileWriterFactory(conf, new CacheConfig(conf)) - .withPath(fs, hfilePath) - .withFileContext(context) - .create(); - - Random rand = new Random(9713312); // Just a fixed seed. - List keyValues = new ArrayList(entryCount); - - for (int i = 0; i < entryCount; ++i) { - byte[] keyBytes = randomOrderedKey(rand, i); - - // A random-length random value. - byte[] valueBytes = randomValue(rand); - KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes); - writer.append(keyValue); - keyValues.add(keyValue); - } - - // Add in an arbitrary order. They will be sorted lexicographically by - // the key. - writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C.")); - writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow")); - writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris")); - - writer.close(); - - - FSDataInputStream fsdis = fs.open(hfilePath); - - // A "manual" version of a new-format HFile reader. This unit test was - // written before the V2 reader was fully implemented. - - long fileSize = fs.getFileStatus(hfilePath).getLen(); - FixedFileTrailer trailer = - FixedFileTrailer.readFromStream(fsdis, fileSize); - - assertEquals(entryCount, trailer.getEntryCount()); - - HFileContext meta = new HFileContextBuilder() - .withHBaseCheckSum(true) - .withIncludesMvcc(false) - .withIncludesTags(false) - .withCompression(compressAlgo) - .build(); - - HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, meta); - // Comparator class name is stored in the trailer in version 2. - CellComparator comparator = trailer.createComparator(); - HFileBlockIndex.BlockIndexReader dataBlockIndexReader = - new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, - trailer.getNumDataIndexLevels()); - HFileBlockIndex.BlockIndexReader metaBlockIndexReader = - new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); - - HFileBlock.BlockIterator blockIter = blockReader.blockRange( - trailer.getLoadOnOpenDataOffset(), - fileSize - trailer.getTrailerSize()); - // Data index. We also read statistics about the block index written after - // the root level. - dataBlockIndexReader.readMultiLevelIndexRoot( - blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), - trailer.getDataIndexCount()); - - if (findMidKey) { - Cell midkey = dataBlockIndexReader.midkey(); - assertNotNull("Midkey should not be null", midkey); - } - - // Meta index. - metaBlockIndexReader.readRootIndex( - blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX) - .getByteStream(), trailer.getMetaIndexCount()); - // File info - FileInfo fileInfo = new FileInfo(); - fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); - byte [] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION); - boolean includeMemstoreTS = keyValueFormatVersion != null && - Bytes.toInt(keyValueFormatVersion) > 0; - - // Counters for the number of key/value pairs and the number of blocks - int entriesRead = 0; - int blocksRead = 0; - long memstoreTS = 0; - - // Scan blocks the way the reader would scan them - fsdis.seek(0); - long curBlockPos = 0; - while (curBlockPos <= trailer.getLastDataBlockOffset()) { - HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false); - assertEquals(BlockType.DATA, block.getBlockType()); - if (meta.isCompressedOrEncrypted()) { - assertFalse(block.isUnpacked()); - block = block.unpack(meta, blockReader); - } - ByteBuff buf = block.getBufferWithoutHeader(); - while (buf.hasRemaining()) { - int keyLen = buf.getInt(); - int valueLen = buf.getInt(); - - byte[] key = new byte[keyLen]; - buf.get(key); - - byte[] value = new byte[valueLen]; - buf.get(value); - - if (includeMemstoreTS) { - ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(), - buf.arrayOffset() + buf.position(), buf.remaining()); - DataInputStream data_input = new DataInputStream(byte_input); - - memstoreTS = WritableUtils.readVLong(data_input); - buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS)); - } - - // A brute-force check to see that all keys and values are correct. - KeyValue kv = keyValues.get(entriesRead); - assertTrue(Bytes.compareTo(key, kv.getKey()) == 0); - assertTrue(Bytes.compareTo(value, 0, value.length, kv.getValueArray(), kv.getValueOffset(), - kv.getValueLength()) == 0); - - ++entriesRead; - } - ++blocksRead; - curBlockPos += block.getOnDiskSizeWithHeader(); - } - LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead=" - + blocksRead); - assertEquals(entryCount, entriesRead); - - // Meta blocks. We can scan until the load-on-open data offset (which is - // the root block index offset in version 2) because we are not testing - // intermediate-level index blocks here. - - int metaCounter = 0; - while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) { - LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " + - trailer.getLoadOnOpenDataOffset()); - HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false) - .unpack(meta, blockReader); - assertEquals(BlockType.META, block.getBlockType()); - Text t = new Text(); - ByteBuff buf = block.getBufferWithoutHeader(); - if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) { - throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName()); - } - Text expectedText = - (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text( - "Moscow") : new Text("Washington, D.C.")); - assertEquals(expectedText, t); - LOG.info("Read meta block data: " + t); - ++metaCounter; - curBlockPos += block.getOnDiskSizeWithHeader(); - } - - fsdis.close(); - } - - - // Static stuff used by various HFile v2 unit tests - - public static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_"; - private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64; - private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128; - - /** - * Generates a random key that is guaranteed to increase as the given index i - * increases. The result consists of a prefix, which is a deterministic - * increasing function of i, and a random suffix. - * - * @param rand - * random number generator to use - * @param i - * @return - */ - public static byte[] randomOrderedKey(Random rand, int i) { - StringBuilder k = new StringBuilder(); - - // The fixed-length lexicographically increasing part of the key. - for (int bitIndex = 31; bitIndex >= 0; --bitIndex) { - if ((i & (1 << bitIndex)) == 0) - k.append("a"); - else - k.append("b"); - } - - // A random-length random suffix of the key. - for (int j = 0; j < rand.nextInt(50); ++j) - k.append(randomReadableChar(rand)); - - byte[] keyBytes = k.toString().getBytes(); - return keyBytes; - } - - public static byte[] randomValue(Random rand) { - StringBuilder v = new StringBuilder(); - for (int j = 0; j < 1 + rand.nextInt(2000); ++j) { - v.append((char) (32 + rand.nextInt(95))); - } - - byte[] valueBytes = v.toString().getBytes(); - return valueBytes; - } - - public static final char randomReadableChar(Random rand) { - int i = rand.nextInt(26 * 2 + 10 + 1); - if (i < 26) - return (char) ('A' + i); - i -= 26; - - if (i < 26) - return (char) ('a' + i); - i -= 26; - - if (i < 10) - return (char) ('0' + i); - i -= 10; - - assert i == 0; - return '_'; - } - - public static byte[] randomRowOrQualifier(Random rand) { - StringBuilder field = new StringBuilder(); - int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH - + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH - - MIN_ROW_OR_QUALIFIER_LENGTH + 1); - for (int i = 0; i < fieldLen; ++i) - field.append(randomReadableChar(rand)); - return field.toString().getBytes(); - } - - public static KeyValue randomKeyValue(Random rand) { - return new KeyValue(randomRowOrQualifier(rand), - COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand), - randomValue(rand)); - } - - -} - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java index 979c9f62ead..15aa912d042 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java @@ -120,7 +120,7 @@ public class TestHFileWriterV3 { .withBlockSize(4096) .withIncludesTags(useTags) .withCompression(compressAlgo).build(); - HFile.Writer writer = new HFileWriterFactory(conf, new CacheConfig(conf)) + HFile.Writer writer = new HFile.WriterFactory(conf, new CacheConfig(conf)) .withPath(fs, hfilePath) .withFileContext(context) .withComparator(CellComparator.COMPARATOR) @@ -130,10 +130,10 @@ public class TestHFileWriterV3 { List keyValues = new ArrayList(entryCount); for (int i = 0; i < entryCount; ++i) { - byte[] keyBytes = TestHFileWriterV2.randomOrderedKey(rand, i); + byte[] keyBytes = RandomKeyValueUtil.randomOrderedKey(rand, i); // A random-length random value. - byte[] valueBytes = TestHFileWriterV2.randomValue(rand); + byte[] valueBytes = RandomKeyValueUtil.randomValue(rand); KeyValue keyValue = null; if (useTags) { ArrayList tags = new ArrayList(); @@ -176,7 +176,7 @@ public class TestHFileWriterV3 { .withHBaseCheckSum(true).build(); HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, meta); - // Comparator class name is stored in the trailer in version 2. + // Comparator class name is stored in the trailer in version 3. CellComparator comparator = trailer.createComparator(); HFileBlockIndex.BlockIndexReader dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, @@ -297,6 +297,5 @@ public class TestHFileWriterV3 { fsdis.close(); } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java index 00674179ff9..5f73500d006 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java @@ -89,7 +89,7 @@ public class TestLazyDataBlockDecompression { */ private static void writeHFile(Configuration conf, CacheConfig cc, FileSystem fs, Path path, HFileContext cxt, int entryCount) throws IOException { - HFile.Writer writer = new HFileWriterFactory(conf, cc) + HFile.Writer writer = new HFile.WriterFactory(conf, cc) .withPath(fs, path) .withFileContext(cxt) .create(); @@ -100,8 +100,8 @@ public class TestLazyDataBlockDecompression { final byte[] qualifier = Bytes.toBytes("q"); for (int i = 0; i < entryCount; i++) { - byte[] keyBytes = TestHFileWriterV2.randomOrderedKey(rand, i); - byte[] valueBytes = TestHFileWriterV2.randomValue(rand); + byte[] keyBytes = RandomKeyValueUtil.randomOrderedKey(rand, i); + byte[] valueBytes = RandomKeyValueUtil.randomValue(rand); // make a real keyvalue so that hfile tool can examine it writer.append(new KeyValue(keyBytes, family, qualifier, valueBytes)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index 053a27e3603..69a77bfab60 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -111,8 +111,8 @@ public class TestPrefetch { final int rowLen = 32; for (int i = 0; i < NUM_KV; ++i) { - byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i); - byte[] v = TestHFileWriterV2.randomValue(RNG); + byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); + byte[] v = RandomKeyValueUtil.randomValue(RNG); int cfLen = RNG.nextInt(k.length - rowLen + 1); KeyValue kv = new KeyValue( k, 0, rowLen, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java index c9af3d3f516..0dbb4e06521 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java @@ -123,9 +123,9 @@ public class TestSeekBeforeWithInlineBlocks { .build(); for (int i = 0; i < NUM_KV; i++) { - byte[] row = TestHFileWriterV2.randomOrderedKey(RAND, i); - byte[] qual = TestHFileWriterV2.randomRowOrQualifier(RAND); - byte[] value = TestHFileWriterV2.randomValue(RAND); + byte[] row = RandomKeyValueUtil.randomOrderedKey(RAND, i); + byte[] qual = RandomKeyValueUtil.randomRowOrQualifier(RAND); + byte[] value = RandomKeyValueUtil.randomValue(RAND); KeyValue kv = new KeyValue(row, FAM, qual, value); storeFileWriter.append(kv); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index da3c17d1088..0916fe6ee9a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.TableName; @@ -49,7 +50,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileScanner; -import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2; import org.apache.hadoop.hbase.wal.DefaultWALProvider; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.util.Bytes; @@ -276,8 +276,8 @@ public class TestCacheOnWriteInSchema { private void writeStoreFile(StoreFile.Writer writer) throws IOException { final int rowLen = 32; for (int i = 0; i < NUM_KV; ++i) { - byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i); - byte[] v = TestHFileWriterV2.randomValue(rand); + byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i); + byte[] v = RandomKeyValueUtil.randomValue(rand); int cfLen = rand.nextInt(k.length - rowLen + 1); KeyValue kv = new KeyValue( k, 0, rowLen, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java index 83002103ca4..44e249d68ef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.client.Scan; @@ -53,7 +54,6 @@ import org.apache.hadoop.hbase.io.hfile.CompoundBloomFilterWriter; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.BloomFilterUtil; @@ -144,7 +144,7 @@ public class TestCompoundBloomFilter { private List createSortedKeyValues(Random rand, int n) { List kvList = new ArrayList(n); for (int i = 0; i < n; ++i) - kvList.add(TestHFileWriterV2.randomKeyValue(rand)); + kvList.add(RandomKeyValueUtil.randomKeyValue(rand)); Collections.sort(kvList, CellComparator.COMPARATOR); return kvList; } @@ -231,7 +231,7 @@ public class TestCompoundBloomFilter { Random rand = new Random(EVALUATION_SEED); int nTrials = NUM_KV[t] * 10; for (int i = 0; i < nTrials; ++i) { - byte[] query = TestHFileWriterV2.randomRowOrQualifier(rand); + byte[] query = RandomKeyValueUtil.randomRowOrQualifier(rand); if (isInBloom(scanner, query, bt, rand)) { numFalsePos += 1; } @@ -280,16 +280,16 @@ public class TestCompoundBloomFilter { private boolean isInBloom(StoreFileScanner scanner, byte[] row, BloomType bt, Random rand) { - return isInBloom(scanner, row, TestHFileWriterV2.randomRowOrQualifier(rand)); + return isInBloom(scanner, row, RandomKeyValueUtil.randomRowOrQualifier(rand)); } private boolean isInBloom(StoreFileScanner scanner, byte[] row, byte[] qualifier) { Scan scan = new Scan(row, row); - scan.addColumn(Bytes.toBytes(TestHFileWriterV2.COLUMN_FAMILY_NAME), qualifier); + scan.addColumn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME), qualifier); Store store = mock(Store.class); HColumnDescriptor hcd = mock(HColumnDescriptor.class); - when(hcd.getName()).thenReturn(Bytes.toBytes(TestHFileWriterV2.COLUMN_FAMILY_NAME)); + when(hcd.getName()).thenReturn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME)); when(store.getFamily()).thenReturn(hcd); return scanner.shouldUseScanner(scan, store, Long.MIN_VALUE); }