From 1cd8c4ae720198b40947ff70900a2d3d1c40905b Mon Sep 17 00:00:00 2001 From: Nicolas Spiegelberg Date: Sat, 5 Feb 2011 14:37:36 +0000 Subject: [PATCH] HBASE-3496 HFile CLI Improvements git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067449 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../apache/hadoop/hbase/io/hfile/HFile.java | 38 ++++-- .../hadoop/hbase/io/hfile/TestSeekTo.java | 113 ++++++++++-------- 3 files changed, 92 insertions(+), 60 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7949addd3aa..c55f9d7f5df 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -72,6 +72,7 @@ Release 0.91.0 - Unreleased copies HBASE-3305 Allow round-robin distribution for table created with multiple regions (ted yu via jgray) + HBASE-3496 HFile CLI Improvements NEW FEATURES diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index b9604430597..06f391ce00b 100644 --- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -1764,12 +1764,12 @@ public class HFile { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("size=" + count); + sb.append("size=" + count).append("\n"); for (int i = 0; i < count ; i++) { - sb.append(", "); - sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])). - append(", offset=").append(blockOffsets[i]). - append(", dataSize=" + blockDataSizes[i]); + sb.append("key=").append(KeyValue.keyToString(blockKeys[i])). + append("\n offset=").append(blockOffsets[i]). + append(", dataSize=" + blockDataSizes[i]). + append("\n"); } return sb.toString(); } @@ -1890,7 +1890,9 @@ public class HFile { Options options = new Options(); options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters"); options.addOption("p", "printkv", false, "Print key/value pairs"); + options.addOption("e", "printkey", false, "Print keys"); options.addOption("m", "printmeta", false, "Print meta data of file"); + options.addOption("b", "printblocks", false, "Print block index meta data"); options.addOption("k", "checkrow", false, "Enable row order check; looks for out-of-order keys"); options.addOption("a", "checkfamily", false, "Enable family check"); @@ -1906,8 +1908,10 @@ public class HFile { CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); boolean verbose = cmd.hasOption("v"); - boolean printKeyValue = cmd.hasOption("p"); + boolean printValue = cmd.hasOption("p"); + boolean printKey = cmd.hasOption("e") || printValue; boolean printMeta = cmd.hasOption("m"); + boolean printBlocks = cmd.hasOption("b"); boolean checkRow = cmd.hasOption("k"); boolean checkFamily = cmd.hasOption("a"); // get configuration, file system and get list of files @@ -1916,7 +1920,6 @@ public class HFile { conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); conf.set("fs.default.name", conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); - FileSystem fs = FileSystem.get(conf); ArrayList files = new ArrayList(); if (cmd.hasOption("f")) { files.add(new Path(cmd.getOptionValue("f"))); @@ -1930,7 +1933,8 @@ public class HFile { String enc = HRegionInfo.encodeRegionName(rn); Path regionDir = new Path(tableDir, enc); if (verbose) System.out.println("region dir -> " + regionDir); - List regionFiles = getStoreFiles(fs, regionDir); + List regionFiles = + getStoreFiles(FileSystem.get(conf), regionDir); if (verbose) System.out.println("Number of region files found -> " + regionFiles.size()); if (verbose) { @@ -1944,6 +1948,7 @@ public class HFile { // iterate over all files found for (Path file : files) { if (verbose) System.out.println("Scanning -> " + file); + FileSystem fs = file.getFileSystem(conf); if (!fs.exists(file)) { System.err.println("ERROR, file doesnt exist: " + file); continue; @@ -1952,7 +1957,7 @@ public class HFile { HFile.Reader reader = new HFile.Reader(fs, file, null, false, false); Map fileInfo = reader.loadFileInfo(); int count = 0; - if (verbose || printKeyValue || checkRow || checkFamily) { + if (verbose || printKey || checkRow || checkFamily) { // scan over file and read key/value's and check if requested HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); @@ -1960,9 +1965,12 @@ public class HFile { do { KeyValue kv = scanner.getKeyValue(); // dump key value - if (printKeyValue) { - System.out.println("K: " + kv + - " V: " + Bytes.toStringBinary(kv.getValue())); + if (printKey) { + System.out.print("K: " + kv); + if (printValue) { + System.out.print(" V: " + Bytes.toStringBinary(kv.getValue())); + } + System.out.println(); } // check if rows are in order if (checkRow && pkv != null) { @@ -1992,7 +2000,7 @@ public class HFile { count++; } while (scanner.next()); } - if (verbose || printKeyValue) { + if (verbose || printKey) { System.out.println("Scanned kv count -> " + count); } // print meta data @@ -2033,6 +2041,10 @@ public class HFile { System.out.println("Could not get bloom data from meta block"); } } + if (printBlocks) { + System.out.println("Block Index:"); + System.out.println(reader.blockIndex); + } reader.close(); } } catch (Exception e) { diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index 79445ed89b8..a1d0ef9d2ab 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.Bytes; /** @@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Bytes; */ public class TestSeekTo extends HBaseTestCase { + static KeyValue toKV(String row) { + return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes + .toBytes("qualifier"), Bytes.toBytes("value")); + } + + static String toRowStr(KeyValue kv) { + return Bytes.toString(kv.getRow()); + } + Path makeNewFile() throws IOException { Path ncTFile = new Path(this.testDir, "basic.hfile"); FSDataOutputStream fout = this.fs.create(ncTFile); - HFile.Writer writer = new HFile.Writer(fout, 40, "none", null); + int blocksize = toKV("a").getLength() * 3; + HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) - writer.append(Bytes.toBytes("c"), Bytes.toBytes("value")); - writer.append(Bytes.toBytes("e"), Bytes.toBytes("value")); - writer.append(Bytes.toBytes("g"), Bytes.toBytes("value")); + writer.append(toKV("c")); + writer.append(toKV("e")); + writer.append(toKV("g")); // block transition - writer.append(Bytes.toBytes("i"), Bytes.toBytes("value")); - writer.append(Bytes.toBytes("k"), Bytes.toBytes("value")); + writer.append(toKV("i")); + writer.append(toKV("k")); writer.close(); fout.close(); return ncTFile; @@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTestCase { HFile.Reader reader = new HFile.Reader(fs, p, null, false, false); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); - assertEquals(false, scanner.seekBefore(Bytes.toBytes("a"))); + assertEquals(false, scanner.seekBefore(toKV("a").getKey())); - assertEquals(false, scanner.seekBefore(Bytes.toBytes("c"))); + assertEquals(false, scanner.seekBefore(toKV("c").getKey())); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("d"))); - assertEquals("c", scanner.getKeyString()); + assertEquals(true, scanner.seekBefore(toKV("d").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("e"))); - assertEquals("c", scanner.getKeyString()); + assertEquals(true, scanner.seekBefore(toKV("e").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("f"))); - assertEquals("e", scanner.getKeyString()); + assertEquals(true, scanner.seekBefore(toKV("f").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("g"))); - assertEquals("e", scanner.getKeyString()); + assertEquals(true, scanner.seekBefore(toKV("g").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("h"))); - assertEquals("g", scanner.getKeyString()); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("i"))); - assertEquals("g", scanner.getKeyString()); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("j"))); - assertEquals("i", scanner.getKeyString()); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("k"))); - assertEquals("i", scanner.getKeyString()); - assertEquals(true, scanner.seekBefore(Bytes.toBytes("l"))); - assertEquals("k", scanner.getKeyString()); + assertEquals(true, scanner.seekBefore(toKV("h").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("i").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("j").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("k").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("l").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); } public void testSeekTo() throws Exception { @@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTestCase { assertEquals(2, reader.blockIndex.count); HFileScanner scanner = reader.getScanner(false, true); // lies before the start of the file. - assertEquals(-1, scanner.seekTo(Bytes.toBytes("a"))); + assertEquals(-1, scanner.seekTo(toKV("a").getKey())); - assertEquals(1, scanner.seekTo(Bytes.toBytes("d"))); - assertEquals("c", scanner.getKeyString()); + assertEquals(1, scanner.seekTo(toKV("d").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); // Across a block boundary now. - assertEquals(1, scanner.seekTo(Bytes.toBytes("h"))); - assertEquals("g", scanner.getKeyString()); + assertEquals(1, scanner.seekTo(toKV("h").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); - assertEquals(1, scanner.seekTo(Bytes.toBytes("l"))); - assertEquals("k", scanner.getKeyString()); + assertEquals(1, scanner.seekTo(toKV("l").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); } public void testBlockContainingKey() throws Exception { @@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTestCase { HFile.Reader reader = new HFile.Reader(fs, p, null, false, false); reader.loadFileInfo(); System.out.println(reader.blockIndex.toString()); + int klen = toKV("a").getKey().length; // falls before the start of the file. - assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1)); - assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1)); - assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1)); - assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1)); - assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1)); - assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1)); - assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1)); - assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1)); - assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1)); - assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1)); - - - + assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(), + 0, klen)); + assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0, + klen)); + assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0, + klen)); + assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0, + klen)); + assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0, + klen)); + assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0, + klen)); + assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0, + klen)); + assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0, + klen)); + assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0, + klen)); + assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0, + klen)); } -} \ No newline at end of file +}