Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3496 HFile CLI Improvements 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067449 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Nicolas Spiegelberg 2011-02-05 14:37:36 +00:00
parent 7a2e1fdd4c
commit 1cd8c4ae72
3 changed files with 92 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -72,6 +72,7 @@ Release 0.91.0 - Unreleased
copies copies
HBASE-3305 Allow round-robin distribution for table created with HBASE-3305 Allow round-robin distribution for table created with
multiple regions (ted yu via jgray) multiple regions (ted yu via jgray)
HBASE-3496 HFile CLI Improvements
NEW FEATURES NEW FEATURES

38
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -1764,12 +1764,12 @@ public class HFile {
@Override @Override
public String toString() { public String toString() {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("size=" + count); sb.append("size=" + count).append("\n");
for (int i = 0; i < count ; i++) { for (int i = 0; i < count ; i++) {
sb.append(", "); sb.append("key=").append(KeyValue.keyToString(blockKeys[i])).
sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])). append("\n offset=").append(blockOffsets[i]).
append(", offset=").append(blockOffsets[i]). append(", dataSize=" + blockDataSizes[i]).
append(", dataSize=" + blockDataSizes[i]); append("\n");
} }
return sb.toString(); return sb.toString();
} }
@ -1890,7 +1890,9 @@ public class HFile {
Options options = new Options(); Options options = new Options();
options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters"); options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
options.addOption("p", "printkv", false, "Print key/value pairs"); options.addOption("p", "printkv", false, "Print key/value pairs");
options.addOption("e", "printkey", false, "Print keys");
options.addOption("m", "printmeta", false, "Print meta data of file"); options.addOption("m", "printmeta", false, "Print meta data of file");
options.addOption("b", "printblocks", false, "Print block index meta data");
options.addOption("k", "checkrow", false, options.addOption("k", "checkrow", false,
"Enable row order check; looks for out-of-order keys"); "Enable row order check; looks for out-of-order keys");
options.addOption("a", "checkfamily", false, "Enable family check"); options.addOption("a", "checkfamily", false, "Enable family check");
@ -1906,8 +1908,10 @@ public class HFile {
CommandLineParser parser = new PosixParser(); CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args); CommandLine cmd = parser.parse(options, args);
boolean verbose = cmd.hasOption("v"); boolean verbose = cmd.hasOption("v");
boolean printKeyValue = cmd.hasOption("p"); boolean printValue = cmd.hasOption("p");
boolean printKey = cmd.hasOption("e") || printValue;
boolean printMeta = cmd.hasOption("m"); boolean printMeta = cmd.hasOption("m");
boolean printBlocks = cmd.hasOption("b");
boolean checkRow = cmd.hasOption("k"); boolean checkRow = cmd.hasOption("k");
boolean checkFamily = cmd.hasOption("a"); boolean checkFamily = cmd.hasOption("a");
// get configuration, file system and get list of files // get configuration, file system and get list of files
@ -1916,7 +1920,6 @@ public class HFile {
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
conf.set("fs.default.name", conf.set("fs.default.name",
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
FileSystem fs = FileSystem.get(conf);
ArrayList<Path> files = new ArrayList<Path>(); ArrayList<Path> files = new ArrayList<Path>();
if (cmd.hasOption("f")) { if (cmd.hasOption("f")) {
files.add(new Path(cmd.getOptionValue("f"))); files.add(new Path(cmd.getOptionValue("f")));
@ -1930,7 +1933,8 @@ public class HFile {
String enc = HRegionInfo.encodeRegionName(rn); String enc = HRegionInfo.encodeRegionName(rn);
Path regionDir = new Path(tableDir, enc); Path regionDir = new Path(tableDir, enc);
if (verbose) System.out.println("region dir -> " + regionDir); if (verbose) System.out.println("region dir -> " + regionDir);
List<Path> regionFiles = getStoreFiles(fs, regionDir); List<Path> regionFiles =
getStoreFiles(FileSystem.get(conf), regionDir);
if (verbose) System.out.println("Number of region files found -> " + if (verbose) System.out.println("Number of region files found -> " +
regionFiles.size()); regionFiles.size());
if (verbose) { if (verbose) {
@ -1944,6 +1948,7 @@ public class HFile {
// iterate over all files found // iterate over all files found
for (Path file : files) { for (Path file : files) {
if (verbose) System.out.println("Scanning -> " + file); if (verbose) System.out.println("Scanning -> " + file);
FileSystem fs = file.getFileSystem(conf);
if (!fs.exists(file)) { if (!fs.exists(file)) {
System.err.println("ERROR, file doesnt exist: " + file); System.err.println("ERROR, file doesnt exist: " + file);
continue; continue;
@ -1952,7 +1957,7 @@ public class HFile {
HFile.Reader reader = new HFile.Reader(fs, file, null, false, false); HFile.Reader reader = new HFile.Reader(fs, file, null, false, false);
Map<byte[],byte[]> fileInfo = reader.loadFileInfo(); Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
int count = 0; int count = 0;
if (verbose || printKeyValue || checkRow || checkFamily) { if (verbose || printKey || checkRow || checkFamily) {
// scan over file and read key/value's and check if requested // scan over file and read key/value's and check if requested
HFileScanner scanner = reader.getScanner(false, false); HFileScanner scanner = reader.getScanner(false, false);
scanner.seekTo(); scanner.seekTo();
@ -1960,9 +1965,12 @@ public class HFile {
do { do {
KeyValue kv = scanner.getKeyValue(); KeyValue kv = scanner.getKeyValue();
// dump key value // dump key value
if (printKeyValue) { if (printKey) {
System.out.println("K: " + kv + System.out.print("K: " + kv);
" V: " + Bytes.toStringBinary(kv.getValue())); if (printValue) {
System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
}
System.out.println();
} }
// check if rows are in order // check if rows are in order
if (checkRow && pkv != null) { if (checkRow && pkv != null) {
@ -1992,7 +2000,7 @@ public class HFile {
count++; count++;
} while (scanner.next()); } while (scanner.next());
} }
if (verbose || printKeyValue) { if (verbose || printKey) {
System.out.println("Scanned kv count -> " + count); System.out.println("Scanned kv count -> " + count);
} }
// print meta data // print meta data
@ -2033,6 +2041,10 @@ public class HFile {
System.out.println("Could not get bloom data from meta block"); System.out.println("Could not get bloom data from meta block");
} }
} }
if (printBlocks) {
System.out.println("Block Index:");
System.out.println(reader.blockIndex);
}
reader.close(); reader.close();
} }
} catch (Exception e) { } catch (Exception e) {

111
src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
View File

@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
/** /**
@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Bytes;
*/ */
public class TestSeekTo extends HBaseTestCase { public class TestSeekTo extends HBaseTestCase {
static KeyValue toKV(String row) {
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes
.toBytes("qualifier"), Bytes.toBytes("value"));
}
static String toRowStr(KeyValue kv) {
return Bytes.toString(kv.getRow());
}
Path makeNewFile() throws IOException { Path makeNewFile() throws IOException {
Path ncTFile = new Path(this.testDir, "basic.hfile"); Path ncTFile = new Path(this.testDir, "basic.hfile");
FSDataOutputStream fout = this.fs.create(ncTFile); FSDataOutputStream fout = this.fs.create(ncTFile);
HFile.Writer writer = new HFile.Writer(fout, 40, "none", null); int blocksize = toKV("a").getLength() * 3;
HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null);
// 4 bytes * 3 * 2 for each key/value + // 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot) // 3 for keys, 15 for values = 42 (woot)
writer.append(Bytes.toBytes("c"), Bytes.toBytes("value")); writer.append(toKV("c"));
writer.append(Bytes.toBytes("e"), Bytes.toBytes("value")); writer.append(toKV("e"));
writer.append(Bytes.toBytes("g"), Bytes.toBytes("value")); writer.append(toKV("g"));
// block transition // block transition
writer.append(Bytes.toBytes("i"), Bytes.toBytes("value")); writer.append(toKV("i"));
writer.append(Bytes.toBytes("k"), Bytes.toBytes("value")); writer.append(toKV("k"));
writer.close(); writer.close();
fout.close(); fout.close();
return ncTFile; return ncTFile;
@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTestCase {
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false); HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
reader.loadFileInfo(); reader.loadFileInfo();
HFileScanner scanner = reader.getScanner(false, true); HFileScanner scanner = reader.getScanner(false, true);
assertEquals(false, scanner.seekBefore(Bytes.toBytes("a"))); assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
assertEquals(false, scanner.seekBefore(Bytes.toBytes("c"))); assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("d"))); assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
assertEquals("c", scanner.getKeyString()); assertEquals("c", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("e"))); assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
assertEquals("c", scanner.getKeyString()); assertEquals("c", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("f"))); assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
assertEquals("e", scanner.getKeyString()); assertEquals("e", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("g"))); assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
assertEquals("e", scanner.getKeyString()); assertEquals("e", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("h"))); assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
assertEquals("g", scanner.getKeyString()); assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("i"))); assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
assertEquals("g", scanner.getKeyString()); assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("j"))); assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
assertEquals("i", scanner.getKeyString()); assertEquals("i", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("k"))); assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
assertEquals("i", scanner.getKeyString()); assertEquals("i", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("l"))); assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
assertEquals("k", scanner.getKeyString()); assertEquals("k", toRowStr(scanner.getKeyValue()));
} }
public void testSeekTo() throws Exception { public void testSeekTo() throws Exception {
@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTestCase {
assertEquals(2, reader.blockIndex.count); assertEquals(2, reader.blockIndex.count);
HFileScanner scanner = reader.getScanner(false, true); HFileScanner scanner = reader.getScanner(false, true);
// lies before the start of the file. // lies before the start of the file.
assertEquals(-1, scanner.seekTo(Bytes.toBytes("a"))); assertEquals(-1, scanner.seekTo(toKV("a").getKey()));
assertEquals(1, scanner.seekTo(Bytes.toBytes("d"))); assertEquals(1, scanner.seekTo(toKV("d").getKey()));
assertEquals("c", scanner.getKeyString()); assertEquals("c", toRowStr(scanner.getKeyValue()));
// Across a block boundary now. // Across a block boundary now.
assertEquals(1, scanner.seekTo(Bytes.toBytes("h"))); assertEquals(1, scanner.seekTo(toKV("h").getKey()));
assertEquals("g", scanner.getKeyString()); assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(1, scanner.seekTo(Bytes.toBytes("l"))); assertEquals(1, scanner.seekTo(toKV("l").getKey()));
assertEquals("k", scanner.getKeyString()); assertEquals("k", toRowStr(scanner.getKeyValue()));
} }
public void testBlockContainingKey() throws Exception { public void testBlockContainingKey() throws Exception {
@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTestCase {
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false); HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
reader.loadFileInfo(); reader.loadFileInfo();
System.out.println(reader.blockIndex.toString()); System.out.println(reader.blockIndex.toString());
int klen = toKV("a").getKey().length;
// falls before the start of the file. // falls before the start of the file.
assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1)); assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(),
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1)); 0, klen));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1)); assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0,
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1)); klen));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1)); assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0,
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1)); klen));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1)); assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0,
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1)); klen));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1)); assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0,
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1)); klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0,
klen));
} }
} }