Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3496 HFile CLI Improvements 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067449 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Nicolas Spiegelberg 2011-02-05 14:37:36 +00:00
parent 7a2e1fdd4c
commit 1cd8c4ae72
3 changed files with 92 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -72,6 +72,7 @@ Release 0.91.0 - Unreleased
copies
HBASE-3305 Allow round-robin distribution for table created with
multiple regions (ted yu via jgray)
HBASE-3496 HFile CLI Improvements
NEW FEATURES

38
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -1764,12 +1764,12 @@ public class HFile {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("size=" + count);
sb.append("size=" + count).append("\n");
for (int i = 0; i < count ; i++) {
sb.append(", ");
sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])).
append(", offset=").append(blockOffsets[i]).
append(", dataSize=" + blockDataSizes[i]);
sb.append("key=").append(KeyValue.keyToString(blockKeys[i])).
append("\n offset=").append(blockOffsets[i]).
append(", dataSize=" + blockDataSizes[i]).
append("\n");
}
return sb.toString();
}
@ -1890,7 +1890,9 @@ public class HFile {
Options options = new Options();
options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
options.addOption("p", "printkv", false, "Print key/value pairs");
options.addOption("e", "printkey", false, "Print keys");
options.addOption("m", "printmeta", false, "Print meta data of file");
options.addOption("b", "printblocks", false, "Print block index meta data");
options.addOption("k", "checkrow", false,
"Enable row order check; looks for out-of-order keys");
options.addOption("a", "checkfamily", false, "Enable family check");
@ -1906,8 +1908,10 @@ public class HFile {
CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args);
boolean verbose = cmd.hasOption("v");
boolean printKeyValue = cmd.hasOption("p");
boolean printValue = cmd.hasOption("p");
boolean printKey = cmd.hasOption("e") || printValue;
boolean printMeta = cmd.hasOption("m");
boolean printBlocks = cmd.hasOption("b");
boolean checkRow = cmd.hasOption("k");
boolean checkFamily = cmd.hasOption("a");
// get configuration, file system and get list of files
@ -1916,7 +1920,6 @@ public class HFile {
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
conf.set("fs.default.name",
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
FileSystem fs = FileSystem.get(conf);
ArrayList<Path> files = new ArrayList<Path>();
if (cmd.hasOption("f")) {
files.add(new Path(cmd.getOptionValue("f")));
@ -1930,7 +1933,8 @@ public class HFile {
String enc = HRegionInfo.encodeRegionName(rn);
Path regionDir = new Path(tableDir, enc);
if (verbose) System.out.println("region dir -> " + regionDir);
List<Path> regionFiles = getStoreFiles(fs, regionDir);
List<Path> regionFiles =
getStoreFiles(FileSystem.get(conf), regionDir);
if (verbose) System.out.println("Number of region files found -> " +
regionFiles.size());
if (verbose) {
@ -1944,6 +1948,7 @@ public class HFile {
// iterate over all files found
for (Path file : files) {
if (verbose) System.out.println("Scanning -> " + file);
FileSystem fs = file.getFileSystem(conf);
if (!fs.exists(file)) {
System.err.println("ERROR, file doesnt exist: " + file);
continue;
@ -1952,7 +1957,7 @@ public class HFile {
HFile.Reader reader = new HFile.Reader(fs, file, null, false, false);
Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
int count = 0;
if (verbose || printKeyValue || checkRow || checkFamily) {
if (verbose || printKey || checkRow || checkFamily) {
// scan over file and read key/value's and check if requested
HFileScanner scanner = reader.getScanner(false, false);
scanner.seekTo();
@ -1960,9 +1965,12 @@ public class HFile {
do {
KeyValue kv = scanner.getKeyValue();
// dump key value
if (printKeyValue) {
System.out.println("K: " + kv +
" V: " + Bytes.toStringBinary(kv.getValue()));
if (printKey) {
System.out.print("K: " + kv);
if (printValue) {
System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
}
System.out.println();
}
// check if rows are in order
if (checkRow && pkv != null) {
@ -1992,7 +2000,7 @@ public class HFile {
count++;
} while (scanner.next());
}
if (verbose || printKeyValue) {
if (verbose || printKey) {
System.out.println("Scanned kv count -> " + count);
}
// print meta data
@ -2033,6 +2041,10 @@ public class HFile {
System.out.println("Could not get bloom data from meta block");
}
}
if (printBlocks) {
System.out.println("Block Index:");
System.out.println(reader.blockIndex);
}
reader.close();
}
} catch (Exception e) {

111
src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
View File

@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes;
/**
@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
public class TestSeekTo extends HBaseTestCase {
static KeyValue toKV(String row) {
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes
.toBytes("qualifier"), Bytes.toBytes("value"));
}
static String toRowStr(KeyValue kv) {
return Bytes.toString(kv.getRow());
}
Path makeNewFile() throws IOException {
Path ncTFile = new Path(this.testDir, "basic.hfile");
FSDataOutputStream fout = this.fs.create(ncTFile);
HFile.Writer writer = new HFile.Writer(fout, 40, "none", null);
int blocksize = toKV("a").getLength() * 3;
HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null);
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
writer.append(Bytes.toBytes("c"), Bytes.toBytes("value"));
writer.append(Bytes.toBytes("e"), Bytes.toBytes("value"));
writer.append(Bytes.toBytes("g"), Bytes.toBytes("value"));
writer.append(toKV("c"));
writer.append(toKV("e"));
writer.append(toKV("g"));
// block transition
writer.append(Bytes.toBytes("i"), Bytes.toBytes("value"));
writer.append(Bytes.toBytes("k"), Bytes.toBytes("value"));
writer.append(toKV("i"));
writer.append(toKV("k"));
writer.close();
fout.close();
return ncTFile;
@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTestCase {
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
reader.loadFileInfo();
HFileScanner scanner = reader.getScanner(false, true);
assertEquals(false, scanner.seekBefore(Bytes.toBytes("a")));
assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
assertEquals(false, scanner.seekBefore(Bytes.toBytes("c")));
assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("d")));
assertEquals("c", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
assertEquals("c", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("e")));
assertEquals("c", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
assertEquals("c", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("f")));
assertEquals("e", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
assertEquals("e", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("g")));
assertEquals("e", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
assertEquals("e", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(Bytes.toBytes("h")));
assertEquals("g", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(Bytes.toBytes("i")));
assertEquals("g", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(Bytes.toBytes("j")));
assertEquals("i", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(Bytes.toBytes("k")));
assertEquals("i", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(Bytes.toBytes("l")));
assertEquals("k", scanner.getKeyString());
assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
assertEquals("i", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
assertEquals("i", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
assertEquals("k", toRowStr(scanner.getKeyValue()));
}
public void testSeekTo() throws Exception {
@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTestCase {
assertEquals(2, reader.blockIndex.count);
HFileScanner scanner = reader.getScanner(false, true);
// lies before the start of the file.
assertEquals(-1, scanner.seekTo(Bytes.toBytes("a")));
assertEquals(-1, scanner.seekTo(toKV("a").getKey()));
assertEquals(1, scanner.seekTo(Bytes.toBytes("d")));
assertEquals("c", scanner.getKeyString());
assertEquals(1, scanner.seekTo(toKV("d").getKey()));
assertEquals("c", toRowStr(scanner.getKeyValue()));
// Across a block boundary now.
assertEquals(1, scanner.seekTo(Bytes.toBytes("h")));
assertEquals("g", scanner.getKeyString());
assertEquals(1, scanner.seekTo(toKV("h").getKey()));
assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(1, scanner.seekTo(Bytes.toBytes("l")));
assertEquals("k", scanner.getKeyString());
assertEquals(1, scanner.seekTo(toKV("l").getKey()));
assertEquals("k", toRowStr(scanner.getKeyValue()));
}
public void testBlockContainingKey() throws Exception {
@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTestCase {
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
reader.loadFileInfo();
System.out.println(reader.blockIndex.toString());
int klen = toKV("a").getKey().length;
// falls before the start of the file.
assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1));
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1));
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1));
assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(),
0, klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0,
klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0,
klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0,
klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0,
klen));
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0,
klen));
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0,
klen));
}
}