HBASE-3496 HFile CLI Improvements
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067449 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
7a2e1fdd4c
commit
1cd8c4ae72
|
@ -72,6 +72,7 @@ Release 0.91.0 - Unreleased
|
||||||
copies
|
copies
|
||||||
HBASE-3305 Allow round-robin distribution for table created with
|
HBASE-3305 Allow round-robin distribution for table created with
|
||||||
multiple regions (ted yu via jgray)
|
multiple regions (ted yu via jgray)
|
||||||
|
HBASE-3496 HFile CLI Improvements
|
||||||
|
|
||||||
|
|
||||||
NEW FEATURES
|
NEW FEATURES
|
||||||
|
|
|
@ -1764,12 +1764,12 @@ public class HFile {
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
sb.append("size=" + count);
|
sb.append("size=" + count).append("\n");
|
||||||
for (int i = 0; i < count ; i++) {
|
for (int i = 0; i < count ; i++) {
|
||||||
sb.append(", ");
|
sb.append("key=").append(KeyValue.keyToString(blockKeys[i])).
|
||||||
sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])).
|
append("\n offset=").append(blockOffsets[i]).
|
||||||
append(", offset=").append(blockOffsets[i]).
|
append(", dataSize=" + blockDataSizes[i]).
|
||||||
append(", dataSize=" + blockDataSizes[i]);
|
append("\n");
|
||||||
}
|
}
|
||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
@ -1890,7 +1890,9 @@ public class HFile {
|
||||||
Options options = new Options();
|
Options options = new Options();
|
||||||
options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
|
options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
|
||||||
options.addOption("p", "printkv", false, "Print key/value pairs");
|
options.addOption("p", "printkv", false, "Print key/value pairs");
|
||||||
|
options.addOption("e", "printkey", false, "Print keys");
|
||||||
options.addOption("m", "printmeta", false, "Print meta data of file");
|
options.addOption("m", "printmeta", false, "Print meta data of file");
|
||||||
|
options.addOption("b", "printblocks", false, "Print block index meta data");
|
||||||
options.addOption("k", "checkrow", false,
|
options.addOption("k", "checkrow", false,
|
||||||
"Enable row order check; looks for out-of-order keys");
|
"Enable row order check; looks for out-of-order keys");
|
||||||
options.addOption("a", "checkfamily", false, "Enable family check");
|
options.addOption("a", "checkfamily", false, "Enable family check");
|
||||||
|
@ -1906,8 +1908,10 @@ public class HFile {
|
||||||
CommandLineParser parser = new PosixParser();
|
CommandLineParser parser = new PosixParser();
|
||||||
CommandLine cmd = parser.parse(options, args);
|
CommandLine cmd = parser.parse(options, args);
|
||||||
boolean verbose = cmd.hasOption("v");
|
boolean verbose = cmd.hasOption("v");
|
||||||
boolean printKeyValue = cmd.hasOption("p");
|
boolean printValue = cmd.hasOption("p");
|
||||||
|
boolean printKey = cmd.hasOption("e") || printValue;
|
||||||
boolean printMeta = cmd.hasOption("m");
|
boolean printMeta = cmd.hasOption("m");
|
||||||
|
boolean printBlocks = cmd.hasOption("b");
|
||||||
boolean checkRow = cmd.hasOption("k");
|
boolean checkRow = cmd.hasOption("k");
|
||||||
boolean checkFamily = cmd.hasOption("a");
|
boolean checkFamily = cmd.hasOption("a");
|
||||||
// get configuration, file system and get list of files
|
// get configuration, file system and get list of files
|
||||||
|
@ -1916,7 +1920,6 @@ public class HFile {
|
||||||
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
|
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
|
||||||
conf.set("fs.default.name",
|
conf.set("fs.default.name",
|
||||||
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
|
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
|
||||||
FileSystem fs = FileSystem.get(conf);
|
|
||||||
ArrayList<Path> files = new ArrayList<Path>();
|
ArrayList<Path> files = new ArrayList<Path>();
|
||||||
if (cmd.hasOption("f")) {
|
if (cmd.hasOption("f")) {
|
||||||
files.add(new Path(cmd.getOptionValue("f")));
|
files.add(new Path(cmd.getOptionValue("f")));
|
||||||
|
@ -1930,7 +1933,8 @@ public class HFile {
|
||||||
String enc = HRegionInfo.encodeRegionName(rn);
|
String enc = HRegionInfo.encodeRegionName(rn);
|
||||||
Path regionDir = new Path(tableDir, enc);
|
Path regionDir = new Path(tableDir, enc);
|
||||||
if (verbose) System.out.println("region dir -> " + regionDir);
|
if (verbose) System.out.println("region dir -> " + regionDir);
|
||||||
List<Path> regionFiles = getStoreFiles(fs, regionDir);
|
List<Path> regionFiles =
|
||||||
|
getStoreFiles(FileSystem.get(conf), regionDir);
|
||||||
if (verbose) System.out.println("Number of region files found -> " +
|
if (verbose) System.out.println("Number of region files found -> " +
|
||||||
regionFiles.size());
|
regionFiles.size());
|
||||||
if (verbose) {
|
if (verbose) {
|
||||||
|
@ -1944,6 +1948,7 @@ public class HFile {
|
||||||
// iterate over all files found
|
// iterate over all files found
|
||||||
for (Path file : files) {
|
for (Path file : files) {
|
||||||
if (verbose) System.out.println("Scanning -> " + file);
|
if (verbose) System.out.println("Scanning -> " + file);
|
||||||
|
FileSystem fs = file.getFileSystem(conf);
|
||||||
if (!fs.exists(file)) {
|
if (!fs.exists(file)) {
|
||||||
System.err.println("ERROR, file doesnt exist: " + file);
|
System.err.println("ERROR, file doesnt exist: " + file);
|
||||||
continue;
|
continue;
|
||||||
|
@ -1952,7 +1957,7 @@ public class HFile {
|
||||||
HFile.Reader reader = new HFile.Reader(fs, file, null, false, false);
|
HFile.Reader reader = new HFile.Reader(fs, file, null, false, false);
|
||||||
Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
|
Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
|
||||||
int count = 0;
|
int count = 0;
|
||||||
if (verbose || printKeyValue || checkRow || checkFamily) {
|
if (verbose || printKey || checkRow || checkFamily) {
|
||||||
// scan over file and read key/value's and check if requested
|
// scan over file and read key/value's and check if requested
|
||||||
HFileScanner scanner = reader.getScanner(false, false);
|
HFileScanner scanner = reader.getScanner(false, false);
|
||||||
scanner.seekTo();
|
scanner.seekTo();
|
||||||
|
@ -1960,9 +1965,12 @@ public class HFile {
|
||||||
do {
|
do {
|
||||||
KeyValue kv = scanner.getKeyValue();
|
KeyValue kv = scanner.getKeyValue();
|
||||||
// dump key value
|
// dump key value
|
||||||
if (printKeyValue) {
|
if (printKey) {
|
||||||
System.out.println("K: " + kv +
|
System.out.print("K: " + kv);
|
||||||
" V: " + Bytes.toStringBinary(kv.getValue()));
|
if (printValue) {
|
||||||
|
System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
|
||||||
|
}
|
||||||
|
System.out.println();
|
||||||
}
|
}
|
||||||
// check if rows are in order
|
// check if rows are in order
|
||||||
if (checkRow && pkv != null) {
|
if (checkRow && pkv != null) {
|
||||||
|
@ -1992,7 +2000,7 @@ public class HFile {
|
||||||
count++;
|
count++;
|
||||||
} while (scanner.next());
|
} while (scanner.next());
|
||||||
}
|
}
|
||||||
if (verbose || printKeyValue) {
|
if (verbose || printKey) {
|
||||||
System.out.println("Scanned kv count -> " + count);
|
System.out.println("Scanned kv count -> " + count);
|
||||||
}
|
}
|
||||||
// print meta data
|
// print meta data
|
||||||
|
@ -2033,6 +2041,10 @@ public class HFile {
|
||||||
System.out.println("Could not get bloom data from meta block");
|
System.out.println("Could not get bloom data from meta block");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (printBlocks) {
|
||||||
|
System.out.println("Block Index:");
|
||||||
|
System.out.println(reader.blockIndex);
|
||||||
|
}
|
||||||
reader.close();
|
reader.close();
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
|
@ -24,6 +24,7 @@ import java.io.IOException;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Bytes;
|
||||||
*/
|
*/
|
||||||
public class TestSeekTo extends HBaseTestCase {
|
public class TestSeekTo extends HBaseTestCase {
|
||||||
|
|
||||||
|
static KeyValue toKV(String row) {
|
||||||
|
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes
|
||||||
|
.toBytes("qualifier"), Bytes.toBytes("value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
static String toRowStr(KeyValue kv) {
|
||||||
|
return Bytes.toString(kv.getRow());
|
||||||
|
}
|
||||||
|
|
||||||
Path makeNewFile() throws IOException {
|
Path makeNewFile() throws IOException {
|
||||||
Path ncTFile = new Path(this.testDir, "basic.hfile");
|
Path ncTFile = new Path(this.testDir, "basic.hfile");
|
||||||
FSDataOutputStream fout = this.fs.create(ncTFile);
|
FSDataOutputStream fout = this.fs.create(ncTFile);
|
||||||
HFile.Writer writer = new HFile.Writer(fout, 40, "none", null);
|
int blocksize = toKV("a").getLength() * 3;
|
||||||
|
HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null);
|
||||||
// 4 bytes * 3 * 2 for each key/value +
|
// 4 bytes * 3 * 2 for each key/value +
|
||||||
// 3 for keys, 15 for values = 42 (woot)
|
// 3 for keys, 15 for values = 42 (woot)
|
||||||
writer.append(Bytes.toBytes("c"), Bytes.toBytes("value"));
|
writer.append(toKV("c"));
|
||||||
writer.append(Bytes.toBytes("e"), Bytes.toBytes("value"));
|
writer.append(toKV("e"));
|
||||||
writer.append(Bytes.toBytes("g"), Bytes.toBytes("value"));
|
writer.append(toKV("g"));
|
||||||
// block transition
|
// block transition
|
||||||
writer.append(Bytes.toBytes("i"), Bytes.toBytes("value"));
|
writer.append(toKV("i"));
|
||||||
writer.append(Bytes.toBytes("k"), Bytes.toBytes("value"));
|
writer.append(toKV("k"));
|
||||||
writer.close();
|
writer.close();
|
||||||
fout.close();
|
fout.close();
|
||||||
return ncTFile;
|
return ncTFile;
|
||||||
|
@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTestCase {
|
||||||
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
|
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
|
||||||
reader.loadFileInfo();
|
reader.loadFileInfo();
|
||||||
HFileScanner scanner = reader.getScanner(false, true);
|
HFileScanner scanner = reader.getScanner(false, true);
|
||||||
assertEquals(false, scanner.seekBefore(Bytes.toBytes("a")));
|
assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
|
||||||
|
|
||||||
assertEquals(false, scanner.seekBefore(Bytes.toBytes("c")));
|
assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
|
||||||
|
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("d")));
|
assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
|
||||||
assertEquals("c", scanner.getKeyString());
|
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("e")));
|
assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
|
||||||
assertEquals("c", scanner.getKeyString());
|
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("f")));
|
assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
|
||||||
assertEquals("e", scanner.getKeyString());
|
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("g")));
|
assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
|
||||||
assertEquals("e", scanner.getKeyString());
|
assertEquals("e", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("h")));
|
assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
|
||||||
assertEquals("g", scanner.getKeyString());
|
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("i")));
|
assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
|
||||||
assertEquals("g", scanner.getKeyString());
|
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("j")));
|
assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
|
||||||
assertEquals("i", scanner.getKeyString());
|
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("k")));
|
assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
|
||||||
assertEquals("i", scanner.getKeyString());
|
assertEquals("i", toRowStr(scanner.getKeyValue()));
|
||||||
assertEquals(true, scanner.seekBefore(Bytes.toBytes("l")));
|
assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
|
||||||
assertEquals("k", scanner.getKeyString());
|
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSeekTo() throws Exception {
|
public void testSeekTo() throws Exception {
|
||||||
|
@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTestCase {
|
||||||
assertEquals(2, reader.blockIndex.count);
|
assertEquals(2, reader.blockIndex.count);
|
||||||
HFileScanner scanner = reader.getScanner(false, true);
|
HFileScanner scanner = reader.getScanner(false, true);
|
||||||
// lies before the start of the file.
|
// lies before the start of the file.
|
||||||
assertEquals(-1, scanner.seekTo(Bytes.toBytes("a")));
|
assertEquals(-1, scanner.seekTo(toKV("a").getKey()));
|
||||||
|
|
||||||
assertEquals(1, scanner.seekTo(Bytes.toBytes("d")));
|
assertEquals(1, scanner.seekTo(toKV("d").getKey()));
|
||||||
assertEquals("c", scanner.getKeyString());
|
assertEquals("c", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
// Across a block boundary now.
|
// Across a block boundary now.
|
||||||
assertEquals(1, scanner.seekTo(Bytes.toBytes("h")));
|
assertEquals(1, scanner.seekTo(toKV("h").getKey()));
|
||||||
assertEquals("g", scanner.getKeyString());
|
assertEquals("g", toRowStr(scanner.getKeyValue()));
|
||||||
|
|
||||||
assertEquals(1, scanner.seekTo(Bytes.toBytes("l")));
|
assertEquals(1, scanner.seekTo(toKV("l").getKey()));
|
||||||
assertEquals("k", scanner.getKeyString());
|
assertEquals("k", toRowStr(scanner.getKeyValue()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBlockContainingKey() throws Exception {
|
public void testBlockContainingKey() throws Exception {
|
||||||
|
@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTestCase {
|
||||||
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
|
HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
|
||||||
reader.loadFileInfo();
|
reader.loadFileInfo();
|
||||||
System.out.println(reader.blockIndex.toString());
|
System.out.println(reader.blockIndex.toString());
|
||||||
|
int klen = toKV("a").getKey().length;
|
||||||
// falls before the start of the file.
|
// falls before the start of the file.
|
||||||
assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1));
|
assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(),
|
||||||
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1));
|
0, klen));
|
||||||
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1));
|
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0,
|
||||||
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1));
|
klen));
|
||||||
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1));
|
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0,
|
||||||
assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1));
|
klen));
|
||||||
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1));
|
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0,
|
||||||
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1));
|
klen));
|
||||||
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1));
|
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0,
|
||||||
assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1));
|
klen));
|
||||||
|
assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0,
|
||||||
|
klen));
|
||||||
|
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0,
|
||||||
|
klen));
|
||||||
|
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0,
|
||||||
|
klen));
|
||||||
|
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0,
|
||||||
|
klen));
|
||||||
|
assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0,
|
||||||
|
klen));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue