Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21171 [amv2] Tool to parse a directory of MasterProcWALs standalone 

Signed-off-by: Mike Drob <mdrob@apache.org>
This commit is contained in:
Michael Stack 2018-09-07 17:07:11 -07:00
parent 49501cd5b5
commit 64a32e6733
2 changed files with 33 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
View File

@ -47,6 +47,7 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
/** /**
* ProcedureWALPrettyPrinter prints the contents of a given ProcedureWAL file * ProcedureWALPrettyPrinter prints the contents of a given ProcedureWAL file
* @see WALProcedureStore#main(String[]) if you want to check parse of a directory of WALs.
*/ */
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@InterfaceStability.Evolving @InterfaceStability.Evolving

38
hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
View File

@ -43,9 +43,11 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
import org.apache.hadoop.hbase.procedure2.util.ByteSlot; import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
@ -64,6 +66,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.collections4.queue.Circula
/** /**
* WAL implementation of the ProcedureStore. * WAL implementation of the ProcedureStore.
* @see ProcedureWALPrettyPrinter for printing content of a single WAL.
* @see #main(String[]) to parse a directory of MasterWALProcs.
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Evolving @InterfaceStability.Evolving
@ -424,7 +428,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
it.next(); // Skip the current log it.next(); // Skip the current log
ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() { ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
long count = 0;
@Override @Override
public void setMaxProcId(long maxProcId) { public void setMaxProcId(long maxProcId) {
@ -433,11 +436,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
@Override @Override
public void load(ProcedureIterator procIter) throws IOException { public void load(ProcedureIterator procIter) throws IOException {
if ((++count % 1000) == 0) {
// Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs
// and procedures to load.
LOG.debug("Loaded {} procedures", this.count);
}
loader.load(procIter); loader.load(procIter);
} }
@ -1324,4 +1322,32 @@ public class WALProcedureStore extends ProcedureStoreBase {
log.close(); log.close();
return log; return log;
} }
/**
* Parses a directory of WALs building up ProcedureState.
* For testing parse and profiling.
* @param args Include pointer to directory of WAL files for a store instance to parse & load.
*/
public static void main(String [] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
if (args == null || args.length != 1) {
System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");
System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");
System.exit(-1);
}
WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,
new WALProcedureStore.LeaseRecovery() {
@Override
public void recoverFileLease(FileSystem fs, Path path) throws IOException {
// no-op
}
});
try {
store.start(16);
ProcedureExecutor pe = new ProcedureExecutor(conf, new Object()/*Pass anything*/, store);
pe.init(1, true);
} finally {
store.stop(true);
}
}
} }