Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25193: Add support for row prefix and type in the WAL Pretty Printer

Browse Source 
Closes #2555

Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
Signed-off-by: Bharath Vissapragada <bharathv@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Sandeep Pal 2020-10-23 12:57:59 +05:30 committed by Viraj Jasani
parent 8e387e9dfa
commit 9eb3a7918f
No known key found for this signature in database
GPG Key ID: B3D6C0B41C8ADFD5
2 changed files with 84 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java
View File

@ -68,7 +68,7 @@ public class HLogPrettyPrinter extends WALPrettyPrinter {
* Basic constructor that simply initializes values to reasonable defaults.
*/
public HLogPrettyPrinter() {
this(false, false, -1l, null, null, null, false, System.out);
this(false, false, -1L, null, null, null, null, false, System.out);
}
/**
@ -92,6 +92,9 @@ public class HLogPrettyPrinter extends WALPrettyPrinter {
* @param row
* when not null, serves as a filter; only log entries from this row
* will be printed
* @param rowPrefix
* when not null, serves as a filter; only log entries with row key
* having this prefix will be printed
* @param persistentOutput
* keeps a single list running for multiple files. if enabled, the
* endPersistentOutput() method must be used!
@ -100,10 +103,10 @@ public class HLogPrettyPrinter extends WALPrettyPrinter {
* PrettyPrinter's output.
*/
public HLogPrettyPrinter(boolean outputValues, boolean outputJSON,
long sequence, String table, String region, String row, boolean persistentOutput,
PrintStream out) {
super(outputValues, outputJSON, sequence, Collections.singleton(table), region, row,
false, persistentOutput, out);
long sequence, String table, String region, String row, String rowPrefix,
boolean persistentOutput, PrintStream out) {
super(outputValues, outputJSON, sequence, Collections.singleton(table), region,
row, rowPrefix,false, persistentOutput, out);
}
public static void main(String[] args) throws IOException {

97
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java
View File

@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@ -50,6 +51,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.GsonUtil;
import org.apache.hadoop.hbase.util.Strings;
import org.apache.hbase.thirdparty.com.google.gson.Gson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -85,7 +87,11 @@ public class WALPrettyPrinter {
// List of tables for filter
private final Set<String> tableSet;
private String region;
// exact row which needs to be filtered
private String row;
// prefix of rows which needs to be filtered
private String rowPrefix;
private boolean outputOnlyRowKey;
// enable in order to output a single list of transactions from several files
private boolean persistentOutput;
@ -101,7 +107,7 @@ public class WALPrettyPrinter {
* Basic constructor that simply initializes values to reasonable defaults.
*/
public WALPrettyPrinter() {
this(false, false, -1, new HashSet<String>(), null, null, false, false, System.out);
this(false, false, -1, new HashSet<String>(), null, null, null, false, false, System.out);
}
/**
@ -125,6 +131,9 @@ public class WALPrettyPrinter {
* @param row
* when not null, serves as a filter; only log entries from this row
* will be printed
* @param rowPrefix
* when not null, serves as a filter; only log entries with row key
* having this prefix will be printed
* @param persistentOutput
* keeps a single list running for multiple files. if enabled, the
* endPersistentOutput() method must be used!
@ -133,7 +142,7 @@ public class WALPrettyPrinter {
* PrettyPrinter's output.
*/
public WALPrettyPrinter(boolean outputValues, boolean outputJSON, long sequence,
Set<String> tableSet, String region, String row, boolean outputOnlyRowKey,
Set<String> tableSet, String region, String row, String rowPrefix, boolean outputOnlyRowKey,
boolean persistentOutput, PrintStream out) {
this.outputValues = outputValues;
this.outputJSON = outputJSON;
@ -141,6 +150,7 @@ public class WALPrettyPrinter {
this.tableSet = tableSet;
this.region = region;
this.row = row;
this.rowPrefix = rowPrefix;
this.outputOnlyRowKey = outputOnlyRowKey;
this.persistentOutput = persistentOutput;
if (persistentOutput) {
@ -218,6 +228,17 @@ public class WALPrettyPrinter {
this.row = row;
}
/**
* sets the rowPrefix key prefix by which output will be filtered
*
* @param rowPrefix
* when not null, serves as a filter; only log entries from this rowPrefix
* will be printed
*/
public void setRowPrefixFilter(String rowPrefix) {
this.rowPrefix = rowPrefix;
}
/**
* Option to print the row key only in case you just need the row keys from the WAL
*/
@ -338,15 +359,12 @@ public class WALPrettyPrinter {
List<Map<String, Object>> actions = new ArrayList<>();
for (Cell cell : edit.getCells()) {
// add atomic operation to txn
Map<String, Object> op = new HashMap<>(toStringMap(cell, outputOnlyRowKey));
if (outputValues) {
op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
Map<String, Object> op =
new HashMap<>(toStringMap(cell, outputOnlyRowKey, rowPrefix, row, outputValues));
if (op.isEmpty()) {
continue;
}
// check row output filter
if (row == null || ((String) op.get("row")).equals(row)) {
actions.add(op);
}
op.put("total_size_sum", CellUtil.estimatedHeapSizeOf(cell));
actions.add(op);
}
if (actions.isEmpty()) {
continue;
@ -363,15 +381,20 @@ public class WALPrettyPrinter {
out.print(GSON.toJson(txn));
} else {
// Pretty output, complete with indentation by atomic action
out.println(String.format(outputTmpl,
if (!outputOnlyRowKey) {
out.println(String.format(outputTmpl,
txn.get("sequence"), txn.get("table"), txn.get("region"), new Date(writeTime)));
}
for (int i = 0; i < actions.size(); i++) {
Map<String, Object> op = actions.get(i);
printCell(out, op, outputValues);
printCell(out, op, outputValues, outputOnlyRowKey);
}
}
out.println("edit heap size: " + entry.getEdit().heapSize());
out.println("position: " + log.getPosition());
if (!outputOnlyRowKey) {
out.println("edit heap size: " + entry.getEdit().heapSize());
out.println("position: " + log.getPosition());
}
}
} finally {
log.close();
@ -381,9 +404,17 @@ public class WALPrettyPrinter {
}
}
public static void printCell(PrintStream out, Map<String, Object> op, boolean outputValues) {
out.println("row=" + op.get("row") + ", column=" +
op.get("family") + ":" + op.get("qualifier"));
public static void printCell(PrintStream out, Map<String, Object> op,
boolean outputValues, boolean outputOnlyRowKey) {
String rowDetails = "row=" + op.get("row");
if (outputOnlyRowKey) {
out.println(rowDetails);
return;
}
rowDetails += ", column=" + op.get("family") + ":" + op.get("qualifier");
rowDetails += ", type=" + op.get("type");
out.println(rowDetails);
if (op.get("tag") != null) {
out.println(" tag: " + op.get("tag"));
}
@ -393,14 +424,24 @@ public class WALPrettyPrinter {
out.println("cell total size sum: " + op.get("total_size_sum"));
}
public static Map<String, Object> toStringMap(Cell cell, boolean printRowKeyOnly) {
public static Map<String, Object> toStringMap(Cell cell,
boolean printRowKeyOnly, String rowPrefix, String row, boolean outputValues) {
Map<String, Object> stringMap = new HashMap<>();
stringMap.put("row",
Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
String rowKey = Bytes.toStringBinary(cell.getRowArray(),
cell.getRowOffset(), cell.getRowLength());
// Row and row prefix are mutually options so both cannot be true at the
// same time. We can include checks in the same condition
// Check if any of the filters are satisfied by the row, if not return empty map
if ((!Strings.isEmpty(rowPrefix) && !rowKey.startsWith(rowPrefix)) ||
(!Strings.isEmpty(row) && !rowKey.equals(row))) {
return stringMap;
}
stringMap.put("row", rowKey);
if (printRowKeyOnly) {
return stringMap;
}
stringMap.put("type", KeyValue.Type.codeToType(cell.getTypeByte()));
stringMap.put("family",
Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
stringMap.put("qualifier", Bytes
@ -408,6 +449,7 @@ public class WALPrettyPrinter {
cell.getQualifierLength()));
stringMap.put("timestamp", cell.getTimestamp());
stringMap.put("vlen", cell.getValueLength());
stringMap.put("total_size_sum", CellUtil.estimatedHeapSizeOf(cell));
if (cell.getTagsLength() > 0) {
List<String> tagsString = new ArrayList<>();
Iterator<Tag> tagsIterator =
@ -419,11 +461,14 @@ public class WALPrettyPrinter {
}
stringMap.put("tag", tagsString);
}
if (outputValues) {
stringMap.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
}
return stringMap;
}
public static Map<String, Object> toStringMap(Cell cell) {
return toStringMap(cell, false);
return toStringMap(cell, false, null, null, false);
}
public static void main(String[] args) throws IOException {
@ -454,6 +499,7 @@ public class WALPrettyPrinter {
options.addOption("k", "outputOnlyRowKey", false,
"Print only row keys");
options.addOption("w", "row", true, "Row to filter by. Pass row name.");
options.addOption("f", "rowPrefix", true, "Row prefix to filter by.");
options.addOption("g", "goto", true, "Position to seek to in the file");
WALPrettyPrinter printer = new WALPrettyPrinter();
@ -487,8 +533,17 @@ public class WALPrettyPrinter {
printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
}
if (cmd.hasOption("w")) {
if (cmd.hasOption("f")) {
throw new ParseException("Row and Row-prefix cannot be supplied together");
}
printer.setRowFilter(cmd.getOptionValue("w"));
}
if (cmd.hasOption("f")) {
if (cmd.hasOption("w")) {
throw new ParseException("Row and Row-prefix cannot be supplied together");
}
printer.setRowPrefixFilter(cmd.getOptionValue("f"));
}
if (cmd.hasOption("g")) {
printer.setPosition(Long.parseLong(cmd.getOptionValue("g")));
}