From 5e3ffb1db6e2d1a42295385c294574b8c8eeebbb Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Fri, 23 Oct 2020 12:23:36 +0530 Subject: [PATCH] HBASE-25193: Add support for row prefix and type in the WAL Pretty Printer Closes #2556 Signed-off-by: Wellington Chevreuil Signed-off-by: Bharath Vissapragada Signed-off-by: Duo Zhang Signed-off-by: Viraj Jasani --- .../region/WALProcedurePrettyPrinter.java | 2 +- .../hadoop/hbase/wal/WALPrettyPrinter.java | 89 ++++++++++++++----- 2 files changed, 70 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java index a4ed7339845..0e60709b5e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java @@ -105,7 +105,7 @@ public class WALProcedurePrettyPrinter extends AbstractHBaseTool { if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())) { // We could have cells other than procedure edits, for example, a flush marker - WALPrettyPrinter.printCell(out, op, false); + WALPrettyPrinter.printCell(out, op, false, false); continue; } long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java index a37efec610e..07bcb1067ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java @@ -46,7 +46,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.hbase.thirdparty.com.google.common.base.Strings; import org.apache.hbase.thirdparty.com.google.gson.Gson; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser; @@ -85,7 +85,12 @@ public class WALPrettyPrinter { // List of tables for filter private final Set tableSet; private String region; + + // exact row which needs to be filtered private String row; + // prefix of rows which needs to be filtered + private String rowPrefix; + private boolean outputOnlyRowKey; // enable in order to output a single list of transactions from several files private boolean persistentOutput; @@ -107,6 +112,7 @@ public class WALPrettyPrinter { tableSet = new HashSet<>(); region = null; row = null; + rowPrefix = null; outputOnlyRowKey = false; persistentOutput = false; firstTxn = true; @@ -181,6 +187,17 @@ public class WALPrettyPrinter { this.row = row; } + /** + * sets the rowPrefix key prefix by which output will be filtered + * + * @param rowPrefix + * when not null, serves as a filter; only log entries with rows + * having this prefix will be printed + */ + public void setRowPrefixFilter(String rowPrefix) { + this.rowPrefix = rowPrefix; + } + /** * Option to print the row key only in case you just need the row keys from the WAL */ @@ -301,15 +318,12 @@ public class WALPrettyPrinter { List> actions = new ArrayList<>(); for (Cell cell : edit.getCells()) { // add atomic operation to txn - Map op = new HashMap<>(toStringMap(cell, outputOnlyRowKey)); - if (outputValues) { - op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell))); + Map op = + new HashMap<>(toStringMap(cell, outputOnlyRowKey, rowPrefix, row, outputValues)); + if (op.isEmpty()) { + continue; } - // check row output filter - if (row == null || ((String) op.get("row")).equals(row)) { - actions.add(op); - } - op.put("total_size_sum", cell.heapSize()); + actions.add(op); } if (actions.isEmpty()) { continue; @@ -326,15 +340,19 @@ public class WALPrettyPrinter { out.print(GSON.toJson(txn)); } else { // Pretty output, complete with indentation by atomic action - out.println(String.format(outputTmpl, + if (!outputOnlyRowKey) { + out.println(String.format(outputTmpl, txn.get("sequence"), txn.get("table"), txn.get("region"), new Date(writeTime))); + } for (int i = 0; i < actions.size(); i++) { Map op = actions.get(i); - printCell(out, op, outputValues); + printCell(out, op, outputValues, outputOnlyRowKey); } } - out.println("edit heap size: " + entry.getEdit().heapSize()); - out.println("position: " + log.getPosition()); + if (!outputOnlyRowKey) { + out.println("edit heap size: " + entry.getEdit().heapSize()); + out.println("position: " + log.getPosition()); + } } } finally { log.close(); @@ -344,9 +362,17 @@ public class WALPrettyPrinter { } } - public static void printCell(PrintStream out, Map op, boolean outputValues) { - out.println("row=" + op.get("row") + ", type=" + op.get("type") + ", column=" + - op.get("family") + ":" + op.get("qualifier")); + public static void printCell(PrintStream out, Map op, + boolean outputValues, boolean outputOnlyRowKey) { + String rowDetails = "row=" + op.get("row"); + if (outputOnlyRowKey) { + out.println(rowDetails); + return; + } + + rowDetails += ", column=" + op.get("family") + ":" + op.get("qualifier"); + rowDetails += ", type=" + op.get("type"); + out.println(rowDetails); if (op.get("tag") != null) { out.println(" tag: " + op.get("tag")); } @@ -356,11 +382,20 @@ public class WALPrettyPrinter { out.println("cell total size sum: " + op.get("total_size_sum")); } - public static Map toStringMap(Cell cell, boolean printRowKeyOnly) { + public static Map toStringMap(Cell cell, + boolean printRowKeyOnly, String rowPrefix, String row, boolean outputValues) { Map stringMap = new HashMap<>(); - stringMap.put("row", - Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); + String rowKey = Bytes.toStringBinary(cell.getRowArray(), + cell.getRowOffset(), cell.getRowLength()); + // Row and row prefix are mutually options so both cannot be true at the + // same time. We can include checks in the same condition + // Check if any of the filters are satisfied by the row, if not return empty map + if ((!Strings.isNullOrEmpty(rowPrefix) && !rowKey.startsWith(rowPrefix)) || + (!Strings.isNullOrEmpty(row) && !rowKey.equals(row))) { + return stringMap; + } + stringMap.put("row", rowKey); if (printRowKeyOnly) { return stringMap; } @@ -372,6 +407,7 @@ public class WALPrettyPrinter { cell.getQualifierLength())); stringMap.put("timestamp", cell.getTimestamp()); stringMap.put("vlen", cell.getValueLength()); + stringMap.put("total_size_sum", cell.heapSize()); if (cell.getTagsLength() > 0) { List tagsString = new ArrayList<>(); Iterator tagsIterator = PrivateCellUtil.tagsIterator(cell); @@ -382,11 +418,14 @@ public class WALPrettyPrinter { } stringMap.put("tag", tagsString); } + if (outputValues) { + stringMap.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell))); + } return stringMap; } public static Map toStringMap(Cell cell) { - return toStringMap(cell, false); + return toStringMap(cell, false, null, null, false); } public static void main(String[] args) throws IOException { @@ -417,6 +456,7 @@ public class WALPrettyPrinter { options.addOption("k", "outputOnlyRowKey", false, "Print only row keys"); options.addOption("w", "row", true, "Row to filter by. Pass row name."); + options.addOption("f", "rowPrefix", true, "Row prefix to filter by."); options.addOption("g", "goto", true, "Position to seek to in the file"); WALPrettyPrinter printer = new WALPrettyPrinter(); @@ -450,8 +490,17 @@ public class WALPrettyPrinter { printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s"))); } if (cmd.hasOption("w")) { + if (cmd.hasOption("f")) { + throw new ParseException("Row and Row-prefix cannot be supplied together"); + } printer.setRowFilter(cmd.getOptionValue("w")); } + if (cmd.hasOption("f")) { + if (cmd.hasOption("w")) { + throw new ParseException("Row and Row-prefix cannot be supplied together"); + } + printer.setRowPrefixFilter(cmd.getOptionValue("f")); + } if (cmd.hasOption("g")) { printer.setPosition(Long.parseLong(cmd.getOptionValue("g"))); }