From 9eb3a7918fb9fd5e445fb482c57324b90a89d117 Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Fri, 23 Oct 2020 12:57:59 +0530 Subject: [PATCH] HBASE-25193: Add support for row prefix and type in the WAL Pretty Printer Closes #2555 Signed-off-by: Wellington Chevreuil Signed-off-by: Bharath Vissapragada Signed-off-by: Duo Zhang Signed-off-by: Viraj Jasani --- .../regionserver/wal/HLogPrettyPrinter.java | 13 ++- .../hadoop/hbase/wal/WALPrettyPrinter.java | 97 +++++++++++++++---- 2 files changed, 84 insertions(+), 26 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java index d4b9d86024a..d151fec578a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java @@ -68,7 +68,7 @@ public class HLogPrettyPrinter extends WALPrettyPrinter { * Basic constructor that simply initializes values to reasonable defaults. */ public HLogPrettyPrinter() { - this(false, false, -1l, null, null, null, false, System.out); + this(false, false, -1L, null, null, null, null, false, System.out); } /** @@ -92,6 +92,9 @@ public class HLogPrettyPrinter extends WALPrettyPrinter { * @param row * when not null, serves as a filter; only log entries from this row * will be printed + * @param rowPrefix + * when not null, serves as a filter; only log entries with row key + * having this prefix will be printed * @param persistentOutput * keeps a single list running for multiple files. if enabled, the * endPersistentOutput() method must be used! @@ -100,10 +103,10 @@ public class HLogPrettyPrinter extends WALPrettyPrinter { * PrettyPrinter's output. */ public HLogPrettyPrinter(boolean outputValues, boolean outputJSON, - long sequence, String table, String region, String row, boolean persistentOutput, - PrintStream out) { - super(outputValues, outputJSON, sequence, Collections.singleton(table), region, row, - false, persistentOutput, out); + long sequence, String table, String region, String row, String rowPrefix, + boolean persistentOutput, PrintStream out) { + super(outputValues, outputJSON, sequence, Collections.singleton(table), region, + row, rowPrefix,false, persistentOutput, out); } public static void main(String[] args) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java index 093a7f2c3fe..773a7472fff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -50,6 +51,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.GsonUtil; +import org.apache.hadoop.hbase.util.Strings; import org.apache.hbase.thirdparty.com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +87,11 @@ public class WALPrettyPrinter { // List of tables for filter private final Set tableSet; private String region; + + // exact row which needs to be filtered private String row; + // prefix of rows which needs to be filtered + private String rowPrefix; private boolean outputOnlyRowKey; // enable in order to output a single list of transactions from several files private boolean persistentOutput; @@ -101,7 +107,7 @@ public class WALPrettyPrinter { * Basic constructor that simply initializes values to reasonable defaults. */ public WALPrettyPrinter() { - this(false, false, -1, new HashSet(), null, null, false, false, System.out); + this(false, false, -1, new HashSet(), null, null, null, false, false, System.out); } /** @@ -125,6 +131,9 @@ public class WALPrettyPrinter { * @param row * when not null, serves as a filter; only log entries from this row * will be printed + * @param rowPrefix + * when not null, serves as a filter; only log entries with row key + * having this prefix will be printed * @param persistentOutput * keeps a single list running for multiple files. if enabled, the * endPersistentOutput() method must be used! @@ -133,7 +142,7 @@ public class WALPrettyPrinter { * PrettyPrinter's output. */ public WALPrettyPrinter(boolean outputValues, boolean outputJSON, long sequence, - Set tableSet, String region, String row, boolean outputOnlyRowKey, + Set tableSet, String region, String row, String rowPrefix, boolean outputOnlyRowKey, boolean persistentOutput, PrintStream out) { this.outputValues = outputValues; this.outputJSON = outputJSON; @@ -141,6 +150,7 @@ public class WALPrettyPrinter { this.tableSet = tableSet; this.region = region; this.row = row; + this.rowPrefix = rowPrefix; this.outputOnlyRowKey = outputOnlyRowKey; this.persistentOutput = persistentOutput; if (persistentOutput) { @@ -218,6 +228,17 @@ public class WALPrettyPrinter { this.row = row; } + /** + * sets the rowPrefix key prefix by which output will be filtered + * + * @param rowPrefix + * when not null, serves as a filter; only log entries from this rowPrefix + * will be printed + */ + public void setRowPrefixFilter(String rowPrefix) { + this.rowPrefix = rowPrefix; + } + /** * Option to print the row key only in case you just need the row keys from the WAL */ @@ -338,15 +359,12 @@ public class WALPrettyPrinter { List> actions = new ArrayList<>(); for (Cell cell : edit.getCells()) { // add atomic operation to txn - Map op = new HashMap<>(toStringMap(cell, outputOnlyRowKey)); - if (outputValues) { - op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell))); + Map op = + new HashMap<>(toStringMap(cell, outputOnlyRowKey, rowPrefix, row, outputValues)); + if (op.isEmpty()) { + continue; } - // check row output filter - if (row == null || ((String) op.get("row")).equals(row)) { - actions.add(op); - } - op.put("total_size_sum", CellUtil.estimatedHeapSizeOf(cell)); + actions.add(op); } if (actions.isEmpty()) { continue; @@ -363,15 +381,20 @@ public class WALPrettyPrinter { out.print(GSON.toJson(txn)); } else { // Pretty output, complete with indentation by atomic action - out.println(String.format(outputTmpl, + if (!outputOnlyRowKey) { + out.println(String.format(outputTmpl, txn.get("sequence"), txn.get("table"), txn.get("region"), new Date(writeTime))); + } for (int i = 0; i < actions.size(); i++) { Map op = actions.get(i); - printCell(out, op, outputValues); + printCell(out, op, outputValues, outputOnlyRowKey); } } - out.println("edit heap size: " + entry.getEdit().heapSize()); - out.println("position: " + log.getPosition()); + if (!outputOnlyRowKey) { + out.println("edit heap size: " + entry.getEdit().heapSize()); + out.println("position: " + log.getPosition()); + + } } } finally { log.close(); @@ -381,9 +404,17 @@ public class WALPrettyPrinter { } } - public static void printCell(PrintStream out, Map op, boolean outputValues) { - out.println("row=" + op.get("row") + ", column=" + - op.get("family") + ":" + op.get("qualifier")); + public static void printCell(PrintStream out, Map op, + boolean outputValues, boolean outputOnlyRowKey) { + String rowDetails = "row=" + op.get("row"); + if (outputOnlyRowKey) { + out.println(rowDetails); + return; + } + + rowDetails += ", column=" + op.get("family") + ":" + op.get("qualifier"); + rowDetails += ", type=" + op.get("type"); + out.println(rowDetails); if (op.get("tag") != null) { out.println(" tag: " + op.get("tag")); } @@ -393,14 +424,24 @@ public class WALPrettyPrinter { out.println("cell total size sum: " + op.get("total_size_sum")); } - public static Map toStringMap(Cell cell, boolean printRowKeyOnly) { + public static Map toStringMap(Cell cell, + boolean printRowKeyOnly, String rowPrefix, String row, boolean outputValues) { Map stringMap = new HashMap<>(); - stringMap.put("row", - Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); + String rowKey = Bytes.toStringBinary(cell.getRowArray(), + cell.getRowOffset(), cell.getRowLength()); + // Row and row prefix are mutually options so both cannot be true at the + // same time. We can include checks in the same condition + // Check if any of the filters are satisfied by the row, if not return empty map + if ((!Strings.isEmpty(rowPrefix) && !rowKey.startsWith(rowPrefix)) || + (!Strings.isEmpty(row) && !rowKey.equals(row))) { + return stringMap; + } + stringMap.put("row", rowKey); if (printRowKeyOnly) { return stringMap; } + stringMap.put("type", KeyValue.Type.codeToType(cell.getTypeByte())); stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); stringMap.put("qualifier", Bytes @@ -408,6 +449,7 @@ public class WALPrettyPrinter { cell.getQualifierLength())); stringMap.put("timestamp", cell.getTimestamp()); stringMap.put("vlen", cell.getValueLength()); + stringMap.put("total_size_sum", CellUtil.estimatedHeapSizeOf(cell)); if (cell.getTagsLength() > 0) { List tagsString = new ArrayList<>(); Iterator tagsIterator = @@ -419,11 +461,14 @@ public class WALPrettyPrinter { } stringMap.put("tag", tagsString); } + if (outputValues) { + stringMap.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell))); + } return stringMap; } public static Map toStringMap(Cell cell) { - return toStringMap(cell, false); + return toStringMap(cell, false, null, null, false); } public static void main(String[] args) throws IOException { @@ -454,6 +499,7 @@ public class WALPrettyPrinter { options.addOption("k", "outputOnlyRowKey", false, "Print only row keys"); options.addOption("w", "row", true, "Row to filter by. Pass row name."); + options.addOption("f", "rowPrefix", true, "Row prefix to filter by."); options.addOption("g", "goto", true, "Position to seek to in the file"); WALPrettyPrinter printer = new WALPrettyPrinter(); @@ -487,8 +533,17 @@ public class WALPrettyPrinter { printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s"))); } if (cmd.hasOption("w")) { + if (cmd.hasOption("f")) { + throw new ParseException("Row and Row-prefix cannot be supplied together"); + } printer.setRowFilter(cmd.getOptionValue("w")); } + if (cmd.hasOption("f")) { + if (cmd.hasOption("w")) { + throw new ParseException("Row and Row-prefix cannot be supplied together"); + } + printer.setRowPrefixFilter(cmd.getOptionValue("f")); + } if (cmd.hasOption("g")) { printer.setPosition(Long.parseLong(cmd.getOptionValue("g"))); }