From 85793655d3e9f8993c82acd816ae96ec2747f4b7 Mon Sep 17 00:00:00 2001 From: bsglz <18031031@qq.com> Date: Wed, 5 Apr 2023 21:42:13 +0800 Subject: [PATCH] HBASE-27765 Add biggest cell related info into web ui (#5151) Signed-off-by: Duo Zhang (cherry picked from commit a363b76d815a71c4acd96abca1a2578af57ede78) --- .../tmpl/regionserver/RegionListTmpl.jamon | 20 +++++++++++++++++ .../hadoop/hbase/io/hfile/HFileInfo.java | 19 ++++++++++++++++ .../hbase/io/hfile/HFileWriterImpl.java | 22 ++++++++++++++++++- .../hbase-webapps/regionserver/region.jsp | 5 +++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index 1a73394a212..0df4d2763b0 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -28,6 +28,9 @@ org.apache.hadoop.hbase.client.RegionInfo; org.apache.hadoop.hbase.client.RegionInfoDisplay; org.apache.hadoop.hbase.regionserver.Region; + org.apache.hadoop.hbase.regionserver.HStoreFile; + org.apache.hadoop.hbase.regionserver.Store; + org.apache.hadoop.hbase.regionserver.StoreFile; org.apache.hadoop.hbase.ServerName; org.apache.hadoop.hbase.HBaseConfiguration; org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -168,6 +171,7 @@ Index Size Bloom Size Data Locality + Len Of Biggest Cell @@ -207,6 +211,21 @@ * TraditionalBinaryPrefix.KILO.value, "B", 1); } } + long lenOfBiggestCellInRegion = -1L; + Region region = regionServer.getRegion(r.getEncodedName()); + if (region != null) { + List stores = region.getStores(); + for (Store store : stores) { + Collection storeFiles = store.getStorefiles(); + for (StoreFile sf : storeFiles) { + long lenOfBiggestCell = ((HStoreFile)sf).getFileInfo().getHFileInfo().getLenOfBiggestCell(); + if (lenOfBiggestCellInRegion < lenOfBiggestCell) { + lenOfBiggestCellInRegion = lenOfBiggestCell; + } + } + } + } + <% displayName %> <%if load != null %> @@ -217,6 +236,7 @@ <% indexSizeStr %> <% bloomSizeStr %> <% load.getDataLocality() %> + <% String.format("%,1d", lenOfBiggestCellInRegion) %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java index 17bdadc461d..1f2e5ec6d96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java @@ -35,6 +35,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -74,6 +75,8 @@ public class HFileInfo implements SortedMap { static final byte[] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN"); static final byte[] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS"); static final byte[] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED"); + static final byte[] KEY_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX + "KEY_OF_BIGGEST_CELL"); + static final byte[] LEN_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX + "LEN_OF_BIGGEST_CELL"); public static final byte[] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN"); private final SortedMap map = new TreeMap<>(Bytes.BYTES_COMPARATOR); @@ -93,6 +96,10 @@ public class HFileInfo implements SortedMap { private int avgKeyLen = -1; /** Average value length read from file info */ private int avgValueLen = -1; + /** Biggest Cell in the file, key only. Filled in when we read in the file info */ + private Cell biggestCell = null; + /** Length of the biggest Cell */ + private long lenOfBiggestCell = -1; private boolean includesMemstoreTS = false; private boolean decodeMemstoreTS = false; @@ -435,6 +442,10 @@ public class HFileInfo implements SortedMap { if (get(HFileInfo.LASTKEY) != null) { lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY)); } + if (get(HFileInfo.KEY_OF_BIGGEST_CELL) != null) { + biggestCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.KEY_OF_BIGGEST_CELL)); + lenOfBiggestCell = Bytes.toLong(get(HFileInfo.LEN_OF_BIGGEST_CELL)); + } avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN)); avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN)); byte[] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION); @@ -511,6 +522,14 @@ public class HFileInfo implements SortedMap { return avgValueLen; } + public String getKeyOfBiggestCell() { + return CellUtil.toString(biggestCell, false); + } + + public long getLenOfBiggestCell() { + return lenOfBiggestCell; + } + public boolean shouldIncludeMemStoreTS() { return includesMemstoreTS; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java index eda5cde46a1..d2dfaf62106 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java @@ -34,7 +34,9 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.ByteBufferExtendedCell; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.MetaCellComparator; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -93,6 +95,11 @@ public class HFileWriterImpl implements HFile.Writer { /** Used for calculating the average value length. */ protected long totalValueLength = 0; + /** Len of the biggest cell. */ + protected long lenOfBiggestCell = 0; + /** Key of the biggest cell. */ + protected byte[] keyOfBiggestCell; + /** Total uncompressed bytes, maybe calculate a compression ratio later. */ protected long totalUncompressedBytes = 0; @@ -741,7 +748,10 @@ public class HFileWriterImpl implements HFile.Writer { totalKeyLength += PrivateCellUtil.estimatedSerializedSizeOfKey(cell); totalValueLength += cell.getValueLength(); - + if (lenOfBiggestCell < PrivateCellUtil.estimatedSerializedSizeOf(cell)) { + lenOfBiggestCell = PrivateCellUtil.estimatedSerializedSizeOf(cell); + keyOfBiggestCell = PrivateCellUtil.getCellKeySerializedAsKeyValueKey(cell); + } // Are we the first key in this block? if (firstCellInBlock == null) { // If cell is big, block will be closed and this firstCellInBlock reference will only last @@ -795,6 +805,16 @@ public class HFileWriterImpl implements HFile.Writer { // Average value length. int avgValueLen = entryCount == 0 ? 0 : (int) (totalValueLength / entryCount); fileInfo.append(HFileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false); + + // Biggest cell. + if (keyOfBiggestCell != null) { + fileInfo.append(HFileInfo.KEY_OF_BIGGEST_CELL, keyOfBiggestCell, false); + fileInfo.append(HFileInfo.LEN_OF_BIGGEST_CELL, Bytes.toBytes(lenOfBiggestCell), false); + LOG.debug("Len of the biggest cell in {} is {}, key is {}", + this.getPath() == null ? "" : this.getPath().toString(), lenOfBiggestCell, + CellUtil.toString(new KeyValue.KeyOnlyKeyValue(keyOfBiggestCell), false)); + } + if (hFileContext.isIncludesTags()) { // When tags are not being written in this file, MAX_TAGS_LEN is excluded // from the FileInfo diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp index 382783ea4ba..267407a288a 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp @@ -23,6 +23,7 @@ import="java.util.List" import="org.apache.hadoop.hbase.client.RegionInfoDisplay" import="org.apache.hadoop.hbase.regionserver.HRegionServer" + import="org.apache.hadoop.hbase.regionserver.HStoreFile" import="org.apache.hadoop.hbase.regionserver.Region" import="org.apache.hadoop.hbase.regionserver.Store" import="org.apache.hadoop.hbase.regionserver.StoreFile" @@ -69,12 +70,16 @@ Store File Size (MB) Modification time + Len Of Biggest Cell + Key Of Biggest Cell <% for(StoreFile sf : storeFiles) { %> <%= sf.getPath() %> <%= (int) (rs.getFileSystem().getLength(sf.getPath()) / 1024 / 1024) %> <%= new Date(sf.getModificationTimestamp()) %> + <%= String.format("%,1d", ((HStoreFile)sf).getFileInfo().getHFileInfo().getLenOfBiggestCell()) %> + <%= ((HStoreFile)sf).getFileInfo().getHFileInfo().getKeyOfBiggestCell() %> <% } %>