From eee337f5c3352874ff225497066bf6f584f3ed0b Mon Sep 17 00:00:00 2001 From: Baiqiang Zhao Date: Tue, 19 Nov 2019 18:55:40 +0800 Subject: [PATCH] HBASE-23278 Add a table-level compaction progress display on the UI (#817) Signed-off-by: Guangxu Cheng --- .../resources/hbase-webapps/master/table.jsp | 542 ++++++++++++------ 1 file changed, 366 insertions(+), 176 deletions(-) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index b45d633d08e..3d5e2303bc8 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -159,10 +159,7 @@ if ( fqtn != null ) { table = new HTable(conf, fqtn); if (table.getTableDescriptor().getRegionReplication() > 1) { - tableHeader = "

Table Regions

"; withReplica = true; - } else { - tableHeader = "

Table Regions

NameRegion ServerReadRequestsWriteRequestsStorefileSizeNum.StorefilesMemSizeLocalityStart KeyEnd KeyReplicaID
"; } if ( !readOnly && action != null ) { %> @@ -213,70 +210,168 @@ if ( fqtn != null ) { <% if(fqtn.equals(TableName.META_TABLE_NAME.getNameAsString())) { %> -<%= tableHeader %> - -<% - // NOTE: Presumes meta with one or more replicas - for (int j = 0; j < numMetaReplicas; j++) { - HRegionInfo meta = RegionReplicaUtil.getRegionInfoForReplica( - HRegionInfo.FIRST_META_REGIONINFO, j); - ServerName metaLocation = metaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1); - for (int i = 0; i < 1; i++) { - String url = ""; - String readReq = "N/A"; - String writeReq = "N/A"; - String fileSize = "N/A"; - String fileCount = "N/A"; - String memSize = "N/A"; - float locality = 0.0f; +

Table Regions

+
+ +
+
+
NameRegion ServerReadRequestsWriteRequestsStorefileSizeNum.StorefilesMemSizeLocalityStart KeyEnd Key
+ + + + + + + + + + + + + <% + if (withReplica) { + %> + + <% + } + %> + + + + <% + // NOTE: Presumes meta with one or more replicas + for (int j = 0; j < numMetaReplicas; j++) { + HRegionInfo meta = RegionReplicaUtil.getRegionInfoForReplica( + HRegionInfo.FIRST_META_REGIONINFO, j); + ServerName metaLocation = metaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1); + for (int i = 0; i < 1; i++) { + String url = ""; + String readReq = "N/A"; + String writeReq = "N/A"; + String fileSize = "N/A"; + String fileCount = "N/A"; + String memSize = "N/A"; + float locality = 0.0f; - if (metaLocation != null) { - ServerLoad sl = master.getServerManager().getLoad(metaLocation); - // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. - url = "//" + URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation) + "/"; - if (sl != null) { - Map map = sl.getRegionsLoad(); - if (map.containsKey(meta.getRegionName())) { - RegionLoad load = map.get(meta.getRegionName()); - readReq = String.format("%,1d", load.getReadRequestsCount()); - writeReq = String.format("%,1d", load.getWriteRequestsCount()); - fileSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024); - fileCount = String.format("%,1d", load.getStorefiles()); - memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024); - locality = load.getDataLocality(); - } - } - } -%> - - <% - String metaLocationString = metaLocation != null ? - StringEscapeUtils.escapeHtml(metaLocation.getHostname().toString()) - + ":" + master.getRegionServerInfoPort(metaLocation) : - "(null)"; - %> - - - - - - - - - - -<% - if (withReplica) { -%> - -<% - } -%> - -<% } %> -<%} %> - -
NameRegion ServerReadRequestsWriteRequestsStorefileSizeNum.StorefilesMemSizeLocalityStart KeyEnd KeyReplicaID
<%= escapeXml(meta.getRegionNameAsString()) %><%= metaLocationString %><%= readReq%><%= writeReq%><%= fileSize%><%= fileCount%><%= memSize%><%= locality%><%= escapeXml(Bytes.toString(meta.getStartKey())) %><%= escapeXml(Bytes.toString(meta.getEndKey())) %><%= meta.getReplicaId() %>
+ if (metaLocation != null) { + ServerLoad sl = master.getServerManager().getLoad(metaLocation); + // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. + url = "//" + URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation) + "/"; + if (sl != null) { + Map map = sl.getRegionsLoad(); + if (map.containsKey(meta.getRegionName())) { + RegionLoad load = map.get(meta.getRegionName()); + readReq = String.format("%,1d", load.getReadRequestsCount()); + writeReq = String.format("%,1d", load.getWriteRequestsCount()); + fileSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024); + fileCount = String.format("%,1d", load.getStorefiles()); + memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024); + locality = load.getDataLocality(); + } + } + } + %> + + <% + String metaLocationString = metaLocation != null ? + StringEscapeUtils.escapeHtml(metaLocation.getHostname().toString()) + + ":" + master.getRegionServerInfoPort(metaLocation) : + "(null)"; + %> + <%= escapeXml(meta.getRegionNameAsString()) %> + <%= metaLocationString %> + <%= readReq%> + <%= writeReq%> + <%= fileSize%> + <%= fileCount%> + <%= memSize%> + <%= locality%> + <%= escapeXml(Bytes.toString(meta.getStartKey())) %> + <%= escapeXml(Bytes.toString(meta.getEndKey())) %> + <% + if (withReplica) { + %> + <%= meta.getReplicaId() %> + <% + } + %> + + <% } %> + <%} %> + + + +
+ + + + + + + + + + + + + <% + // NOTE: Presumes meta with one or more replicas + for (int j = 0; j < numMetaReplicas; j++) { + HRegionInfo meta = RegionReplicaUtil.getRegionInfoForReplica( + HRegionInfo.FIRST_META_REGIONINFO, j); + ServerName metaLocation = metaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1); + for (int i = 0; i < 1; i++) { + String url = ""; + long compactingKVs = 0; + long compactedKVs = 0; + String compactionProgress = ""; + + if (metaLocation != null) { + ServerLoad sl = master.getServerManager().getLoad(metaLocation); + // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. + url = "//" + URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation) + "/"; + if (sl != null) { + Map map = sl.getRegionsLoad(); + if (map.containsKey(meta.getRegionName())) { + RegionLoad load = map.get(meta.getRegionName()); + compactingKVs = load.getTotalCompactingKVs(); + compactedKVs = load.getCurrentCompactedKVs(); + if (compactingKVs > 0) { + compactionProgress = String.format("%.2f", 100 * ((float) + compactedKVs / compactingKVs)) + "%"; + } + } + } + } + %> + + <% + String metaLocationString = metaLocation != null ? + StringEscapeUtils.escapeHtml(metaLocation.getHostname().toString()) + + ":" + master.getRegionServerInfoPort(metaLocation) : + "(null)"; + %> + + + + + + + + <% } %> + <%} %> + +
NameRegion ServerNum. Compacting KVsNum. Compacted KVsRemaining KVsCompaction Progress
<%= escapeXml(meta.getRegionNameAsString()) %><%= metaLocationString %><%= String.format("%,1d", compactingKVs)%><%= String.format("%,1d", compactedKVs)%><%= String.format("%,1d", compactingKVs - compactedKVs)%><%= compactionProgress%>
+
+ + <%} else { Admin admin = master.getConnection().getAdmin(); try { %> @@ -363,6 +458,9 @@ if ( fqtn != null ) { long totalSize = 0; long totalStoreFileCount = 0; long totalMemSize = 0; + long totalCompactingKVs = 0; + long totalCompactedKVs = 0; + String percentDone = ""; String urlRegionServer = null; Map regDistribution = new TreeMap(); Map primaryRegDistribution = new TreeMap(); @@ -388,6 +486,8 @@ if ( fqtn != null ) { totalStoreFileCount += regionload.getStorefiles(); totalMemSize += regionload.getMemStoreSizeMB(); totalStoreFileSizeMB += regionload.getStorefileSizeMB(); + totalCompactingKVs += regionload.getTotalCompactingKVs(); + totalCompactedKVs += regionload.getCurrentCompactedKVs(); } else { RegionLoad load0 = new RegionLoad(ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(HBaseProtos.RegionSpecifier.newBuilder().setValue(ByteString.copyFrom(regionInfo.getRegionName())).build()).build()); regionsToLoad.put(regionInfo, load0); @@ -401,121 +501,209 @@ if ( fqtn != null ) { regionsToLoad.put(regionInfo, load0); } } + if (totalCompactingKVs > 0) { + percentDone = String.format("%.2f", 100 * + ((float) totalCompactedKVs / totalCompactingKVs)) + "%"; + } if(regions != null && regions.size() > 0) { %>

Table Regions

- - - - - - - - - - - - - -<% - if (withReplica) { -%> - -<% - } -%> - - - +
+ +
+
+
Name(<%= String.format("%,1d", regions.size())%>)Region ServerReadRequests
(<%= String.format("%,1d", totalReadReq)%>)		WriteRequests
(<%= String.format("%,1d", totalWriteReq)%>)		StorefileSize
(<%= StringUtils.byteDesc(totalSize*1024l*1024)%>)		Num.Storefiles
(<%= String.format("%,1d", totalStoreFileCount)%>)		MemSize
(<%= StringUtils.byteDesc(totalMemSize*1024l*1024)%>)		LocalityStart KeyEnd KeyReplicaID
+ + + + + + + + + + + + + <% + if (withReplica) { + %> + + <% + } + %> + + + + <% + List> entryList = new ArrayList>(regionsToLoad.entrySet()); + numRegions = regions.size(); + int numRegionsRendered = 0; + // render all regions + if (numRegionsToRender < 0) { + numRegionsToRender = numRegions; + } + for (Map.Entry hriEntry : entryList) { + HRegionInfo regionInfo = hriEntry.getKey(); + ServerName addr = regions.get(regionInfo); + RegionLoad load = hriEntry.getValue(); + String readReq = "N/A"; + String writeReq = "N/A"; + String regionSize = "N/A"; + String fileCount = "N/A"; + String memSize = "N/A"; + float locality = 0.0f; + if(load != null) { + readReq = String.format("%,1d", load.getReadRequestsCount()); + writeReq = String.format("%,1d", load.getWriteRequestsCount()); + regionSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024); + fileCount = String.format("%,1d", load.getStorefiles()); + memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024); + locality = load.getDataLocality(); + } -<% - List> entryList = new ArrayList>(regionsToLoad.entrySet()); - numRegions = regions.size(); - int numRegionsRendered = 0; - // render all regions - if (numRegionsToRender < 0) { - numRegionsToRender = numRegions; - } - for (Map.Entry hriEntry : entryList) { - HRegionInfo regionInfo = hriEntry.getKey(); - ServerName addr = regions.get(regionInfo); - RegionLoad load = hriEntry.getValue(); - String readReq = "N/A"; - String writeReq = "N/A"; - String regionSize = "N/A"; - String fileCount = "N/A"; - String memSize = "N/A"; - float locality = 0.0f; - if(load != null) { - readReq = String.format("%,1d", load.getReadRequestsCount()); - writeReq = String.format("%,1d", load.getWriteRequestsCount()); - regionSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024); - fileCount = String.format("%,1d", load.getStorefiles()); - memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024); - locality = load.getDataLocality(); - } + if (addr != null) { + ServerLoad sl = master.getServerManager().getLoad(addr); + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = + "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/"; + if(sl != null) { + Integer i = regDistribution.get(addr); + if (null == i) i = Integer.valueOf(0); + regDistribution.put(addr, i + 1); + if (withReplica && RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) { + i = primaryRegDistribution.get(addr); + if (null == i) i = Integer.valueOf(0); + primaryRegDistribution.put(addr, i+1); + } + } + } + if (numRegionsRendered < numRegionsToRender) { + numRegionsRendered++; + %> + + + <% + if (urlRegionServer != null) { + %> + + <% + } else { + %> + + <% + } + %> + + + + + + + + + <% + if (withReplica) { + %> + + <% + } + %> + + <% } %> + <% } %> + +
Name(<%= String.format("%,1d", regions.size())%>)Region ServerReadRequests
(<%= String.format("%,1d", totalReadReq)%>)		WriteRequests
(<%= String.format("%,1d", totalWriteReq)%>)		StorefileSize
(<%= StringUtils.byteDesc(totalSize*1024l*1024)%>)		Num.Storefiles
(<%= String.format("%,1d", totalStoreFileCount)%>)		MemSize
(<%= StringUtils.byteDesc(totalMemSize*1024l*1024)%>)		LocalityStart KeyEnd KeyReplicaID
<%= escapeXml(Bytes.toStringBinary(regionInfo.getRegionName())) %> + <%= StringEscapeUtils.escapeHtml(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> + not deployed<%= readReq%><%= writeReq%><%= regionSize%><%= fileCount%><%= memSize%><%= locality%><%= escapeXml(Bytes.toStringBinary(regionInfo.getStartKey()))%><%= escapeXml(Bytes.toStringBinary(regionInfo.getEndKey()))%><%= regionInfo.getReplicaId() %>
+ <% if (numRegions > numRegionsRendered) { + String allRegionsUrl = "?name=" + URLEncoder.encode(fqtn,"UTF-8") + "&numRegions=all"; + %> +

This table has <%= numRegions %> regions in total, in order to improve the page load time, + only <%= numRegionsRendered %> regions are displayed here, click + here to see all regions.

+ <% } %> + +
+ + + + + + + + + + + + + <% + numRegionsRendered = 0; + for (Map.Entry hriEntry : entryList) { + HRegionInfo regionInfo = hriEntry.getKey(); + ServerName addr = regions.get(regionInfo); + RegionLoad load = hriEntry.getValue(); + long compactingKVs = 0; + long compactedKVs = 0; + String compactionProgress = ""; + if(load != null) { + compactingKVs = load.getTotalCompactingKVs(); + compactedKVs = load.getCurrentCompactedKVs(); + if (compactingKVs > 0) { + compactionProgress = String.format("%.2f", 100 * ((float) + compactedKVs / compactingKVs)) + "%"; + } + } - if (addr != null) { - ServerLoad sl = master.getServerManager().getLoad(addr); - // This port might be wrong if RS actually ended up using something else. - urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/"; - if(sl != null) { - Integer i = regDistribution.get(addr); - if (null == i) i = Integer.valueOf(0); - regDistribution.put(addr, i + 1); - if (withReplica && RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) { - i = primaryRegDistribution.get(addr); - if (null == i) i = Integer.valueOf(0); - primaryRegDistribution.put(addr, i+1); - } - } - } - if (numRegionsRendered < numRegionsToRender) { - numRegionsRendered++; -%> - - - <% - if (urlRegionServer != null) { - %> - - <% - } else { - %> - - <% - } - %> - - - - - - - - - <% - if (withReplica) { - %> - - <% - } - %> - -<% } %> -<% } %> - -
Name(<%= String.format("%,1d", regions.size())%>)Region ServerNum. Compacting KVs
(<%= String.format("%,1d", totalCompactingKVs)%>)		Num. Compacted KVs
(<%= String.format("%,1d", totalCompactedKVs)%>)		Remaining KVs
(<%= String.format("%,1d", totalCompactingKVs - totalCompactedKVs)%>)		Compaction Progress
(<%= percentDone %>)
<%= escapeXml(Bytes.toStringBinary(regionInfo.getRegionName())) %> - <%= StringEscapeUtils.escapeHtml(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> - not deployed<%= readReq%><%= writeReq%><%= regionSize%><%= fileCount%><%= memSize%><%= locality%><%= escapeXml(Bytes.toStringBinary(regionInfo.getStartKey()))%><%= escapeXml(Bytes.toStringBinary(regionInfo.getEndKey()))%><%= regionInfo.getReplicaId() %>
-<% if (numRegions > numRegionsRendered) { - String allRegionsUrl = "?name=" + URLEncoder.encode(fqtn,"UTF-8") + "&numRegions=all"; -%> -

This table has <%= numRegions %> regions in total, in order to improve the page load time, - only <%= numRegionsRendered %> regions are displayed here, click - here to see all regions.

-<% } %> + if (addr != null) { + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = + "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/"; + } + if (numRegionsRendered < numRegionsToRender) { + numRegionsRendered++; + %> + + <%= escapeXml(Bytes.toStringBinary(regionInfo.getRegionName())) %> + <% + if (urlRegionServer != null) { + %> + + <%= StringEscapeUtils.escapeHtml(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> + + <% + } else { + %> + not deployed + <% + } + %> + <%= String.format("%,1d", compactingKVs)%> + <%= String.format("%,1d", compactedKVs)%> + <%= String.format("%,1d", compactingKVs - compactedKVs)%> + <%= compactionProgress%> + + <% } %> + <% } %> + + + <% if (numRegions > numRegionsRendered) { + String allRegionsUrl = "?name=" + URLEncoder.encode(fqtn,"UTF-8") + "&numRegions=all"; + %> +

This table has <%= numRegions %> regions in total, in order to improve the page load time, + only <%= numRegionsRendered %> regions are displayed here, click + here to see all regions.

+ <% } %> +
+ +

Regions by Region Server

<% if (withReplica) { @@ -652,6 +840,8 @@ $(document).ready(function() $("#regionServerTable").tablesorter(); $("#regionServerDetailsTable").tablesorter(); $("#tableRegionTable").tablesorter(); + $("#tableCompactStatsTable").tablesorter(); + $("#metaTableCompactStatsTable").tablesorter(); } );