From a62f543c651cc53fff91e03799fc9d4befff8700 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 31 Oct 2014 15:41:57 -0700 Subject: [PATCH] HBASE-12361 Show data locality of region in table page (Liu Shaohui) --- .../org/apache/hadoop/hbase/RegionLoad.java | 11 ++ .../generated/ClusterStatusProtos.java | 168 +++++++++++++++--- .../src/main/protobuf/ClusterStatus.proto | 3 + .../tmpl/regionserver/RegionListTmpl.jamon | 2 + .../hbase/regionserver/HRegionServer.java | 5 +- .../resources/hbase-webapps/master/table.jsp | 11 +- 6 files changed, 172 insertions(+), 28 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java index 62e11374d9d..234c5aed910 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java @@ -160,6 +160,15 @@ public class RegionLoad { return regionLoadPB.getStoreUncompressedSizeMB(); } + /** + * @return the data locality of region in the regionserver. + */ + public float getDataLocality() { + if (regionLoadPB.hasDataLocality()) { + return regionLoadPB.getDataLocality(); + } + return 0.0f; + } /** * @see java.lang.Object#toString() */ @@ -205,6 +214,8 @@ public class RegionLoad { compactionProgressPct); sb = Strings.appendKeyValue(sb, "completeSequenceId", this.getCompleteSequenceId()); + sb = Strings.appendKeyValue(sb, "dataLocality", + this.getDataLocality()); return sb.toString(); } } diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java index c558485f03f..5bc44ff2dfe 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java @@ -2153,6 +2153,24 @@ public final class ClusterStatusProtos { * */ long getCompleteSequenceId(); + + // optional float data_locality = 16; + /** + * optional float data_locality = 16; + * + * 
+     ** The current data locality for region in the regionserver 
+     *
+ */ + boolean hasDataLocality(); + /** + * optional float data_locality = 16; + * + * 
+     ** The current data locality for region in the regionserver 
+     *
+ */ + float getDataLocality(); } /** * Protobuf type {@code RegionLoad} @@ -2288,6 +2306,11 @@ public final class ClusterStatusProtos { completeSequenceId_ = input.readUInt64(); break; } + case 133: { + bitField0_ |= 0x00008000; + dataLocality_ = input.readFloat(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -2706,6 +2729,30 @@ public final class ClusterStatusProtos { return completeSequenceId_; } + // optional float data_locality = 16; + public static final int DATA_LOCALITY_FIELD_NUMBER = 16; + private float dataLocality_; + /** + * optional float data_locality = 16; + * + * 
+     ** The current data locality for region in the regionserver 
+     *
+ */ + public boolean hasDataLocality() { + return ((bitField0_ & 0x00008000) == 0x00008000); + } + /** + * optional float data_locality = 16; + * + * 
+     ** The current data locality for region in the regionserver 
+     *
+ */ + public float getDataLocality() { + return dataLocality_; + } + private void initFields() { regionSpecifier_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance(); stores_ = 0; @@ -2722,6 +2769,7 @@ public final class ClusterStatusProtos { totalStaticIndexSizeKB_ = 0; totalStaticBloomSizeKB_ = 0; completeSequenceId_ = 0L; + dataLocality_ = 0F; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2788,6 +2836,9 @@ public final class ClusterStatusProtos { if (((bitField0_ & 0x00004000) == 0x00004000)) { output.writeUInt64(15, completeSequenceId_); } + if (((bitField0_ & 0x00008000) == 0x00008000)) { + output.writeFloat(16, dataLocality_); + } getUnknownFields().writeTo(output); } @@ -2857,6 +2908,10 @@ public final class ClusterStatusProtos { size += com.google.protobuf.CodedOutputStream .computeUInt64Size(15, completeSequenceId_); } + if (((bitField0_ & 0x00008000) == 0x00008000)) { + size += com.google.protobuf.CodedOutputStream + .computeFloatSize(16, dataLocality_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2955,6 +3010,10 @@ public final class ClusterStatusProtos { result = result && (getCompleteSequenceId() == other.getCompleteSequenceId()); } + result = result && (hasDataLocality() == other.hasDataLocality()); + if (hasDataLocality()) { + result = result && (Float.floatToIntBits(getDataLocality()) == Float.floatToIntBits(other.getDataLocality())); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -3028,6 +3087,11 @@ public final class ClusterStatusProtos { hash = (37 * hash) + COMPLETE_SEQUENCE_ID_FIELD_NUMBER; hash = (53 * hash) + hashLong(getCompleteSequenceId()); } + if (hasDataLocality()) { + hash = (37 * hash) + DATA_LOCALITY_FIELD_NUMBER; + hash = (53 * hash) + Float.floatToIntBits( + getDataLocality()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -3172,6 +3236,8 @@ public final class ClusterStatusProtos { bitField0_ = (bitField0_ & ~0x00002000); completeSequenceId_ = 0L; bitField0_ = (bitField0_ & ~0x00004000); + dataLocality_ = 0F; + bitField0_ = (bitField0_ & ~0x00008000); return this; } @@ -3264,6 +3330,10 @@ public final class ClusterStatusProtos { to_bitField0_ |= 0x00004000; } result.completeSequenceId_ = completeSequenceId_; + if (((from_bitField0_ & 0x00008000) == 0x00008000)) { + to_bitField0_ |= 0x00008000; + } + result.dataLocality_ = dataLocality_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -3325,6 +3395,9 @@ public final class ClusterStatusProtos { if (other.hasCompleteSequenceId()) { setCompleteSequenceId(other.getCompleteSequenceId()); } + if (other.hasDataLocality()) { + setDataLocality(other.getDataLocality()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -4215,6 +4288,55 @@ public final class ClusterStatusProtos { return this; } + // optional float data_locality = 16; + private float dataLocality_ ; + /** + * optional float data_locality = 16; + * + * 
+       ** The current data locality for region in the regionserver 
+       *
+ */ + public boolean hasDataLocality() { + return ((bitField0_ & 0x00008000) == 0x00008000); + } + /** + * optional float data_locality = 16; + * + * 
+       ** The current data locality for region in the regionserver 
+       *
+ */ + public float getDataLocality() { + return dataLocality_; + } + /** + * optional float data_locality = 16; + * + * 
+       ** The current data locality for region in the regionserver 
+       *
+ */ + public Builder setDataLocality(float value) { + bitField0_ |= 0x00008000; + dataLocality_ = value; + onChanged(); + return this; + } + /** + * optional float data_locality = 16; + * + * 
+       ** The current data locality for region in the regionserver 
+       *
+ */ + public Builder clearDataLocality() { + bitField0_ = (bitField0_ & ~0x00008000); + dataLocality_ = 0F; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:RegionLoad) } @@ -10350,7 +10472,7 @@ public final class ClusterStatusProtos { "PLITTING_NEW\020\r\022\017\n\013MERGING_NEW\020\016\"X\n\022Regio", "nInTransition\022\036\n\004spec\030\001 \002(\0132\020.RegionSpec" + "ifier\022\"\n\014region_state\030\002 \002(\0132\014.RegionStat" + - "e\"\320\003\n\nRegionLoad\022*\n\020region_specifier\030\001 \002" + + "e\"\347\003\n\nRegionLoad\022*\n\020region_specifier\030\001 \002" + "(\0132\020.RegionSpecifier\022\016\n\006stores\030\002 \001(\r\022\022\n\n" + "storefiles\030\003 \001(\r\022\"\n\032store_uncompressed_s" + "ize_MB\030\004 \001(\r\022\031\n\021storefile_size_MB\030\005 \001(\r\022" + @@ -10361,27 +10483,27 @@ public final class ClusterStatusProtos { "ompacted_KVs\030\013 \001(\004\022\032\n\022root_index_size_KB" + "\030\014 \001(\r\022\"\n\032total_static_index_size_KB\030\r \001" + "(\r\022\"\n\032total_static_bloom_size_KB\030\016 \001(\r\022\034" + - "\n\024complete_sequence_id\030\017 \001(\004\"\212\002\n\nServerL" + - "oad\022\032\n\022number_of_requests\030\001 \001(\r\022 \n\030total" + - "_number_of_requests\030\002 \001(\r\022\024\n\014used_heap_M" + - "B\030\003 \001(\r\022\023\n\013max_heap_MB\030\004 \001(\r\022!\n\014region_l" + - "oads\030\005 \003(\0132\013.RegionLoad\022\"\n\014coprocessors\030" + - "\006 \003(\0132\014.Coprocessor\022\031\n\021report_start_time" + - "\030\007 \001(\004\022\027\n\017report_end_time\030\010 \001(\004\022\030\n\020info_", - "server_port\030\t \001(\r\"O\n\016LiveServerInfo\022\033\n\006s" + - "erver\030\001 \002(\0132\013.ServerName\022 \n\013server_load\030" + - "\002 \002(\0132\013.ServerLoad\"\340\002\n\rClusterStatus\022/\n\r" + - "hbase_version\030\001 \001(\0132\030.HBaseVersionFileCo" + - "ntent\022%\n\014live_servers\030\002 \003(\0132\017.LiveServer" + - "Info\022!\n\014dead_servers\030\003 \003(\0132\013.ServerName\022" + - "2\n\025regions_in_transition\030\004 \003(\0132\023.RegionI" + - "nTransition\022\036\n\ncluster_id\030\005 \001(\0132\n.Cluste" + - "rId\022)\n\023master_coprocessors\030\006 \003(\0132\014.Copro" + - "cessor\022\033\n\006master\030\007 \001(\0132\013.ServerName\022#\n\016b", - "ackup_masters\030\010 \003(\0132\013.ServerName\022\023\n\013bala" + - "ncer_on\030\t \001(\010BF\n*org.apache.hadoop.hbase" + - ".protobuf.generatedB\023ClusterStatusProtos" + - "H\001\240\001\001" + "\n\024complete_sequence_id\030\017 \001(\004\022\025\n\rdata_loc" + + "ality\030\020 \001(\002\"\212\002\n\nServerLoad\022\032\n\022number_of_" + + "requests\030\001 \001(\r\022 \n\030total_number_of_reques" + + "ts\030\002 \001(\r\022\024\n\014used_heap_MB\030\003 \001(\r\022\023\n\013max_he" + + "ap_MB\030\004 \001(\r\022!\n\014region_loads\030\005 \003(\0132\013.Regi" + + "onLoad\022\"\n\014coprocessors\030\006 \003(\0132\014.Coprocess" + + "or\022\031\n\021report_start_time\030\007 \001(\004\022\027\n\017report_", + "end_time\030\010 \001(\004\022\030\n\020info_server_port\030\t \001(\r" + + "\"O\n\016LiveServerInfo\022\033\n\006server\030\001 \002(\0132\013.Ser" + + "verName\022 \n\013server_load\030\002 \002(\0132\013.ServerLoa" + + "d\"\340\002\n\rClusterStatus\022/\n\rhbase_version\030\001 \001" + + "(\0132\030.HBaseVersionFileContent\022%\n\014live_ser" + + "vers\030\002 \003(\0132\017.LiveServerInfo\022!\n\014dead_serv" + + "ers\030\003 \003(\0132\013.ServerName\0222\n\025regions_in_tra" + + "nsition\030\004 \003(\0132\023.RegionInTransition\022\036\n\ncl" + + "uster_id\030\005 \001(\0132\n.ClusterId\022)\n\023master_cop" + + "rocessors\030\006 \003(\0132\014.Coprocessor\022\033\n\006master\030", + "\007 \001(\0132\013.ServerName\022#\n\016backup_masters\030\010 \003" + + "(\0132\013.ServerName\022\023\n\013balancer_on\030\t \001(\010BF\n*" + + "org.apache.hadoop.hbase.protobuf.generat" + + "edB\023ClusterStatusProtosH\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -10405,7 +10527,7 @@ public final class ClusterStatusProtos { internal_static_RegionLoad_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_RegionLoad_descriptor, - new java.lang.String[] { "RegionSpecifier", "Stores", "Storefiles", "StoreUncompressedSizeMB", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "TotalCompactingKVs", "CurrentCompactedKVs", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "CompleteSequenceId", }); + new java.lang.String[] { "RegionSpecifier", "Stores", "Storefiles", "StoreUncompressedSizeMB", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "TotalCompactingKVs", "CurrentCompactedKVs", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "CompleteSequenceId", "DataLocality", }); internal_static_ServerLoad_descriptor = getDescriptor().getMessageTypes().get(3); internal_static_ServerLoad_fieldAccessorTable = new diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto index dbf00dcdaa2..7e7839564df 100644 --- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto @@ -110,6 +110,9 @@ message RegionLoad { /** the most recent sequence Id from cache flush */ optional uint64 complete_sequence_id = 15; + + /** The current data locality for region in the regionserver */ + optional float data_locality = 16; } /* Server-level protobufs */ diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index 3ff4cb6dd51..6ca8ec6dd72 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -154,6 +154,7 @@ Storefile Size Index Size Bloom Size + Data Locality <%for HRegionInfo r: onlineRegions %> @@ -171,6 +172,7 @@ <% load.getStorefileSizeMB() %>m <% load.getTotalStaticIndexSizeKB() %>k <% load.getTotalStaticBloomSizeKB() %>k + <% load.getDataLocality() %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 71f06f2f6a8..45e5558497f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1357,6 +1357,8 @@ public class HRegionServer extends HasThread implements (int) (store.getTotalStaticBloomSize() / 1024); } } + float dataLocality = + r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname()); if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } @@ -1379,7 +1381,8 @@ public class HRegionServer extends HasThread implements .setWriteRequestsCount(r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) - .setCompleteSequenceId(r.lastFlushSeqId); + .setCompleteSequenceId(r.lastFlushSeqId) + .setDataLocality(dataLocality); return regionLoadBldr.build(); } diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 519f88310f0..2b7829ca77c 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -48,10 +48,10 @@ String tableHeader; boolean withReplica = false; if (table.getTableDescriptor().getRegionReplication() > 1) { - tableHeader = "

Table Regions

"; + tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyRequestsReplicaID
"; withReplica = true; } else { - tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyLocalityRequestsReplicaID
"; + tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyRequests
"; } ServerName rl = metaTableLocator.getMetaRegionLocation(master.getZooKeeper()); boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); @@ -212,9 +212,10 @@ - + + <% } %>
NameRegion ServerStart KeyEnd KeyLocalityRequests
<%= escapeXml(meta.getRegionNameAsString()) %> <%= metaLocation.getHostname().toString() + ":" + master.getRegionServerInfoPort(metaLocation) %>- <%= escapeXml(Bytes.toString(meta.getStartKey())) %> <%= escapeXml(Bytes.toString(meta.getEndKey())) %>--
@@ -268,7 +269,7 @@ HRegionInfo regionInfo = hriEntry.getKey(); ServerName addr = hriEntry.getValue(); long req = 0; - + float locality = 0.0f; String urlRegionServer = null; if (addr != null) { @@ -277,6 +278,7 @@ Map map = sl.getRegionsLoad(); if (map.containsKey(regionInfo.getRegionName())) { req = map.get(regionInfo.getRegionName()).getRequestsCount(); + locality = map.get(regionInfo.getRegionName()).getDataLocality(); } Integer i = regDistribution.get(addr); if (null == i) i = Integer.valueOf(0); @@ -305,6 +307,7 @@ conf))) %> <%= escapeXml(Bytes.toStringBinary(HRegionInfo.getEndKeyForDisplay(regionInfo, conf))) %> + <%= locality%> <%= req%> <% if (withReplica) {