From 8742265d85a0262c21201b31c2f5fabe35890b0b Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 11 Dec 2019 08:56:23 -0800 Subject: [PATCH] HBASE-23554 Encoded regionname to regionname utility (#923) Adds shell command regioninfo: hbase(main):001:0> regioninfo '0e6aa5c19ae2b2627649dc7708ce27d0' {ENCODED => 0e6aa5c19ae2b2627649dc7708ce27d0, NAME => 'TestTable,,1575941375972.0e6aa5c19ae2b2627649dc7708ce27d0.', STARTKEY => '', ENDKEY => '00000000000000000000299441'} Took 0.4737 seconds Signed-off-by: Sean Busbey Signed-off-by: Duo Zhang --- .../org/apache/hadoop/hbase/HRegionInfo.java | 3 +- .../hadoop/hbase/client/RegionInfo.java | 61 +++++++++++-------- .../hbase/shaded/protobuf/ProtobufUtil.java | 18 +++--- .../hbase/master/MasterRpcServices.java | 49 ++++++++++----- .../hbase/regionserver/RSRpcServices.java | 3 + .../hadoop/hbase/client/TestAdmin2.java | 28 +++++++++ hbase-shell/src/main/ruby/shell.rb | 1 + .../main/ruby/shell/commands/regioninfo.rb | 47 ++++++++++++++ 8 files changed, 157 insertions(+), 53 deletions(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/regioninfo.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index fc039265c57..77602d521ad 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -456,8 +456,7 @@ public class HRegionInfo implements RegionInfo, Comparable { */ @Deprecated @InterfaceAudience.Private - public static byte [][] parseRegionName(final byte [] regionName) - throws IOException { + public static byte [][] parseRegionName(final byte [] regionName) throws IOException { return RegionInfo.parseRegionName(regionName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java index 7a03e052a31..2f9e88d6d77 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.util.HashKey; import org.apache.hadoop.hbase.util.JenkinsHash; import org.apache.hadoop.hbase.util.MD5Hash; import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.util.StringUtils; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -346,18 +345,15 @@ public interface RegionInfo { return parseRegionName(regionName)[1]; } - @InterfaceAudience.Private - static boolean isEncodedRegionName(byte[] regionName) throws IOException { - try { - parseRegionName(regionName); - return false; - } catch (IOException e) { - if (StringUtils.stringifyException(e) - .contains(INVALID_REGION_NAME_FORMAT_MESSAGE)) { - return true; - } - throw e; - } + /** + * Figure if the passed bytes represent an encoded region name or not. + * @param regionName A Region name either encoded or not. + * @return True if regionName represents an encoded name. + */ + @InterfaceAudience.Private // For use by internals only. + public static boolean isEncodedRegionName(byte[] regionName) throws IOException { + // If not parseable as region name, presume encoded. TODO: add stringency; e.g. if hex. + return parseRegionNameOrReturnNull(regionName) == null && regionName.length <= MD5_HEX_LENGTH; } /** @@ -596,15 +592,28 @@ public interface RegionInfo { /** * Separate elements of a regionName. - * @return Array of byte[] containing tableName, startKey and id + * @return Array of byte[] containing tableName, startKey and id OR null if + * not parseable as a region name. + * @throws IOException if not parseable as regionName. */ - static byte [][] parseRegionName(final byte[] regionName) - throws IOException { - // Region name is of the format: - // tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.] - // startkey can contain the delimiter (',') so we parse from the start and end + static byte [][] parseRegionName(final byte[] regionName) throws IOException { + byte [][] result = parseRegionNameOrReturnNull(regionName); + if (result == null) { + throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE + ": " + Bytes.toStringBinary(regionName)); + } + return result; + } - // parse from start + /** + * Separate elements of a regionName. + * Region name is of the format: + * tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]. + * Startkey can contain the delimiter (',') so we parse from the start and then parse from + * the end. + * @return Array of byte[] containing tableName, startKey and id OR null if not parseable + * as a region name. + */ + static byte [][] parseRegionNameOrReturnNull(final byte[] regionName) { int offset = -1; for (int i = 0; i < regionName.length; i++) { if (regionName[i] == HConstants.DELIMITER) { @@ -613,8 +622,7 @@ public interface RegionInfo { } } if (offset == -1) { - throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE - + ": " + Bytes.toStringBinary(regionName)); + return null; } byte[] tableName = new byte[offset]; System.arraycopy(regionName, 0, tableName, 0, offset); @@ -622,9 +630,9 @@ public interface RegionInfo { int endOffset = regionName.length; // check whether regionName contains encodedName - if (regionName.length > MD5_HEX_LENGTH + 2 - && regionName[regionName.length-1] == ENC_SEPARATOR - && regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) { + if (regionName.length > MD5_HEX_LENGTH + 2 && + regionName[regionName.length-1] == ENC_SEPARATOR && + regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) { endOffset = endOffset - MD5_HEX_LENGTH - 2; } @@ -645,8 +653,7 @@ public interface RegionInfo { } } if (offset == -1) { - throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE - + ": " + Bytes.toStringBinary(regionName)); + return null; } byte [] startKey = HConstants.EMPTY_BYTE_ARRAY; if(offset != tableName.length + 1) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index c6e48d42014..9176e495f27 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -1714,21 +1714,21 @@ public final class ProtobufUtil { // Start helpers for Admin /** - * A helper to retrieve region info given a region name - * using admin protocol. + * A helper to retrieve region info given a region name or an + * encoded region name using admin protocol. * - * @param admin - * @param regionName * @return the retrieved region info - * @throws IOException */ - public static org.apache.hadoop.hbase.client.RegionInfo getRegionInfo(final RpcController controller, - final AdminService.BlockingInterface admin, final byte[] regionName) throws IOException { + public static org.apache.hadoop.hbase.client.RegionInfo getRegionInfo( + final RpcController controller, final AdminService.BlockingInterface admin, + final byte[] regionName) throws IOException { try { GetRegionInfoRequest request = + org.apache.hadoop.hbase.client.RegionInfo.isEncodedRegionName(regionName)? + GetRegionInfoRequest.newBuilder().setRegion(RequestConverter. + buildRegionSpecifier(RegionSpecifierType.ENCODED_REGION_NAME, regionName)).build(): RequestConverter.buildGetRegionInfoRequest(regionName); - GetRegionInfoResponse response = - admin.getRegionInfo(controller, request); + GetRegionInfoResponse response = admin.getRegionInfo(controller, request); return toRegionInfo(response.getRegionInfo()); } catch (ServiceException se) { throw getRemoteException(se); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 5ef32dae010..612c731ab31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1,5 +1,4 @@ -/** - * +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -1775,24 +1774,44 @@ public class MasterRpcServices extends RSRpcServices } } + /** + * This method implements Admin getRegionInfo. On RegionServer, it is + * able to return RegionInfo and detail. On Master, it just returns + * RegionInfo. On Master it has been hijacked to return Mob detail. + * Master implementation is good for querying full region name if + * you only have the encoded name (useful around region replicas + * for example which do not have a row in hbase:meta). + */ @Override @QosPriority(priority=HConstants.ADMIN_QOS) public GetRegionInfoResponse getRegionInfo(final RpcController controller, final GetRegionInfoRequest request) throws ServiceException { - byte[] regionName = request.getRegion().getValue().toByteArray(); - TableName tableName = RegionInfo.getTable(regionName); - if (MobUtils.isMobRegionName(tableName, regionName)) { - // a dummy region info contains the compaction state. - RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName); - GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder(); - builder.setRegionInfo(ProtobufUtil.toRegionInfo(mobRegionInfo)); - if (request.hasCompactionState() && request.getCompactionState()) { - builder.setCompactionState(master.getMobCompactionState(tableName)); - } - return builder.build(); - } else { - return super.getRegionInfo(controller, request); + RegionInfo ri = null; + try { + ri = getRegionInfo(request.getRegion()); + } catch(UnknownRegionException ure) { + throw new ServiceException(ure); } + GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder(); + if (ri != null) { + builder.setRegionInfo(ProtobufUtil.toRegionInfo(ri)); + } else { + // Is it a MOB name? These work differently. + byte [] regionName = request.getRegion().getValue().toByteArray(); + TableName tableName = RegionInfo.getTable(regionName); + if (MobUtils.isMobRegionName(tableName, regionName)) { + // a dummy region info contains the compaction state. + RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName); + builder.setRegionInfo(ProtobufUtil.toRegionInfo(mobRegionInfo)); + if (request.hasCompactionState() && request.getCompactionState()) { + builder.setCompactionState(master.getMobCompactionState(tableName)); + } + } else { + // If unknown RegionInfo and not a MOB region, it is unknown. + throw new ServiceException(new UnknownRegionException(Bytes.toString(regionName))); + } + } + return builder.build(); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 379e254010b..5dcc0b07ea2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1776,6 +1776,9 @@ public class RSRpcServices implements HBaseRPCErrorHandler, } } + // Master implementation of this Admin Service differs given it is not + // able to supply detail only known to RegionServer. See note on + // MasterRpcServers#getRegionInfo. @Override @QosPriority(priority=HConstants.ADMIN_QOS) public GetRegionInfoResponse getRegionInfo(final RpcController controller, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java index 55b85bb1fc3..dbe86c27435 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java @@ -694,6 +694,34 @@ public class TestAdmin2 extends TestAdminBase { // Make sure that the store size is still the actual file system's store size. Assert.assertEquals(expectedStoreFilesSize, store.getSize()); } + + // Test querying using the encoded name only. When encoded name passed, + // and the target server is the Master, we return the full region name. + // Convenience. + testGetWithEncodedRegionName(conn, region.getRegionInfo()); + testGetWithRegionName(conn, region.getRegionInfo()); + // Try querying meta encoded name. + testGetWithEncodedRegionName(conn, RegionInfoBuilder.FIRST_META_REGIONINFO); + testGetWithRegionName(conn, RegionInfoBuilder.FIRST_META_REGIONINFO); + } + + /** + * Do get of RegionInfo from Master using encoded region name. + */ + private void testGetWithEncodedRegionName(ClusterConnection conn, RegionInfo inputRI) + throws IOException { + RegionInfo ri = ProtobufUtil.getRegionInfo(null, + conn.getAdmin(TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName()), + inputRI.getEncodedNameAsBytes()); + assertEquals(inputRI, ri); + } + + private void testGetWithRegionName(ClusterConnection conn, RegionInfo inputRI) + throws IOException { + RegionInfo ri = ProtobufUtil.getRegionInfo(null, + conn.getAdmin(TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName()), + inputRI.getRegionName()); + assertEquals(inputRI, ri); } @Test diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index ddc361ac9f9..89229e7feef 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -364,6 +364,7 @@ Shell.load_command_group( clear_block_cache stop_master stop_regionserver + regioninfo rit list_decommissioned_regionservers decommission_regionservers diff --git a/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb b/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb new file mode 100644 index 00000000000..c611705be0b --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Regioninfo < Command + def help + <<-EOF +Return RegionInfo. Takes Region name or an encoded Region name +(Of use when all you have is an encoded Region name). + +Examples: +Below we pass first encoded region name and then full region name. + + hbase(main):002:0> regioninfo '1588230740' + {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} + hbase(main):002:0> regioninfo 'hbase:meta,,1' + {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} + +EOF + end + + def command(regionname) + connection = org.apache.hadoop.hbase.client.ConnectionFactory.createConnection() + admin = connection.getAdmin() + sn = servername != nil ? ServerName.valueOf(servername): admin.getMaster() + puts org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRegionInfo(nil, + connection.getAdmin(sn), regionname.to_java_bytes) + end + end + end +end