diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java similarity index 59% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index f0124d3314d..9b376f52c98 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -30,15 +30,12 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Objects; import java.util.Set; -import java.util.SortedMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell.Type; +import org.apache.hadoop.hbase.ClientMetaTableAccessor.QueryType; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; @@ -56,7 +53,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.client.coprocessor.Batch; -import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.RowFilter; @@ -84,74 +80,43 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos. import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse; /** - *

* Read/write operations on hbase:meta region as well as assignment information stored * to hbase:meta. - *

- *

+ *

* Some of the methods of this class take ZooKeeperWatcher as a param. The only reason for this is * when this class is used on client-side (e.g. HBaseAdmin), we want to use short-lived connection * (opened before each operation, closed right after), while when used on HM or HRS (like in * AssignmentManager) we want permanent connection. - *

- *

+ *

* HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the same table * range (table, startKey, endKey). For every range, there will be at least one HRI defined which is * called default replica. - *

- *

- *

Meta layout

+ *

+ *

Meta layout

For each table there is single row named for the table with a 'table' column + * family. The column family currently has one column in it, the 'state' column: * *
- * For each table there is single row named for the table with a 'table' column family.
- * The column family currently has one column in it, the 'state' column:
- *
- * table:state             => contains table state
- *
- * Then for each table range ('Region'), there is a single row, formatted as:
- * <tableName>,<startKey>,<regionId>,<encodedRegionName>.
- * This row is the serialized regionName of the default region replica.
- * Columns are:
- * info:regioninfo         => contains serialized HRI for the default region replica
- * info:server             => contains hostname:port (in string form) for the server hosting
- *                            the default regionInfo replica
- * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting
- *                                 the regionInfo replica with replicaId
- * info:serverstartcode    => contains server start code (in binary long form) for the server
- *                            hosting the default regionInfo replica
- * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for
- *                                          the server hosting the regionInfo replica with
- *                                          replicaId
- * info:seqnumDuringOpen   => contains seqNum (in binary long form) for the region at the time
- *                             the server opened the region with default replicaId
- * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region
- *                                           at the time the server opened the region with
- *                                           replicaId
- * info:splitA             => contains a serialized HRI for the first daughter region if the
- *                             region is split
- * info:splitB             => contains a serialized HRI for the second daughter region if the
- *                             region is split
- * info:merge*             => contains a serialized HRI for a merge parent region. There will be two
- *                             or more of these columns in a row. A row that has these columns is
- *                             undergoing a merge and is the result of the merge. Columns listed
- *                             in marge* columns are the parents of this merged region. Example
- *                             columns: info:merge0001, info:merge0002. You make also see 'mergeA',
- *                             and 'mergeB'. This is old form replaced by the new format that allows
- *                             for more than two parents to be merged at a time.
- * TODO: Add rep_barrier for serial replication explaination. See SerialReplicationChecker.
+ * table:state => contains table state
  * 
- *

- *

+ * + * For the catalog family, see the comments of {@link CatalogFamilyFormat} for more details. + *

+ * TODO: Add rep_barrier for serial replication explanation. See SerialReplicationChecker. + *

* The actual layout of meta should be encapsulated inside MetaTableAccessor methods, and should not * leak out of it (through Result objects, etc) - *

+ * @see CatalogFamilyFormat + * @see ClientMetaTableAccessor */ @InterfaceAudience.Private -public class MetaTableAccessor { +public final class MetaTableAccessor { private static final Logger LOG = LoggerFactory.getLogger(MetaTableAccessor.class); private static final Logger METALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + private MetaTableAccessor() { + } + @VisibleForTesting public static final byte[] REPLICATION_PARENT_QUALIFIER = Bytes.toBytes("parent"); @@ -159,31 +124,6 @@ public class MetaTableAccessor { private static final byte SEPARATED_BYTE = 0x00; - @InterfaceAudience.Private - public enum QueryType { - ALL(HConstants.TABLE_FAMILY, HConstants.CATALOG_FAMILY), - REGION(HConstants.CATALOG_FAMILY), - TABLE(HConstants.TABLE_FAMILY), - REPLICATION(HConstants.REPLICATION_BARRIER_FAMILY); - - private final byte[][] families; - - QueryType(byte[]... families) { - this.families = families; - } - - byte[][] getFamilies() { - return this.families; - } - } - - /** The delimiter for meta columns for replicaIds > 0 */ - static final char META_REPLICA_ID_DELIMITER = '_'; - - /** A regex for parsing server columns from meta. See above javadoc for meta layout */ - private static final Pattern SERVER_COLUMN_PATTERN - = Pattern.compile("^server(_[0-9a-fA-F]{4})?$"); - //////////////////////// // Reading operations // //////////////////////// @@ -193,8 +133,8 @@ public class MetaTableAccessor { * @param connection connection we're using * @param visitor Visitor invoked against each row in regions family. */ - public static void fullScanRegions(Connection connection, final Visitor visitor) - throws IOException { + public static void fullScanRegions(Connection connection, + final ClientMetaTableAccessor.Visitor visitor) throws IOException { scanMeta(connection, null, null, QueryType.REGION, visitor); } @@ -211,8 +151,8 @@ public class MetaTableAccessor { * @param connection connection we're using * @param visitor Visitor invoked against each row in tables family. */ - public static void fullScanTables(Connection connection, final Visitor visitor) - throws IOException { + public static void fullScanTables(Connection connection, + final ClientMetaTableAccessor.Visitor visitor) throws IOException { scanMeta(connection, null, null, QueryType.TABLE, visitor); } @@ -223,7 +163,7 @@ public class MetaTableAccessor { * @return List of {@link Result} */ private static List fullScan(Connection connection, QueryType type) throws IOException { - CollectAllVisitor v = new CollectAllVisitor(); + ClientMetaTableAccessor.CollectAllVisitor v = new ClientMetaTableAccessor.CollectAllVisitor(); scanMeta(connection, null, null, type, v); return v.getResults(); } @@ -243,19 +183,6 @@ public class MetaTableAccessor { return connection.getTable(TableName.META_TABLE_NAME); } - /** - * @param t Table to use (will be closed when done). - * @param g Get to run - */ - private static Result get(final Table t, final Get g) throws IOException { - if (t == null) return null; - try { - return t.get(g); - } finally { - t.close(); - } - } - /** * Gets the region info and assignment for the specified region. * @param connection connection we're using @@ -264,12 +191,10 @@ public class MetaTableAccessor { * @deprecated use {@link #getRegionLocation(Connection, byte[])} instead */ @Deprecated - public static Pair getRegion(Connection connection, byte [] regionName) + public static Pair getRegion(Connection connection, byte[] regionName) throws IOException { HRegionLocation location = getRegionLocation(connection, regionName); - return location == null - ? null - : new Pair<>(location.getRegion(), location.getServerName()); + return location == null ? null : new Pair<>(location.getRegion(), location.getServerName()); } /** @@ -279,21 +204,24 @@ public class MetaTableAccessor { * @return HRegionLocation for the given region */ public static HRegionLocation getRegionLocation(Connection connection, byte[] regionName) - throws IOException { + throws IOException { byte[] row = regionName; RegionInfo parsedInfo = null; try { - parsedInfo = parseRegionInfoFromRegionName(regionName); - row = getMetaKeyForRegion(parsedInfo); + parsedInfo = CatalogFamilyFormat.parseRegionInfoFromRegionName(regionName); + row = CatalogFamilyFormat.getMetaKeyForRegion(parsedInfo); } catch (Exception parseEx) { // Ignore. This is used with tableName passed as regionName. } Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); - Result r = get(getMetaHTable(connection), get); - RegionLocations locations = getRegionLocations(r); - return locations == null ? null - : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); + Result r; + try (Table t = getMetaHTable(connection)) { + r = t.get(get); + } + RegionLocations locations = CatalogFamilyFormat.getRegionLocations(r); + return locations == null ? null : + locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); } /** @@ -303,42 +231,21 @@ public class MetaTableAccessor { * @return HRegionLocation for the given region */ public static HRegionLocation getRegionLocation(Connection connection, RegionInfo regionInfo) - throws IOException { - return getRegionLocation(getCatalogFamilyRow(connection, regionInfo), - regionInfo, regionInfo.getReplicaId()); + throws IOException { + return CatalogFamilyFormat.getRegionLocation(getCatalogFamilyRow(connection, regionInfo), + regionInfo, regionInfo.getReplicaId()); } /** * @return Return the {@link HConstants#CATALOG_FAMILY} row from hbase:meta. */ public static Result getCatalogFamilyRow(Connection connection, RegionInfo ri) - throws IOException { - Get get = new Get(getMetaKeyForRegion(ri)); + throws IOException { + Get get = new Get(CatalogFamilyFormat.getMetaKeyForRegion(ri)); get.addFamily(HConstants.CATALOG_FAMILY); - return get(getMetaHTable(connection), get); - } - - /** Returns the row key to use for this regionInfo */ - public static byte[] getMetaKeyForRegion(RegionInfo regionInfo) { - return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName(); - } - - /** Returns an HRI parsed from this regionName. Not all the fields of the HRI - * is stored in the name, so the returned object should only be used for the fields - * in the regionName. - */ - // This should be moved to RegionInfo? TODO. - public static RegionInfo parseRegionInfoFromRegionName(byte[] regionName) throws IOException { - byte[][] fields = RegionInfo.parseRegionName(regionName); - long regionId = Long.parseLong(Bytes.toString(fields[2])); - int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0; - return RegionInfoBuilder.newBuilder(TableName.valueOf(fields[0])) - .setStartKey(fields[1]) - .setEndKey(fields[2]) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(replicaId) - .build(); + try (Table t = getMetaHTable(connection)) { + return t.get(get); + } } /** @@ -347,26 +254,27 @@ public class MetaTableAccessor { * @param regionName region we're looking for * @return result of the specified region */ - public static Result getRegionResult(Connection connection, - byte[] regionName) throws IOException { + public static Result getRegionResult(Connection connection, byte[] regionName) + throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); - return get(getMetaHTable(connection), get); + try (Table t = getMetaHTable(connection)) { + return t.get(get); + } } /** - * Scans META table for a row whose key contains the specified regionEncodedName, - * returning a single related Result instance if any row is found, null otherwise. - * + * Scans META table for a row whose key contains the specified regionEncodedName, returning + * a single related Result instance if any row is found, null otherwise. * @param connection the connection to query META table. * @param regionEncodedName the region encoded name to look for at META. * @return Result instance with the row related info in META, null otherwise. * @throws IOException if any errors occur while querying META. */ - public static Result scanByRegionEncodedName(Connection connection, - String regionEncodedName) throws IOException { - RowFilter rowFilter = new RowFilter(CompareOperator.EQUAL, - new SubstringComparator(regionEncodedName)); + public static Result scanByRegionEncodedName(Connection connection, String regionEncodedName) + throws IOException { + RowFilter rowFilter = + new RowFilter(CompareOperator.EQUAL, new SubstringComparator(regionEncodedName)); Scan scan = getMetaScan(connection, 1); scan.setFilter(rowFilter); ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan); @@ -374,26 +282,26 @@ public class MetaTableAccessor { } /** - * @return Return all regioninfos listed in the 'info:merge*' columns of - * the regionName row. + * @return Return all regioninfos listed in the 'info:merge*' columns of the + * regionName row. */ @Nullable public static List getMergeRegions(Connection connection, byte[] regionName) - throws IOException { + throws IOException { return getMergeRegions(getRegionResult(connection, regionName).rawCells()); } /** - * @return Deserialized values of pairs taken from column values that match - * the regex 'info:merge.*' in array of cells. + * @return Deserialized values of <qualifier,regioninfo> pairs taken from column values that + * match the regex 'info:merge.*' in array of cells. */ @Nullable - public static Map getMergeRegionsWithName(Cell [] cells) { + public static Map getMergeRegionsWithName(Cell[] cells) { if (cells == null) { return null; } Map regionsToMerge = null; - for (Cell cell: cells) { + for (Cell cell : cells) { if (!isMergeQualifierPrefix(cell)) { continue; } @@ -411,21 +319,21 @@ public class MetaTableAccessor { } /** - * @return Deserialized regioninfo values taken from column values that match - * the regex 'info:merge.*' in array of cells. + * @return Deserialized regioninfo values taken from column values that match the regex + * 'info:merge.*' in array of cells. */ @Nullable - public static List getMergeRegions(Cell [] cells) { + public static List getMergeRegions(Cell[] cells) { Map mergeRegionsWithName = getMergeRegionsWithName(cells); return (mergeRegionsWithName == null) ? null : new ArrayList<>(mergeRegionsWithName.values()); } /** - * @return True if any merge regions present in cells; i.e. - * the column in cell matches the regex 'info:merge.*'. + * @return True if any merge regions present in cells; i.e. the column in + * cell matches the regex 'info:merge.*'. */ - public static boolean hasMergeRegions(Cell [] cells) { - for (Cell cell: cells) { + public static boolean hasMergeRegions(Cell[] cells) { + for (Cell cell : cells) { if (!isMergeQualifierPrefix(cell)) { continue; } @@ -444,71 +352,67 @@ public class MetaTableAccessor { } /** - * Checks if the specified table exists. Looks at the hbase:meta table hosted on - * the specified server. + * Checks if the specified table exists. Looks at the hbase:meta table hosted on the specified + * server. * @param connection connection we're using * @param tableName table to check * @return true if the table exists in meta, false if not */ - public static boolean tableExists(Connection connection, - final TableName tableName) - throws IOException { + public static boolean tableExists(Connection connection, final TableName tableName) + throws IOException { // Catalog tables always exist. return tableName.equals(TableName.META_TABLE_NAME) || - getTableState(connection, tableName) != null; + getTableState(connection, tableName) != null; } /** * Lists all of the regions currently in META. - * * @param connection to connect with * @param excludeOfflinedSplitParents False if we are to include offlined/splitparents regions, - * true and we'll leave out offlined regions from returned list + * true and we'll leave out offlined regions from returned list * @return List of all user-space regions. */ @VisibleForTesting public static List getAllRegions(Connection connection, - boolean excludeOfflinedSplitParents) - throws IOException { + boolean excludeOfflinedSplitParents) throws IOException { List> result; - result = getTableRegionsAndLocations(connection, null, - excludeOfflinedSplitParents); + result = getTableRegionsAndLocations(connection, null, excludeOfflinedSplitParents); return getListOfRegionInfos(result); } /** - * Gets all of the regions of the specified table. Do not use this method - * to get meta table regions, use methods in MetaTableLocator instead. + * Gets all of the regions of the specified table. Do not use this method to get meta table + * regions, use methods in MetaTableLocator instead. * @param connection connection we're using * @param tableName table we're looking for * @return Ordered list of {@link RegionInfo}. */ public static List getTableRegions(Connection connection, TableName tableName) - throws IOException { + throws IOException { return getTableRegions(connection, tableName, false); } /** - * Gets all of the regions of the specified table. Do not use this method - * to get meta table regions, use methods in MetaTableLocator instead. + * Gets all of the regions of the specified table. Do not use this method to get meta table + * regions, use methods in MetaTableLocator instead. * @param connection connection we're using * @param tableName table we're looking for - * @param excludeOfflinedSplitParents If true, do not include offlined split - * parents in the return. + * @param excludeOfflinedSplitParents If true, do not include offlined split parents in the + * return. * @return Ordered list of {@link RegionInfo}. */ public static List getTableRegions(Connection connection, TableName tableName, - final boolean excludeOfflinedSplitParents) throws IOException { + final boolean excludeOfflinedSplitParents) throws IOException { List> result = getTableRegionsAndLocations(connection, tableName, excludeOfflinedSplitParents); return getListOfRegionInfos(result); } - private static List getListOfRegionInfos( - final List> pairs) { + private static List + getListOfRegionInfos(final List> pairs) { if (pairs == null || pairs.isEmpty()) { return Collections.emptyList(); } @@ -520,70 +424,19 @@ public class MetaTableAccessor { } /** - * @param tableName table we're working with - * @return start row for scanning META according to query type - */ - public static byte[] getTableStartRowForMeta(TableName tableName, QueryType type) { - if (tableName == null) { - return null; - } - switch (type) { - case REGION: - byte[] startRow = new byte[tableName.getName().length + 2]; - System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length); - startRow[startRow.length - 2] = HConstants.DELIMITER; - startRow[startRow.length - 1] = HConstants.DELIMITER; - return startRow; - case ALL: - case TABLE: - default: - return tableName.getName(); - } - } - - /** - * @param tableName table we're working with - * @return stop row for scanning META according to query type - */ - public static byte[] getTableStopRowForMeta(TableName tableName, QueryType type) { - if (tableName == null) { - return null; - } - final byte[] stopRow; - switch (type) { - case REGION: - stopRow = new byte[tableName.getName().length + 3]; - System.arraycopy(tableName.getName(), 0, stopRow, 0, tableName.getName().length); - stopRow[stopRow.length - 3] = ' '; - stopRow[stopRow.length - 2] = HConstants.DELIMITER; - stopRow[stopRow.length - 1] = HConstants.DELIMITER; - break; - case ALL: - case TABLE: - default: - stopRow = new byte[tableName.getName().length + 1]; - System.arraycopy(tableName.getName(), 0, stopRow, 0, tableName.getName().length); - stopRow[stopRow.length - 1] = ' '; - break; - } - return stopRow; - } - - /** - * This method creates a Scan object that will only scan catalog rows that - * belong to the specified table. It doesn't specify any columns. - * This is a better alternative to just using a start row and scan until - * it hits a new table since that requires parsing the HRI to get the table - * name. + * This method creates a Scan object that will only scan catalog rows that belong to the specified + * table. It doesn't specify any columns. This is a better alternative to just using a start row + * and scan until it hits a new table since that requires parsing the HRI to get the table name. * @param tableName bytes of table's name * @return configured Scan object + * @deprecated This is internal so please remove it when we get a chance. */ @Deprecated public static Scan getScanForTableName(Connection connection, TableName tableName) { // Start key is just the table name with delimiters - byte[] startKey = getTableStartRowForMeta(tableName, QueryType.REGION); + byte[] startKey = ClientMetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REGION); // Stop key appends the smallest possible char to the table name - byte[] stopKey = getTableStopRowForMeta(tableName, QueryType.REGION); + byte[] stopKey = ClientMetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REGION); Scan scan = getMetaScan(connection, -1); scan.withStartRow(startKey); @@ -593,11 +446,10 @@ public class MetaTableAccessor { private static Scan getMetaScan(Connection connection, int rowUpperLimit) { Scan scan = new Scan(); - int scannerCaching = connection.getConfiguration() - .getInt(HConstants.HBASE_META_SCANNER_CACHING, - HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); + int scannerCaching = connection.getConfiguration().getInt(HConstants.HBASE_META_SCANNER_CACHING, + HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); if (connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, - HConstants.DEFAULT_USE_META_REPLICAS)) { + HConstants.DEFAULT_USE_META_REPLICAS)) { scan.setConsistency(Consistency.TIMELINE); } if (rowUpperLimit > 0) { @@ -607,6 +459,7 @@ public class MetaTableAccessor { scan.setCaching(scannerCaching); return scan; } + /** * Do not use this method to get meta table regions, use methods in MetaTableLocator instead. * @param connection connection we're using @@ -614,8 +467,7 @@ public class MetaTableAccessor { * @return Return list of regioninfos and server. */ public static List> - getTableRegionsAndLocations(Connection connection, TableName tableName) - throws IOException { + getTableRegionsAndLocations(Connection connection, TableName tableName) throws IOException { return getTableRegionsAndLocations(connection, tableName, true); } @@ -628,61 +480,33 @@ public class MetaTableAccessor { */ // What happens here when 1M regions in hbase:meta? This won't scale? public static List> getTableRegionsAndLocations( - Connection connection, @Nullable final TableName tableName, - final boolean excludeOfflinedSplitParents) throws IOException { + Connection connection, @Nullable final TableName tableName, + final boolean excludeOfflinedSplitParents) throws IOException { if (tableName != null && tableName.equals(TableName.META_TABLE_NAME)) { - throw new IOException("This method can't be used to locate meta regions;" - + " use MetaTableLocator instead"); + throw new IOException( + "This method can't be used to locate meta regions;" + " use MetaTableLocator instead"); } // Make a version of CollectingVisitor that collects RegionInfo and ServerAddress - CollectingVisitor> visitor = - new CollectingVisitor>() { - private RegionLocations current = null; - - @Override - public boolean visit(Result r) throws IOException { - current = getRegionLocations(r); - if (current == null || current.getRegionLocation().getRegion() == null) { - LOG.warn("No serialized RegionInfo in " + r); - return true; - } - RegionInfo hri = current.getRegionLocation().getRegion(); - if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; - // Else call super and add this Result to the collection. - return super.visit(r); - } - - @Override - void add(Result r) { - if (current == null) { - return; - } - for (HRegionLocation loc : current.getRegionLocations()) { - if (loc != null) { - this.results.add(new Pair<>(loc.getRegion(), loc.getServerName())); - } - } - } - }; + ClientMetaTableAccessor.CollectRegionLocationsVisitor visitor = + new ClientMetaTableAccessor.CollectRegionLocationsVisitor(excludeOfflinedSplitParents); scanMeta(connection, - getTableStartRowForMeta(tableName, QueryType.REGION), - getTableStopRowForMeta(tableName, QueryType.REGION), - QueryType.REGION, visitor); + ClientMetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REGION), + ClientMetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REGION), QueryType.REGION, + visitor); return visitor.getResults(); } - public static void fullScanMetaAndPrint(Connection connection) - throws IOException { - Visitor v = r -> { - if (r == null || r.isEmpty()) { + public static void fullScanMetaAndPrint(Connection connection) throws IOException { + ClientMetaTableAccessor.Visitor v = r -> { + if (r == null || r.isEmpty()) { return true; } LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r); - TableState state = getTableState(r); + TableState state = CatalogFamilyFormat.getTableState(r); if (state != null) { LOG.info("fullScanMetaAndPrint.Table State={}" + state); } else { - RegionLocations locations = getRegionLocations(r); + RegionLocations locations = CatalogFamilyFormat.getRegionLocations(r); if (locations == null) { return true; } @@ -697,42 +521,43 @@ public class MetaTableAccessor { scanMeta(connection, null, null, QueryType.ALL, v); } - public static void scanMetaForTableRegions(Connection connection, Visitor visitor, - TableName tableName) throws IOException { + public static void scanMetaForTableRegions(Connection connection, + ClientMetaTableAccessor.Visitor visitor, TableName tableName) throws IOException { scanMeta(connection, tableName, QueryType.REGION, Integer.MAX_VALUE, visitor); } private static void scanMeta(Connection connection, TableName table, QueryType type, int maxRows, - final Visitor visitor) throws IOException { - scanMeta(connection, getTableStartRowForMeta(table, type), getTableStopRowForMeta(table, type), - type, maxRows, visitor); + final ClientMetaTableAccessor.Visitor visitor) throws IOException { + scanMeta(connection, ClientMetaTableAccessor.getTableStartRowForMeta(table, type), + ClientMetaTableAccessor.getTableStopRowForMeta(table, type), type, maxRows, visitor); } private static void scanMeta(Connection connection, @Nullable final byte[] startRow, - @Nullable final byte[] stopRow, QueryType type, final Visitor visitor) throws IOException { + @Nullable final byte[] stopRow, QueryType type, final ClientMetaTableAccessor.Visitor visitor) + throws IOException { scanMeta(connection, startRow, stopRow, type, Integer.MAX_VALUE, visitor); } /** * Performs a scan of META table for given table starting from given row. * @param connection connection we're using - * @param visitor visitor to call - * @param tableName table withing we scan - * @param row start scan from this row - * @param rowLimit max number of rows to return + * @param visitor visitor to call + * @param tableName table withing we scan + * @param row start scan from this row + * @param rowLimit max number of rows to return */ - public static void scanMeta(Connection connection, final Visitor visitor, - final TableName tableName, final byte[] row, final int rowLimit) throws IOException { + public static void scanMeta(Connection connection, final ClientMetaTableAccessor.Visitor visitor, + final TableName tableName, final byte[] row, final int rowLimit) throws IOException { byte[] startRow = null; byte[] stopRow = null; if (tableName != null) { - startRow = getTableStartRowForMeta(tableName, QueryType.REGION); + startRow = ClientMetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REGION); if (row != null) { RegionInfo closestRi = getClosestRegionInfo(connection, tableName, row); startRow = RegionInfo.createRegionName(tableName, closestRi.getStartKey(), HConstants.ZEROES, false); } - stopRow = getTableStopRowForMeta(tableName, QueryType.REGION); + stopRow = ClientMetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REGION); } scanMeta(connection, startRow, stopRow, QueryType.REGION, rowLimit, visitor); } @@ -740,23 +565,21 @@ public class MetaTableAccessor { /** * Performs a scan of META table. * @param connection connection we're using - * @param startRow Where to start the scan. Pass null if want to begin scan - * at first row. - * @param stopRow Where to stop the scan. Pass null if want to scan all rows - * from the start one + * @param startRow Where to start the scan. Pass null if want to begin scan at first row. + * @param stopRow Where to stop the scan. Pass null if want to scan all rows from the start one * @param type scanned part of meta * @param maxRows maximum rows to return * @param visitor Visitor invoked against each row. */ static void scanMeta(Connection connection, @Nullable final byte[] startRow, - @Nullable final byte[] stopRow, QueryType type, int maxRows, final Visitor visitor) - throws IOException { + @Nullable final byte[] stopRow, QueryType type, int maxRows, + final ClientMetaTableAccessor.Visitor visitor) throws IOException { scanMeta(connection, startRow, stopRow, type, null, maxRows, visitor); } private static void scanMeta(Connection connection, @Nullable final byte[] startRow, - @Nullable final byte[] stopRow, QueryType type, @Nullable Filter filter, int maxRows, - final Visitor visitor) throws IOException { + @Nullable final byte[] stopRow, QueryType type, @Nullable Filter filter, int maxRows, + final ClientMetaTableAccessor.Visitor visitor) throws IOException { int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; Scan scan = getMetaScan(connection, rowUpperLimit); @@ -784,10 +607,16 @@ public class MetaTableAccessor { try (ResultScanner scanner = metaTable.getScanner(scan)) { Result data; while ((data = scanner.next()) != null) { - if (data.isEmpty()) continue; + if (data.isEmpty()) { + continue; + } // Break if visit returns false. - if (!visitor.visit(data)) break; - if (++currentRow >= rowUpperLimit) break; + if (!visitor.visit(data)) { + break; + } + if (++currentRow >= rowUpperLimit) { + break; + } } } } @@ -806,7 +635,7 @@ public class MetaTableAccessor { */ @NonNull private static RegionInfo getClosestRegionInfo(Connection connection, - @NonNull final TableName tableName, @NonNull final byte[] row) throws IOException { + @NonNull final TableName tableName, @NonNull final byte[] row) throws IOException { byte[] searchRow = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false); Scan scan = getMetaScan(connection, 1); scan.setReversed(true); @@ -814,182 +643,32 @@ public class MetaTableAccessor { try (ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan)) { Result result = resultScanner.next(); if (result == null) { - throw new TableNotFoundException("Cannot find row in META " + - " for table: " + tableName + ", row=" + Bytes.toStringBinary(row)); + throw new TableNotFoundException("Cannot find row in META " + " for table: " + tableName + + ", row=" + Bytes.toStringBinary(row)); } - RegionInfo regionInfo = getRegionInfo(result); + RegionInfo regionInfo = CatalogFamilyFormat.getRegionInfo(result); if (regionInfo == null) { - throw new IOException("RegionInfo was null or empty in Meta for " + - tableName + ", row=" + Bytes.toStringBinary(row)); + throw new IOException("RegionInfo was null or empty in Meta for " + tableName + ", row=" + + Bytes.toStringBinary(row)); } return regionInfo; } } - /** - * Returns the column family used for meta columns. - * @return HConstants.CATALOG_FAMILY. - */ - public static byte[] getCatalogFamily() { - return HConstants.CATALOG_FAMILY; - } - - /** - * Returns the column family used for table columns. - * @return HConstants.TABLE_FAMILY. - */ - private static byte[] getTableFamily() { - return HConstants.TABLE_FAMILY; - } - - /** - * Returns the column qualifier for serialized region info - * @return HConstants.REGIONINFO_QUALIFIER - */ - public static byte[] getRegionInfoColumn() { - return HConstants.REGIONINFO_QUALIFIER; - } - - /** - * Returns the column qualifier for serialized table state - * @return HConstants.TABLE_STATE_QUALIFIER - */ - private static byte[] getTableStateColumn() { - return HConstants.TABLE_STATE_QUALIFIER; - } - - /** - * Returns the column qualifier for serialized region state - * @return HConstants.STATE_QUALIFIER - */ - private static byte[] getRegionStateColumn() { - return HConstants.STATE_QUALIFIER; - } - - /** - * Returns the column qualifier for serialized region state - * @param replicaId the replicaId of the region - * @return a byte[] for state qualifier - */ - @VisibleForTesting - static byte[] getRegionStateColumn(int replicaId) { - return replicaId == 0 ? HConstants.STATE_QUALIFIER - : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for serialized region state - * @param replicaId the replicaId of the region - * @return a byte[] for sn column qualifier - */ - public static byte[] getServerNameColumn(int replicaId) { - return replicaId == 0 ? HConstants.SERVERNAME_QUALIFIER - : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for server column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for server column qualifier - */ - @VisibleForTesting - public static byte[] getServerColumn(int replicaId) { - return replicaId == 0 - ? HConstants.SERVER_QUALIFIER - : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for server start code column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for server start code column qualifier - */ - @VisibleForTesting - public static byte[] getStartCodeColumn(int replicaId) { - return replicaId == 0 - ? HConstants.STARTCODE_QUALIFIER - : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for seqNum column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for seqNum column qualifier - */ - @VisibleForTesting - public static byte[] getSeqNumColumn(int replicaId) { - return replicaId == 0 - ? HConstants.SEQNUM_QUALIFIER - : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Parses the replicaId from the server column qualifier. See top of the class javadoc - * for the actual meta layout - * @param serverColumn the column qualifier - * @return an int for the replicaId - */ - @VisibleForTesting - static int parseReplicaIdFromServerColumn(byte[] serverColumn) { - String serverStr = Bytes.toString(serverColumn); - - Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr); - if (matcher.matches() && matcher.groupCount() > 0) { - String group = matcher.group(1); - if (group != null && group.length() > 0) { - return Integer.parseInt(group.substring(1), 16); - } else { - return 0; - } - } - return -1; - } - - /** - * Returns a {@link ServerName} from catalog table {@link Result}. - * @param r Result to pull from - * @return A ServerName instance or null if necessary fields not found or empty. - */ - @Nullable - @InterfaceAudience.Private // for use by HMaster#getTableRegionRow which is used for testing only - public static ServerName getServerName(final Result r, final int replicaId) { - byte[] serverColumn = getServerColumn(replicaId); - Cell cell = r.getColumnLatestCell(getCatalogFamily(), serverColumn); - if (cell == null || cell.getValueLength() == 0) return null; - String hostAndPort = Bytes.toString( - cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); - byte[] startcodeColumn = getStartCodeColumn(replicaId); - cell = r.getColumnLatestCell(getCatalogFamily(), startcodeColumn); - if (cell == null || cell.getValueLength() == 0) return null; - try { - return ServerName.valueOf(hostAndPort, - Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); - } catch (IllegalArgumentException e) { - LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e); - return null; - } - } - /** * Returns the {@link ServerName} from catalog table {@link Result} where the region is - * transitioning on. It should be the same as {@link MetaTableAccessor#getServerName(Result,int)} - * if the server is at OPEN state. - * + * transitioning on. It should be the same as + * {@link CatalogFamilyFormat#getServerName(Result,int)} if the server is at OPEN state. * @param r Result to pull the transitioning server name from - * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)} - * if necessary fields not found or empty. + * @return A ServerName instance or {@link CatalogFamilyFormat#getServerName(Result,int)} if + * necessary fields not found or empty. */ @Nullable public static ServerName getTargetServerName(final Result r, final int replicaId) { final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, - getServerNameColumn(replicaId)); + CatalogFamilyFormat.getServerNameColumn(replicaId)); if (cell == null || cell.getValueLength() == 0) { - RegionLocations locations = MetaTableAccessor.getRegionLocations(r); + RegionLocations locations = CatalogFamilyFormat.getRegionLocations(r); if (locations != null) { HRegionLocation location = locations.getRegionLocation(replicaId); if (location != null) { @@ -998,137 +677,36 @@ public class MetaTableAccessor { } return null; } - return ServerName.parseServerName(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), - cell.getValueLength())); + return ServerName.parseServerName( + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } /** - * The latest seqnum that the server writing to meta observed when opening the region. - * E.g. the seqNum when the result of {@link #getServerName(Result, int)} was written. - * @param r Result to pull the seqNum from - * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written. - */ - private static long getSeqNumDuringOpen(final Result r, final int replicaId) { - Cell cell = r.getColumnLatestCell(getCatalogFamily(), getSeqNumColumn(replicaId)); - if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM; - return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); - } - - /** - * Returns the daughter regions by reading the corresponding columns of the catalog table - * Result. + * Returns the daughter regions by reading the corresponding columns of the catalog table Result. * @param data a Result object from the catalog table scan * @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent */ public static PairOfSameType getDaughterRegions(Result data) { - RegionInfo splitA = getRegionInfo(data, HConstants.SPLITA_QUALIFIER); - RegionInfo splitB = getRegionInfo(data, HConstants.SPLITB_QUALIFIER); + RegionInfo splitA = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITA_QUALIFIER); + RegionInfo splitB = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITB_QUALIFIER); return new PairOfSameType<>(splitA, splitB); } - /** - * Returns an HRegionLocationList extracted from the result. - * @return an HRegionLocationList containing all locations for the region range or null if - * we can't deserialize the result. - */ - @Nullable - public static RegionLocations getRegionLocations(final Result r) { - if (r == null) return null; - RegionInfo regionInfo = getRegionInfo(r, getRegionInfoColumn()); - if (regionInfo == null) return null; - - List locations = new ArrayList<>(1); - NavigableMap> familyMap = r.getNoVersionMap(); - - locations.add(getRegionLocation(r, regionInfo, 0)); - - NavigableMap infoMap = familyMap.get(getCatalogFamily()); - if (infoMap == null) return new RegionLocations(locations); - - // iterate until all serverName columns are seen - int replicaId = 0; - byte[] serverColumn = getServerColumn(replicaId); - SortedMap serverMap; - serverMap = infoMap.tailMap(serverColumn, false); - - if (serverMap.isEmpty()) return new RegionLocations(locations); - - for (Map.Entry entry : serverMap.entrySet()) { - replicaId = parseReplicaIdFromServerColumn(entry.getKey()); - if (replicaId < 0) { - break; - } - HRegionLocation location = getRegionLocation(r, regionInfo, replicaId); - // In case the region replica is newly created, it's location might be null. We usually do not - // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. - if (location.getServerName() == null) { - locations.add(null); - } else { - locations.add(location); - } - } - - return new RegionLocations(locations); - } - - /** - * Returns the HRegionLocation parsed from the given meta row Result - * for the given regionInfo and replicaId. The regionInfo can be the default region info - * for the replica. - * @param r the meta row result - * @param regionInfo RegionInfo for default replica - * @param replicaId the replicaId for the HRegionLocation - * @return HRegionLocation parsed from the given meta row Result for the given replicaId - */ - private static HRegionLocation getRegionLocation(final Result r, final RegionInfo regionInfo, - final int replicaId) { - ServerName serverName = getServerName(r, replicaId); - long seqNum = getSeqNumDuringOpen(r, replicaId); - RegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId); - return new HRegionLocation(replicaInfo, serverName, seqNum); - } - - /** - * Returns RegionInfo object from the column - * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog - * table Result. - * @param data a Result object from the catalog table scan - * @return RegionInfo or null - */ - public static RegionInfo getRegionInfo(Result data) { - return getRegionInfo(data, HConstants.REGIONINFO_QUALIFIER); - } - - /** - * Returns the RegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and - * qualifier of the catalog table result. - * @param r a Result object from the catalog table scan - * @param qualifier Column family qualifier - * @return An RegionInfo instance or null. - */ - @Nullable - public static RegionInfo getRegionInfo(final Result r, byte [] qualifier) { - Cell cell = r.getColumnLatestCell(getCatalogFamily(), qualifier); - if (cell == null) return null; - return RegionInfo.parseFromOrNull(cell.getValueArray(), - cell.getValueOffset(), cell.getValueLength()); - } - /** * Fetch table state for given table from META table * @param conn connection to use * @param tableName table to fetch state for */ @Nullable - public static TableState getTableState(Connection conn, TableName tableName) - throws IOException { + public static TableState getTableState(Connection conn, TableName tableName) throws IOException { if (tableName.equals(TableName.META_TABLE_NAME)) { return new TableState(tableName, TableState.State.ENABLED); } Table metaHTable = getMetaHTable(conn); - Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getTableStateColumn()); + Get get = new Get(tableName.getName()).addColumn(HConstants.TABLE_FAMILY, + HConstants.TABLE_STATE_QUALIFIER); Result result = metaHTable.get(get); - return getTableState(result); + return CatalogFamilyFormat.getTableState(result); } /** @@ -1136,11 +714,10 @@ public class MetaTableAccessor { * @param conn connection to use * @return map {tableName -> state} */ - public static Map getTableStates(Connection conn) - throws IOException { + public static Map getTableStates(Connection conn) throws IOException { final Map states = new LinkedHashMap<>(); - Visitor collector = r -> { - TableState state = getTableState(r); + ClientMetaTableAccessor.Visitor collector = r -> { + TableState state = CatalogFamilyFormat.getTableState(r); if (state != null) { states.put(state.getTableName(), state); } @@ -1151,115 +728,15 @@ public class MetaTableAccessor { } /** - * Updates state in META - * Do not use. For internal use only. + * Updates state in META Do not use. For internal use only. * @param conn connection to use * @param tableName table to look for */ - public static void updateTableState(Connection conn, TableName tableName, - TableState.State actual) throws IOException { + public static void updateTableState(Connection conn, TableName tableName, TableState.State actual) + throws IOException { updateTableState(conn, new TableState(tableName, actual)); } - /** - * Decode table state from META Result. - * Should contain cell from HConstants.TABLE_FAMILY - * @return null if not found - */ - @Nullable - public static TableState getTableState(Result r) throws IOException { - Cell cell = r.getColumnLatestCell(getTableFamily(), getTableStateColumn()); - if (cell == null) { - return null; - } - try { - return TableState.parseFrom(TableName.valueOf(r.getRow()), - Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), - cell.getValueOffset() + cell.getValueLength())); - } catch (DeserializationException e) { - throw new IOException(e); - } - } - - /** - * Implementations 'visit' a catalog table row. - */ - public interface Visitor { - /** - * Visit the catalog table row. - * @param r A row from catalog table - * @return True if we are to proceed scanning the table, else false if - * we are to stop now. - */ - boolean visit(final Result r) throws IOException; - } - - /** - * Implementations 'visit' a catalog table row but with close() at the end. - */ - public interface CloseableVisitor extends Visitor, Closeable { - } - - /** - * A {@link Visitor} that collects content out of passed {@link Result}. - */ - static abstract class CollectingVisitor implements Visitor { - final List results = new ArrayList<>(); - @Override - public boolean visit(Result r) throws IOException { - if (r != null && !r.isEmpty()) { - add(r); - } - return true; - } - - abstract void add(Result r); - - /** - * @return Collected results; wait till visits complete to collect all - * possible results - */ - List getResults() { - return this.results; - } - } - - /** - * Collects all returned. - */ - static class CollectAllVisitor extends CollectingVisitor { - @Override - void add(Result r) { - this.results.add(r); - } - } - - /** - * A Visitor that skips offline regions and split parents - */ - public static abstract class DefaultVisitorBase implements Visitor { - - DefaultVisitorBase() { - super(); - } - - public abstract boolean visitInternal(Result rowResult) throws IOException; - - @Override - public boolean visit(Result rowResult) throws IOException { - RegionInfo info = getRegionInfo(rowResult); - if (info == null) { - return true; - } - - //skip over offline and split regions - if (!(info.isOffline() || info.isSplit())) { - return visitInternal(rowResult); - } - return true; - } - } - /** * Count regions in hbase:meta for passed table. * @param c Configuration object @@ -1267,7 +744,7 @@ public class MetaTableAccessor { * @return Count or regions in table tableName */ public static int getRegionCount(final Configuration c, final TableName tableName) - throws IOException { + throws IOException { try (Connection connection = ConnectionFactory.createConnection(c)) { return getRegionCount(connection, tableName); } @@ -1280,7 +757,7 @@ public class MetaTableAccessor { * @return Count or regions in table tableName */ public static int getRegionCount(final Connection connection, final TableName tableName) - throws IOException { + throws IOException { try (RegionLocator locator = connection.getRegionLocator(tableName)) { List locations = locator.getAllRegionLocations(); return locations == null ? 0 : locations.size(); @@ -1305,7 +782,7 @@ public class MetaTableAccessor { throw new IllegalArgumentException("Can't make a delete for null region"); } Delete delete = new Delete(regionInfo.getRegionName()); - delete.addFamily(getCatalogFamily(), ts); + delete.addFamily(HConstants.CATALOG_FAMILY, ts); return delete; } @@ -1313,26 +790,18 @@ public class MetaTableAccessor { * Adds split daughters to the Put */ private static Put addDaughtersToPut(Put put, RegionInfo splitA, RegionInfo splitB) - throws IOException { + throws IOException { if (splitA != null) { - put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(put.getRow()) - .setFamily(HConstants.CATALOG_FAMILY) - .setQualifier(HConstants.SPLITA_QUALIFIER) - .setTimestamp(put.getTimestamp()) - .setType(Type.Put) - .setValue(RegionInfo.toByteArray(splitA)) - .build()); + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.SPLITA_QUALIFIER) + .setTimestamp(put.getTimestamp()).setType(Type.Put).setValue(RegionInfo.toByteArray(splitA)) + .build()); } if (splitB != null) { - put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(put.getRow()) - .setFamily(HConstants.CATALOG_FAMILY) - .setQualifier(HConstants.SPLITB_QUALIFIER) - .setTimestamp(put.getTimestamp()) - .setType(Type.Put) - .setValue(RegionInfo.toByteArray(splitB)) - .build()); + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.SPLITB_QUALIFIER) + .setTimestamp(put.getTimestamp()).setType(Type.Put).setValue(RegionInfo.toByteArray(splitB)) + .build()); } return put; } @@ -1363,7 +832,7 @@ public class MetaTableAccessor { * @param ps Put to add to hbase:meta */ public static void putsToMetaTable(final Connection connection, final List ps) - throws IOException { + throws IOException { if (ps.isEmpty()) { return; } @@ -1384,7 +853,7 @@ public class MetaTableAccessor { * @param d Delete to add to hbase:meta */ private static void deleteFromMetaTable(final Connection connection, final Delete d) - throws IOException { + throws IOException { List dels = new ArrayList<>(1); dels.add(d); deleteFromMetaTable(connection, dels); @@ -1393,10 +862,10 @@ public class MetaTableAccessor { /** * Delete the passed deletes from the hbase:meta table. * @param connection connection we're using - * @param deletes Deletes to add to hbase:meta This list should support #remove. + * @param deletes Deletes to add to hbase:meta This list should support #remove. */ private static void deleteFromMetaTable(final Connection connection, final List deletes) - throws IOException { + throws IOException { try (Table t = getMetaHTable(connection)) { debugLogMutations(deletes); t.delete(deletes); @@ -1412,20 +881,22 @@ public class MetaTableAccessor { */ public static void removeRegionReplicasFromMeta(Set metaRows, int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection) - throws IOException { + throws IOException { int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove; for (byte[] row : metaRows) { long now = EnvironmentEdgeManager.currentTime(); Delete deleteReplicaLocations = new Delete(row); for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) { - deleteReplicaLocations.addColumns(getCatalogFamily(), - getServerColumn(i), now); - deleteReplicaLocations.addColumns(getCatalogFamily(), - getSeqNumColumn(i), now); - deleteReplicaLocations.addColumns(getCatalogFamily(), - getStartCodeColumn(i), now); - deleteReplicaLocations.addColumns(getCatalogFamily(), getServerNameColumn(i), now); - deleteReplicaLocations.addColumns(getCatalogFamily(), getRegionStateColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getServerColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getSeqNumColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getStartCodeColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getServerNameColumn(i), now); + deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getRegionStateColumn(i), now); } deleteFromMetaTable(connection, deleteReplicaLocations); @@ -1433,14 +904,10 @@ public class MetaTableAccessor { } private static Put addRegionStateToPut(Put put, RegionState.State state) throws IOException { - put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(put.getRow()) - .setFamily(HConstants.CATALOG_FAMILY) - .setQualifier(getRegionStateColumn()) - .setTimestamp(put.getTimestamp()) - .setType(Cell.Type.Put) - .setValue(Bytes.toBytes(state.name())) - .build()); + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.STATE_QUALIFIER) + .setTimestamp(put.getTimestamp()).setType(Cell.Type.Put).setValue(Bytes.toBytes(state.name())) + .build()); return put; } @@ -1448,18 +915,17 @@ public class MetaTableAccessor { * Update state column in hbase:meta. */ public static void updateRegionState(Connection connection, RegionInfo ri, - RegionState.State state) throws IOException { + RegionState.State state) throws IOException { Put put = new Put(RegionReplicaUtil.getRegionInfoForDefaultReplica(ri).getRegionName()); - MetaTableAccessor.putsToMetaTable(connection, - Collections.singletonList(addRegionStateToPut(put, state))); + putsToMetaTable(connection, Collections.singletonList(addRegionStateToPut(put, state))); } /** * Adds daughter region infos to hbase:meta row for the specified region. Note that this does not * add its daughter's as different rows, but adds information about the daughters in the same row * as the parent. Use - * {@link #splitRegion(Connection, RegionInfo, long, RegionInfo, RegionInfo, ServerName, int)} - * if you want to do that. + * {@link #splitRegion(Connection, RegionInfo, long, RegionInfo, RegionInfo, ServerName, int)} if + * you want to do that. * @param connection connection we're using * @param regionInfo RegionInfo of parent region * @param splitA first split daughter of the parent regionInfo @@ -1467,7 +933,7 @@ public class MetaTableAccessor { * @throws IOException if problem connecting or updating meta */ public static void addSplitsToParent(Connection connection, RegionInfo regionInfo, - RegionInfo splitA, RegionInfo splitB) throws IOException { + RegionInfo splitA, RegionInfo splitB) throws IOException { try (Table meta = getMetaHTable(connection)) { Put put = makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime()); addDaughtersToPut(put, splitA, splitB); @@ -1479,43 +945,43 @@ public class MetaTableAccessor { /** * Adds a (single) hbase:meta row for the specified new region and its daughters. Note that this - * does not add its daughter's as different rows, but adds information about the daughters - * in the same row as the parent. Use - * {@link #splitRegion(Connection, RegionInfo, long, RegionInfo, RegionInfo, ServerName, int)} - * if you want to do that. + * does not add its daughter's as different rows, but adds information about the daughters in the + * same row as the parent. Use + * {@link #splitRegion(Connection, RegionInfo, long, RegionInfo, RegionInfo, ServerName, int)} if + * you want to do that. * @param connection connection we're using * @param regionInfo region information * @throws IOException if problem connecting or updating meta */ @VisibleForTesting public static void addRegionToMeta(Connection connection, RegionInfo regionInfo) - throws IOException { + throws IOException { addRegionsToMeta(connection, Collections.singletonList(regionInfo), 1); } /** - * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions - * is CLOSED. + * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions is + * CLOSED. * @param connection connection we're using * @param regionInfos region information list * @throws IOException if problem connecting or updating meta */ public static void addRegionsToMeta(Connection connection, List regionInfos, - int regionReplication) throws IOException { + int regionReplication) throws IOException { addRegionsToMeta(connection, regionInfos, regionReplication, EnvironmentEdgeManager.currentTime()); } /** - * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions - * is CLOSED. + * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions is + * CLOSED. * @param connection connection we're using * @param regionInfos region information list * @param ts desired timestamp * @throws IOException if problem connecting or updating meta */ private static void addRegionsToMeta(Connection connection, List regionInfos, - int regionReplication, long ts) throws IOException { + int regionReplication, long ts) throws IOException { List puts = new ArrayList<>(); for (RegionInfo regionInfo : regionInfos) { if (RegionReplicaUtil.isDefaultReplica(regionInfo)) { @@ -1534,46 +1000,42 @@ public class MetaTableAccessor { LOG.info("Added {} regions to meta.", puts.size()); } + @VisibleForTesting static Put addMergeRegions(Put put, Collection mergeRegions) throws IOException { int limit = 10000; // Arbitrary limit. No room in our formatted 'task0000' below for more. int max = mergeRegions.size(); if (max > limit) { // Should never happen!!!!! But just in case. - throw new RuntimeException("Can't merge " + max + " regions in one go; " + limit + - " is upper-limit."); + throw new RuntimeException( + "Can't merge " + max + " regions in one go; " + limit + " is upper-limit."); } int counter = 0; - for (RegionInfo ri: mergeRegions) { + for (RegionInfo ri : mergeRegions) { String qualifier = String.format(HConstants.MERGE_QUALIFIER_PREFIX_STR + "%04d", counter++); - put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY). - setRow(put.getRow()). - setFamily(HConstants.CATALOG_FAMILY). - setQualifier(Bytes.toBytes(qualifier)). - setTimestamp(put.getTimestamp()). - setType(Type.Put). - setValue(RegionInfo.toByteArray(ri)). - build()); + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.CATALOG_FAMILY).setQualifier(Bytes.toBytes(qualifier)) + .setTimestamp(put.getTimestamp()).setType(Type.Put).setValue(RegionInfo.toByteArray(ri)) + .build()); } return put; } /** - * Merge regions into one in an atomic operation. Deletes the merging regions in - * hbase:meta and adds the merged region. + * Merge regions into one in an atomic operation. Deletes the merging regions in hbase:meta and + * adds the merged region. * @param connection connection we're using * @param mergedRegion the merged region - * @param parentSeqNum Parent regions to merge and their next open sequence id used - * by serial replication. Set to -1 if not needed by this table. + * @param parentSeqNum Parent regions to merge and their next open sequence id used by serial + * replication. Set to -1 if not needed by this table. * @param sn the location of the region */ public static void mergeRegions(Connection connection, RegionInfo mergedRegion, - Map parentSeqNum, ServerName sn, int regionReplication) - throws IOException { + Map parentSeqNum, ServerName sn, int regionReplication) throws IOException { try (Table meta = getMetaHTable(connection)) { long time = HConstants.LATEST_TIMESTAMP; List mutations = new ArrayList<>(); List replicationParents = new ArrayList<>(); - for (Map.Entry e: parentSeqNum.entrySet()) { + for (Map.Entry e : parentSeqNum.entrySet()) { RegionInfo ri = e.getKey(); long seqNum = e.getValue(); // Deletes for merging regions @@ -1628,14 +1090,12 @@ public class MetaTableAccessor { * @param sn the location of the region */ public static void splitRegion(Connection connection, RegionInfo parent, long parentOpenSeqNum, - RegionInfo splitA, RegionInfo splitB, ServerName sn, int regionReplication) - throws IOException { + RegionInfo splitA, RegionInfo splitB, ServerName sn, int regionReplication) throws IOException { try (Table meta = getMetaHTable(connection)) { long time = EnvironmentEdgeManager.currentTime(); // Put for parent - Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent) - .setOffline(true) - .setSplit(true).build(), time); + Put putParent = makePutFromRegionInfo( + RegionInfoBuilder.newBuilder(parent).setOffline(true).setSplit(true).build(), time); addDaughtersToPut(putParent, splitA, splitB); // Puts for daughters @@ -1686,7 +1146,8 @@ public class MetaTableAccessor { */ public static Put makePutFromTableState(TableState state, long ts) { Put put = new Put(state.getTableName().getName(), ts); - put.addColumn(getTableFamily(), getTableStateColumn(), state.convert().toByteArray()); + put.addColumn(HConstants.TABLE_FAMILY, HConstants.TABLE_STATE_QUALIFIER, + state.convert().toByteArray()); return put; } @@ -1695,28 +1156,27 @@ public class MetaTableAccessor { * @param connection to use for deletion * @param table to delete state for */ - public static void deleteTableState(Connection connection, TableName table) - throws IOException { + public static void deleteTableState(Connection connection, TableName table) throws IOException { long time = EnvironmentEdgeManager.currentTime(); Delete delete = new Delete(table.getName()); - delete.addColumns(getTableFamily(), getTableStateColumn(), time); + delete.addColumns(HConstants.TABLE_FAMILY, HConstants.TABLE_STATE_QUALIFIER, time); deleteFromMetaTable(connection, delete); LOG.info("Deleted table " + table + " state from META"); } - private static void multiMutate(Table table, byte[] row, - Mutation... mutations) throws IOException { + private static void multiMutate(Table table, byte[] row, Mutation... mutations) + throws IOException { multiMutate(table, row, Arrays.asList(mutations)); } /** - * Performs an atomic multi-mutate operation against the given table. Used by the likes of - * merge and split as these want to make atomic mutations across multiple rows. + * Performs an atomic multi-mutate operation against the given table. Used by the likes of merge + * and split as these want to make atomic mutations across multiple rows. * @throws IOException even if we encounter a RuntimeException, we'll still wrap it in an IOE. */ @VisibleForTesting static void multiMutate(final Table table, byte[] row, final List mutations) - throws IOException { + throws IOException { debugLogMutations(mutations); Batch.Call callable = instance -> { MutateRowsRequest.Builder builder = MutateRowsRequest.newBuilder(); @@ -1769,7 +1229,7 @@ public class MetaTableAccessor { */ @VisibleForTesting public static void updateRegionLocation(Connection connection, RegionInfo regionInfo, - ServerName sn, long openSeqNum, long masterSystemTime) throws IOException { + ServerName sn, long openSeqNum, long masterSystemTime) throws IOException { updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime); } @@ -1787,9 +1247,9 @@ public class MetaTableAccessor { * is down on other end. */ private static void updateLocation(Connection connection, RegionInfo regionInfo, ServerName sn, - long openSeqNum, long masterSystemTime) throws IOException { + long openSeqNum, long masterSystemTime) throws IOException { // region replicas are kept in the primary region's row - Put put = new Put(getMetaKeyForRegion(regionInfo), masterSystemTime); + Put put = new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), masterSystemTime); addRegionInfo(put, regionInfo); addLocation(put, sn, openSeqNum, regionInfo.getReplicaId()); putToMetaTable(connection, put); @@ -1802,9 +1262,9 @@ public class MetaTableAccessor { * @param regionInfo region to be deleted from META */ public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo) - throws IOException { + throws IOException { Delete delete = new Delete(regionInfo.getRegionName()); - delete.addFamily(getCatalogFamily(), HConstants.LATEST_TIMESTAMP); + delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP); deleteFromMetaTable(connection, delete); LOG.info("Deleted " + regionInfo.getRegionNameAsString()); } @@ -1815,7 +1275,7 @@ public class MetaTableAccessor { * @param regionsInfo list of regions to be deleted from META */ public static void deleteRegionInfos(Connection connection, List regionsInfo) - throws IOException { + throws IOException { deleteRegionInfos(connection, regionsInfo, EnvironmentEdgeManager.currentTime()); } @@ -1825,12 +1285,11 @@ public class MetaTableAccessor { * @param regionsInfo list of regions to be deleted from META */ private static void deleteRegionInfos(Connection connection, List regionsInfo, - long ts) - throws IOException { + long ts) throws IOException { List deletes = new ArrayList<>(regionsInfo.size()); for (RegionInfo hri : regionsInfo) { Delete e = new Delete(hri.getRegionName()); - e.addFamily(getCatalogFamily(), ts); + e.addFamily(HConstants.CATALOG_FAMILY, ts); deletes.add(e); } deleteFromMetaTable(connection, deletes); @@ -1845,7 +1304,7 @@ public class MetaTableAccessor { * @param regionInfos list of regions to be added to META */ public static void overwriteRegions(Connection connection, List regionInfos, - int regionReplication) throws IOException { + int regionReplication) throws IOException { // use master time for delete marker and the Put long now = EnvironmentEdgeManager.currentTime(); deleteRegionInfos(connection, regionInfos, now); @@ -1866,7 +1325,7 @@ public class MetaTableAccessor { * @param mergeRegion the merged region */ public static void deleteMergeQualifiers(Connection connection, final RegionInfo mergeRegion) - throws IOException { + throws IOException { Delete delete = new Delete(mergeRegion.getRegionName()); // NOTE: We are doing a new hbase:meta read here. Cell[] cells = getRegionResult(connection, mergeRegion.getRegionName()).rawCells(); @@ -1880,7 +1339,7 @@ public class MetaTableAccessor { } byte[] qualifier = CellUtil.cloneQualifier(cell); qualifiers.add(qualifier); - delete.addColumns(getCatalogFamily(), qualifier, HConstants.LATEST_TIMESTAMP); + delete.addColumns(HConstants.CATALOG_FAMILY, qualifier, HConstants.LATEST_TIMESTAMP); } // There will be race condition that a GCMultipleMergedRegionsProcedure is scheduled while @@ -1894,53 +1353,36 @@ public class MetaTableAccessor { deleteFromMetaTable(connection, delete); LOG.info("Deleted merge references in " + mergeRegion.getRegionNameAsString() + - ", deleted qualifiers " + qualifiers.stream().map(Bytes::toStringBinary). - collect(Collectors.joining(", "))); + ", deleted qualifiers " + + qualifiers.stream().map(Bytes::toStringBinary).collect(Collectors.joining(", "))); } - public static Put addRegionInfo(final Put p, final RegionInfo hri) - throws IOException { - p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(HConstants.REGIONINFO_QUALIFIER) - .setTimestamp(p.getTimestamp()) - .setType(Type.Put) - // Serialize the Default Replica HRI otherwise scan of hbase:meta - // shows an info:regioninfo value with encoded name and region - // name that differs from that of the hbase;meta row. - .setValue(RegionInfo.toByteArray(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri))) - .build()); + public static Put addRegionInfo(final Put p, final RegionInfo hri) throws IOException { + p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow()) + .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.REGIONINFO_QUALIFIER) + .setTimestamp(p.getTimestamp()).setType(Type.Put) + // Serialize the Default Replica HRI otherwise scan of hbase:meta + // shows an info:regioninfo value with encoded name and region + // name that differs from that of the hbase;meta row. + .setValue(RegionInfo.toByteArray(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri))) + .build()); return p; } public static Put addLocation(Put p, ServerName sn, long openSeqNum, int replicaId) - throws IOException { + throws IOException { CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); - return p.add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getServerColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Cell.Type.Put) - .setValue(Bytes.toBytes(sn.getAddress().toString())) - .build()) - .add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getStartCodeColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Cell.Type.Put) - .setValue(Bytes.toBytes(sn.getStartcode())) - .build()) - .add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getSeqNumColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Type.Put) - .setValue(Bytes.toBytes(openSeqNum)) - .build()); + return p + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getServerColumn(replicaId)).setTimestamp(p.getTimestamp()) + .setType(Cell.Type.Put).setValue(Bytes.toBytes(sn.getAddress().toString())).build()) + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getStartCodeColumn(replicaId)) + .setTimestamp(p.getTimestamp()).setType(Cell.Type.Put) + .setValue(Bytes.toBytes(sn.getStartcode())).build()) + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getSeqNumColumn(replicaId)).setTimestamp(p.getTimestamp()) + .setType(Type.Put).setValue(Bytes.toBytes(openSeqNum)).build()); } private static void writeRegionName(ByteArrayOutputStream out, byte[] regionName) { @@ -1994,7 +1436,7 @@ public class MetaTableAccessor { } public static Put makePutForReplicationBarrier(RegionInfo regionInfo, long openSeqNum, long ts) - throws IOException { + throws IOException { Put put = new Put(regionInfo.getRegionName(), ts); addReplicationBarrier(put, openSeqNum); return put; @@ -2004,39 +1446,24 @@ public class MetaTableAccessor { * See class comment on SerialReplicationChecker */ public static void addReplicationBarrier(Put put, long openSeqNum) throws IOException { - put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(put.getRow()) - .setFamily(HConstants.REPLICATION_BARRIER_FAMILY) - .setQualifier(HConstants.SEQNUM_QUALIFIER) - .setTimestamp(put.getTimestamp()) - .setType(Type.Put) - .setValue(Bytes.toBytes(openSeqNum)) + put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow()) + .setFamily(HConstants.REPLICATION_BARRIER_FAMILY).setQualifier(HConstants.SEQNUM_QUALIFIER) + .setTimestamp(put.getTimestamp()).setType(Type.Put).setValue(Bytes.toBytes(openSeqNum)) .build()); } private static Put addEmptyLocation(Put p, int replicaId) throws IOException { CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); - return p.add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getServerColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Type.Put) - .build()) - .add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getStartCodeColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Cell.Type.Put) - .build()) - .add(builder.clear() - .setRow(p.getRow()) - .setFamily(getCatalogFamily()) - .setQualifier(getSeqNumColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Cell.Type.Put) - .build()); + return p + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getServerColumn(replicaId)).setTimestamp(p.getTimestamp()) + .setType(Type.Put).build()) + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getStartCodeColumn(replicaId)) + .setTimestamp(p.getTimestamp()).setType(Cell.Type.Put).build()) + .add(builder.clear().setRow(p.getRow()).setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getSeqNumColumn(replicaId)).setTimestamp(p.getTimestamp()) + .setType(Cell.Type.Put).build()); } public static final class ReplicationBarrierResult { @@ -2082,7 +1509,7 @@ public class MetaTableAccessor { private static ReplicationBarrierResult getReplicationBarrierResult(Result result) { long[] barriers = getReplicationBarriers(result); - byte[] stateBytes = result.getValue(getCatalogFamily(), getRegionStateColumn()); + byte[] stateBytes = result.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER); RegionState.State state = stateBytes != null ? RegionState.State.valueOf(Bytes.toString(stateBytes)) : null; byte[] parentRegionsBytes = @@ -2093,12 +1520,12 @@ public class MetaTableAccessor { } public static ReplicationBarrierResult getReplicationBarrierResult(Connection conn, - TableName tableName, byte[] row, byte[] encodedRegionName) throws IOException { + TableName tableName, byte[] row, byte[] encodedRegionName) throws IOException { byte[] metaStartKey = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false); byte[] metaStopKey = RegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW, "", false); Scan scan = new Scan().withStartRow(metaStartKey).withStopRow(metaStopKey) - .addColumn(getCatalogFamily(), getRegionStateColumn()) + .addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER) .addFamily(HConstants.REPLICATION_BARRIER_FAMILY).readAllVersions().setReversed(true) .setCaching(10); try (Table table = getMetaHTable(conn); ResultScanner scanner = table.getScanner(scan)) { @@ -2121,7 +1548,7 @@ public class MetaTableAccessor { } public static long[] getReplicationBarrier(Connection conn, byte[] regionName) - throws IOException { + throws IOException { try (Table table = getMetaHTable(conn)) { Result result = table.get(new Get(regionName) .addColumn(HConstants.REPLICATION_BARRIER_FAMILY, HConstants.SEQNUM_QUALIFIER) @@ -2131,10 +1558,12 @@ public class MetaTableAccessor { } public static List> getTableEncodedRegionNameAndLastBarrier(Connection conn, - TableName tableName) throws IOException { + TableName tableName) throws IOException { List> list = new ArrayList<>(); - scanMeta(conn, getTableStartRowForMeta(tableName, QueryType.REPLICATION), - getTableStopRowForMeta(tableName, QueryType.REPLICATION), QueryType.REPLICATION, r -> { + scanMeta(conn, + ClientMetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REPLICATION), + ClientMetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REPLICATION), + QueryType.REPLICATION, r -> { byte[] value = r.getValue(HConstants.REPLICATION_BARRIER_FAMILY, HConstants.SEQNUM_QUALIFIER); if (value == null) { @@ -2149,11 +1578,12 @@ public class MetaTableAccessor { } public static List getTableEncodedRegionNamesForSerialReplication(Connection conn, - TableName tableName) throws IOException { + TableName tableName) throws IOException { List list = new ArrayList<>(); - scanMeta(conn, getTableStartRowForMeta(tableName, QueryType.REPLICATION), - getTableStopRowForMeta(tableName, QueryType.REPLICATION), QueryType.REPLICATION, - new FirstKeyOnlyFilter(), Integer.MAX_VALUE, r -> { + scanMeta(conn, + ClientMetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REPLICATION), + ClientMetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REPLICATION), + QueryType.REPLICATION, new FirstKeyOnlyFilter(), Integer.MAX_VALUE, r -> { list.add(RegionInfo.encodeRegionName(r.getRow())); return true; }); @@ -2176,13 +1606,9 @@ public class MetaTableAccessor { } private static Put addSequenceNum(Put p, long openSeqNum, int replicaId) throws IOException { - return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) - .setRow(p.getRow()) - .setFamily(HConstants.CATALOG_FAMILY) - .setQualifier(getSeqNumColumn(replicaId)) - .setTimestamp(p.getTimestamp()) - .setType(Type.Put) - .setValue(Bytes.toBytes(openSeqNum)) - .build()); + return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow()) + .setFamily(HConstants.CATALOG_FAMILY) + .setQualifier(CatalogFamilyFormat.getSeqNumColumn(replicaId)).setTimestamp(p.getTimestamp()) + .setType(Type.Put).setValue(Bytes.toBytes(openSeqNum)).build()); } } diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java index 4b78135d14d..7ce8d6cf9d0 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java @@ -31,11 +31,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; - +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.MetaTableAccessor.Visitor; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -106,12 +106,12 @@ public class SnapshotOfRegionAssignmentFromMeta { LOG.info("Start to scan the hbase:meta for the current region assignment " + "snappshot"); // TODO: at some point this code could live in the MetaTableAccessor - Visitor v = new Visitor() { + ClientMetaTableAccessor.Visitor v = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result result) throws IOException { try { if (result == null || result.isEmpty()) return true; - RegionLocations rl = MetaTableAccessor.getRegionLocations(result); + RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result); if (rl == null) return true; RegionInfo hri = rl.getRegionLocation(0).getRegion(); if (hri == null) return true; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java new file mode 100644 index 00000000000..cec43aeef39 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogFamilyFormat.java @@ -0,0 +1,349 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.SortedMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; + +/** + * Helper class for generating/parsing + * {@value org.apache.hadoop.hbase.HConstants#CATALOG_FAMILY_STR} family cells in meta table. + *

+ * The cells in catalog family are: + * + *

+ * For each table range ('Region'), there is a single row, formatted as:
+ * <tableName>,<startKey>,<regionId>,<encodedRegionName>.
+ * This row is the serialized regionName of the default region replica.
+ * Columns are:
+ * info:regioninfo         => contains serialized HRI for the default region replica
+ * info:server             => contains hostname:port (in string form) for the server hosting
+ *                            the default regionInfo replica
+ * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting
+ *                                 the regionInfo replica with replicaId
+ * info:serverstartcode    => contains server start code (in binary long form) for the server
+ *                            hosting the default regionInfo replica
+ * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for
+ *                                          the server hosting the regionInfo replica with
+ *                                          replicaId
+ * info:seqnumDuringOpen   => contains seqNum (in binary long form) for the region at the time
+ *                            the server opened the region with default replicaId
+ * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region
+ *                                           at the time the server opened the region with
+ *                                           replicaId
+ * info:splitA             => contains a serialized HRI for the first daughter region if the
+ *                            region is split
+ * info:splitB             => contains a serialized HRI for the second daughter region if the
+ *                            region is split
+ * info:merge*             => contains a serialized HRI for a merge parent region. There will be two
+ *                            or more of these columns in a row. A row that has these columns is
+ *                            undergoing a merge and is the result of the merge. Columns listed
+ *                            in marge* columns are the parents of this merged region. Example
+ *                            columns: info:merge0001, info:merge0002. You make also see 'mergeA',
+ *                            and 'mergeB'. This is old form replaced by the new format that allows
+ *                            for more than two parents to be merged at a time.
+ * 
+ */ +@InterfaceAudience.Private +public class CatalogFamilyFormat { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogFamilyFormat.class); + + /** A regex for parsing server columns from meta. See above javadoc for meta layout */ + private static final Pattern SERVER_COLUMN_PATTERN = + Pattern.compile("^server(_[0-9a-fA-F]{4})?$"); + + /** + * Returns an HRI parsed from this regionName. Not all the fields of the HRI is stored in the + * name, so the returned object should only be used for the fields in the regionName. + *

+ * Since the returned object does not contain all the fields, we do not expose this method in + * public API, such as {@link RegionInfo} or {@link RegionInfoBuilder}. + */ + public static RegionInfo parseRegionInfoFromRegionName(byte[] regionName) throws IOException { + byte[][] fields = RegionInfo.parseRegionName(regionName); + long regionId = Long.parseLong(Bytes.toString(fields[2])); + int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0; + return RegionInfoBuilder.newBuilder(TableName.valueOf(fields[0])).setStartKey(fields[1]) + .setEndKey(fields[2]).setSplit(false).setRegionId(regionId).setReplicaId(replicaId).build(); + } + + /** + * Returns the RegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and + * qualifier of the catalog table result. + * @param r a Result object from the catalog table scan + * @param qualifier Column family qualifier + * @return An RegionInfo instance or null. + */ + @Nullable + public static RegionInfo getRegionInfo(final Result r, byte[] qualifier) { + Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, qualifier); + if (cell == null) { + return null; + } + return RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + } + + /** + * Returns RegionInfo object from the column + * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog table Result. + * @param data a Result object from the catalog table scan + * @return RegionInfo or null + */ + public static RegionInfo getRegionInfo(Result data) { + return getRegionInfo(data, HConstants.REGIONINFO_QUALIFIER); + } + + /** + * Returns the HRegionLocation parsed from the given meta row Result for the given regionInfo and + * replicaId. The regionInfo can be the default region info for the replica. + * @param r the meta row result + * @param regionInfo RegionInfo for default replica + * @param replicaId the replicaId for the HRegionLocation + * @return HRegionLocation parsed from the given meta row Result for the given replicaId + */ + public static HRegionLocation getRegionLocation(final Result r, final RegionInfo regionInfo, + final int replicaId) { + ServerName serverName = getServerName(r, replicaId); + long seqNum = getSeqNumDuringOpen(r, replicaId); + RegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId); + return new HRegionLocation(replicaInfo, serverName, seqNum); + } + + /** + * Returns an HRegionLocationList extracted from the result. + * @return an HRegionLocationList containing all locations for the region range or null if we + * can't deserialize the result. + */ + @Nullable + public static RegionLocations getRegionLocations(final Result r) { + if (r == null) { + return null; + } + RegionInfo regionInfo = getRegionInfo(r, HConstants.REGIONINFO_QUALIFIER); + if (regionInfo == null) { + return null; + } + + List locations = new ArrayList<>(1); + NavigableMap> familyMap = r.getNoVersionMap(); + + locations.add(getRegionLocation(r, regionInfo, 0)); + + NavigableMap infoMap = familyMap.get(HConstants.CATALOG_FAMILY); + if (infoMap == null) { + return new RegionLocations(locations); + } + + // iterate until all serverName columns are seen + int replicaId = 0; + byte[] serverColumn = getServerColumn(replicaId); + SortedMap serverMap; + serverMap = infoMap.tailMap(serverColumn, false); + + if (serverMap.isEmpty()) { + return new RegionLocations(locations); + } + + for (Map.Entry entry : serverMap.entrySet()) { + replicaId = parseReplicaIdFromServerColumn(entry.getKey()); + if (replicaId < 0) { + break; + } + HRegionLocation location = getRegionLocation(r, regionInfo, replicaId); + // In case the region replica is newly created, it's location might be null. We usually do not + // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. + if (location.getServerName() == null) { + locations.add(null); + } else { + locations.add(location); + } + } + + return new RegionLocations(locations); + } + + /** + * Returns a {@link ServerName} from catalog table {@link Result}. + * @param r Result to pull from + * @return A ServerName instance or null if necessary fields not found or empty. + */ + @Nullable + public static ServerName getServerName(Result r, int replicaId) { + byte[] serverColumn = getServerColumn(replicaId); + Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, serverColumn); + if (cell == null || cell.getValueLength() == 0) { + return null; + } + String hostAndPort = + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + byte[] startcodeColumn = getStartCodeColumn(replicaId); + cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, startcodeColumn); + if (cell == null || cell.getValueLength() == 0) { + return null; + } + try { + return ServerName.valueOf(hostAndPort, + Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); + } catch (IllegalArgumentException e) { + LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e); + return null; + } + } + + /** + * Returns the column qualifier for server column for replicaId + * @param replicaId the replicaId of the region + * @return a byte[] for server column qualifier + */ + public static byte[] getServerColumn(int replicaId) { + return replicaId == 0 ? HConstants.SERVER_QUALIFIER : + Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); + } + + /** + * Returns the column qualifier for server start code column for replicaId + * @param replicaId the replicaId of the region + * @return a byte[] for server start code column qualifier + */ + public static byte[] getStartCodeColumn(int replicaId) { + return replicaId == 0 ? HConstants.STARTCODE_QUALIFIER : + Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); + } + + /** + * The latest seqnum that the server writing to meta observed when opening the region. E.g. the + * seqNum when the result of {@link getServerName} was written. + * @param r Result to pull the seqNum from + * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written. + */ + private static long getSeqNumDuringOpen(final Result r, final int replicaId) { + Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId)); + if (cell == null || cell.getValueLength() == 0) { + return HConstants.NO_SEQNUM; + } + return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + } + + /** + * Returns the column qualifier for seqNum column for replicaId + * @param replicaId the replicaId of the region + * @return a byte[] for seqNum column qualifier + */ + public static byte[] getSeqNumColumn(int replicaId) { + return replicaId == 0 ? HConstants.SEQNUM_QUALIFIER : + Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); + } + + /** The delimiter for meta columns for replicaIds > 0 */ + @VisibleForTesting + static final char META_REPLICA_ID_DELIMITER = '_'; + + /** + * Parses the replicaId from the server column qualifier. See top of the class javadoc for the + * actual meta layout + * @param serverColumn the column qualifier + * @return an int for the replicaId + */ + @VisibleForTesting + static int parseReplicaIdFromServerColumn(byte[] serverColumn) { + String serverStr = Bytes.toString(serverColumn); + + Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr); + if (matcher.matches() && matcher.groupCount() > 0) { + String group = matcher.group(1); + if (group != null && group.length() > 0) { + return Integer.parseInt(group.substring(1), 16); + } else { + return 0; + } + } + return -1; + } + + /** Returns the row key to use for this regionInfo */ + public static byte[] getMetaKeyForRegion(RegionInfo regionInfo) { + return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName(); + } + + /** + * Returns the column qualifier for serialized region state + * @param replicaId the replicaId of the region + * @return a byte[] for state qualifier + */ + @VisibleForTesting + static byte[] getRegionStateColumn(int replicaId) { + return replicaId == 0 ? HConstants.STATE_QUALIFIER : + Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); + } + + /** + * Returns the column qualifier for serialized region state + * @param replicaId the replicaId of the region + * @return a byte[] for sn column qualifier + */ + public static byte[] getServerNameColumn(int replicaId) { + return replicaId == 0 ? HConstants.SERVERNAME_QUALIFIER : + Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER + + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); + } + + /** + * Decode table state from META Result. Should contain cell from HConstants.TABLE_FAMILY + * @return null if not found + */ + @Nullable + public static TableState getTableState(Result r) throws IOException { + Cell cell = r.getColumnLatestCell(HConstants.TABLE_FAMILY, HConstants.TABLE_STATE_QUALIFIER); + if (cell == null) { + return null; + } + try { + return TableState.parseFrom(TableName.valueOf(r.getRow()), + Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), + cell.getValueOffset() + cell.getValueLength())); + } catch (DeserializationException e) { + throw new IOException(e); + } + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java similarity index 50% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java rename to hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java index 3d4159175db..ecc65733c12 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java @@ -19,33 +19,23 @@ package org.apache.hadoop.hbase; import static org.apache.hadoop.hbase.util.FutureUtils.addListener; +import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.NavigableMap; import java.util.Optional; -import java.util.SortedMap; import java.util.concurrent.CompletableFuture; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.MetaTableAccessor.CollectingVisitor; -import org.apache.hadoop.hbase.MetaTableAccessor.QueryType; -import org.apache.hadoop.hbase.MetaTableAccessor.Visitor; import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer; import org.apache.hadoop.hbase.client.AsyncTable; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan.ReadType; import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; @@ -53,32 +43,45 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * The asynchronous meta table accessor. Used to read/write region and assignment information store - * in hbase:meta. + * The (asynchronous) meta table accessor used at client side. Used to read/write region and + * assignment information store in hbase:meta. * @since 2.0.0 + * @see CatalogFamilyFormat */ @InterfaceAudience.Private -public class AsyncMetaTableAccessor { +public final class ClientMetaTableAccessor { - private static final Logger LOG = LoggerFactory.getLogger(AsyncMetaTableAccessor.class); + private static final Logger LOG = LoggerFactory.getLogger(ClientMetaTableAccessor.class); + private ClientMetaTableAccessor() { + } - /** The delimiter for meta columns for replicaIds > 0 */ - private static final char META_REPLICA_ID_DELIMITER = '_'; + @InterfaceAudience.Private + public enum QueryType { + ALL(HConstants.TABLE_FAMILY, HConstants.CATALOG_FAMILY), REGION(HConstants.CATALOG_FAMILY), + TABLE(HConstants.TABLE_FAMILY), REPLICATION(HConstants.REPLICATION_BARRIER_FAMILY); - /** A regex for parsing server columns from meta. See above javadoc for meta layout */ - private static final Pattern SERVER_COLUMN_PATTERN = Pattern - .compile("^server(_[0-9a-fA-F]{4})?$"); + private final byte[][] families; + + QueryType(byte[]... families) { + this.families = families; + } + + byte[][] getFamilies() { + return this.families; + } + } public static CompletableFuture tableExists(AsyncTable metaTable, - TableName tableName) { + TableName tableName) { return getTableState(metaTable, tableName).thenApply(Optional::isPresent); } public static CompletableFuture> getTableState(AsyncTable metaTable, - TableName tableName) { + TableName tableName) { CompletableFuture> future = new CompletableFuture<>(); - Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getStateColumn()); + Get get = new Get(tableName.getName()).addColumn(HConstants.TABLE_FAMILY, + HConstants.TABLE_STATE_QUALIFIER); addListener(metaTable.get(get), (result, error) -> { if (error != null) { future.completeExceptionally(error); @@ -99,12 +102,12 @@ public class AsyncMetaTableAccessor { * @param regionName region we're looking for * @return HRegionLocation for the given region */ - public static CompletableFuture> getRegionLocation( - AsyncTable metaTable, byte[] regionName) { + public static CompletableFuture> + getRegionLocation(AsyncTable metaTable, byte[] regionName) { CompletableFuture> future = new CompletableFuture<>(); try { - RegionInfo parsedRegionInfo = MetaTableAccessor.parseRegionInfoFromRegionName(regionName); - addListener(metaTable.get(new Get(MetaTableAccessor.getMetaKeyForRegion(parsedRegionInfo)) + RegionInfo parsedRegionInfo = CatalogFamilyFormat.parseRegionInfoFromRegionName(regionName); + addListener(metaTable.get(new Get(CatalogFamilyFormat.getMetaKeyForRegion(parsedRegionInfo)) .addFamily(HConstants.CATALOG_FAMILY)), (r, err) -> { if (err != null) { future.completeExceptionally(err); @@ -126,8 +129,8 @@ public class AsyncMetaTableAccessor { * @param encodedRegionName region we're looking for * @return HRegionLocation for the given region */ - public static CompletableFuture> getRegionLocationWithEncodedName( - AsyncTable metaTable, byte[] encodedRegionName) { + public static CompletableFuture> + getRegionLocationWithEncodedName(AsyncTable metaTable, byte[] encodedRegionName) { CompletableFuture> future = new CompletableFuture<>(); addListener( metaTable @@ -139,7 +142,7 @@ public class AsyncMetaTableAccessor { } String encodedRegionNameStr = Bytes.toString(encodedRegionName); results.stream().filter(result -> !result.isEmpty()) - .filter(result -> MetaTableAccessor.getRegionInfo(result) != null).forEach(result -> { + .filter(result -> CatalogFamilyFormat.getRegionInfo(result) != null).forEach(result -> { getRegionLocations(result).ifPresent(locations -> { for (HRegionLocation location : locations.getRegionLocations()) { if (location != null && @@ -156,16 +159,7 @@ public class AsyncMetaTableAccessor { } private static Optional getTableState(Result r) throws IOException { - Cell cell = r.getColumnLatestCell(getTableFamily(), getStateColumn()); - if (cell == null) return Optional.empty(); - try { - return Optional.of(TableState.parseFrom( - TableName.valueOf(r.getRow()), - Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), cell.getValueOffset() - + cell.getValueLength()))); - } catch (DeserializationException e) { - throw new IOException("Failed to parse table state from result: " + r, e); - } + return Optional.ofNullable(CatalogFamilyFormat.getTableState(r)); } /** @@ -176,7 +170,7 @@ public class AsyncMetaTableAccessor { * {@link CompletableFuture}. */ public static CompletableFuture> getTableHRegionLocations( - AsyncTable metaTable, TableName tableName) { + AsyncTable metaTable, TableName tableName) { CompletableFuture> future = new CompletableFuture<>(); addListener(getTableRegionsAndLocations(metaTable, tableName, true), (locations, err) -> { if (err != null) { @@ -202,46 +196,17 @@ public class AsyncMetaTableAccessor { * {@link CompletableFuture}. */ private static CompletableFuture>> getTableRegionsAndLocations( - final AsyncTable metaTable, - final TableName tableName, final boolean excludeOfflinedSplitParents) { + final AsyncTable metaTable, final TableName tableName, + final boolean excludeOfflinedSplitParents) { CompletableFuture>> future = new CompletableFuture<>(); if (TableName.META_TABLE_NAME.equals(tableName)) { future.completeExceptionally(new IOException( - "This method can't be used to locate meta regions;" + " use MetaTableLocator instead")); + "This method can't be used to locate meta regions;" + " use MetaTableLocator instead")); } // Make a version of CollectingVisitor that collects RegionInfo and ServerAddress - CollectingVisitor> visitor = - new CollectingVisitor>() { - private RegionLocations current = null; - - @Override - public boolean visit(Result r) throws IOException { - Optional currentRegionLocations = getRegionLocations(r); - current = currentRegionLocations.orElse(null); - if (current == null || current.getRegionLocation().getRegion() == null) { - LOG.warn("No serialized RegionInfo in " + r); - return true; - } - RegionInfo hri = current.getRegionLocation().getRegion(); - if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; - // Else call super and add this Result to the collection. - return super.visit(r); - } - - @Override - void add(Result r) { - if (current == null) { - return; - } - for (HRegionLocation loc : current.getRegionLocations()) { - if (loc != null) { - this.results.add(new Pair(loc.getRegion(), loc - .getServerName())); - } - } - } - }; + CollectRegionLocationsVisitor visitor = + new CollectRegionLocationsVisitor(excludeOfflinedSplitParents); addListener(scanMeta(metaTable, tableName, QueryType.REGION, visitor), (v, error) -> { if (error != null) { @@ -261,7 +226,7 @@ public class AsyncMetaTableAccessor { * @param visitor Visitor invoked against each row */ private static CompletableFuture scanMeta(AsyncTable metaTable, - TableName tableName, QueryType type, final Visitor visitor) { + TableName tableName, QueryType type, final Visitor visitor) { return scanMeta(metaTable, getTableStartRowForMeta(tableName, type), getTableStopRowForMeta(tableName, type), type, Integer.MAX_VALUE, visitor); } @@ -276,7 +241,7 @@ public class AsyncMetaTableAccessor { * @param visitor Visitor invoked against each row */ private static CompletableFuture scanMeta(AsyncTable metaTable, - byte[] startRow, byte[] stopRow, QueryType type, int maxRows, final Visitor visitor) { + byte[] startRow, byte[] stopRow, QueryType type, int maxRows, final Visitor visitor) { int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; Scan scan = getMetaScan(metaTable, rowUpperLimit); for (byte[] family : type.getFamilies()) { @@ -290,9 +255,9 @@ public class AsyncMetaTableAccessor { } if (LOG.isDebugEnabled()) { - LOG.debug("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) - + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" - + rowUpperLimit + " with caching=" + scan.getCaching()); + LOG.debug("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) + + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" + + rowUpperLimit + " with caching=" + scan.getCaching()); } CompletableFuture future = new CompletableFuture(); @@ -311,7 +276,7 @@ public class AsyncMetaTableAccessor { private final CompletableFuture future; MetaTableScanResultConsumer(int rowUpperLimit, Visitor visitor, - CompletableFuture future) { + CompletableFuture future) { this.rowUpperLimit = rowUpperLimit; this.visitor = visitor; this.future = future; @@ -355,6 +320,98 @@ public class AsyncMetaTableAccessor { } } + /** + * Implementations 'visit' a catalog table row. + */ + public interface Visitor { + /** + * Visit the catalog table row. + * @param r A row from catalog table + * @return True if we are to proceed scanning the table, else false if we are to stop now. + */ + boolean visit(final Result r) throws IOException; + } + + /** + * Implementations 'visit' a catalog table row but with close() at the end. + */ + public interface CloseableVisitor extends Visitor, Closeable { + } + + /** + * A {@link Visitor} that collects content out of passed {@link Result}. + */ + private static abstract class CollectingVisitor implements Visitor { + final List results = new ArrayList<>(); + + @Override + public boolean visit(Result r) throws IOException { + if (r != null && !r.isEmpty()) { + add(r); + } + return true; + } + + abstract void add(Result r); + + /** + * @return Collected results; wait till visits complete to collect all possible results + */ + List getResults() { + return this.results; + } + } + + static class CollectRegionLocationsVisitor + extends CollectingVisitor> { + + private final boolean excludeOfflinedSplitParents; + + private RegionLocations current = null; + + CollectRegionLocationsVisitor(boolean excludeOfflinedSplitParents) { + this.excludeOfflinedSplitParents = excludeOfflinedSplitParents; + } + + @Override + public boolean visit(Result r) throws IOException { + Optional currentRegionLocations = getRegionLocations(r); + current = currentRegionLocations.orElse(null); + if (current == null || current.getRegionLocation().getRegion() == null) { + LOG.warn("No serialized RegionInfo in " + r); + return true; + } + RegionInfo hri = current.getRegionLocation().getRegion(); + if (excludeOfflinedSplitParents && hri.isSplitParent()) { + return true; + } + // Else call super and add this Result to the collection. + return super.visit(r); + } + + @Override + void add(Result r) { + if (current == null) { + return; + } + for (HRegionLocation loc : current.getRegionLocations()) { + if (loc != null) { + this.results.add(new Pair(loc.getRegion(), loc.getServerName())); + } + } + } + } + + /** + * Collects all returned. + */ + static class CollectAllVisitor extends CollectingVisitor { + @Override + void add(Result r) { + this.results.add(r); + } + } + private static Scan getMetaScan(AsyncTable metaTable, int rowUpperLimit) { Scan scan = new Scan(); int scannerCaching = metaTable.getConfiguration().getInt(HConstants.HBASE_META_SCANNER_CACHING, @@ -376,102 +433,15 @@ public class AsyncMetaTableAccessor { * @return an HRegionLocationList containing all locations for the region range or null if we * can't deserialize the result. */ - private static Optional getRegionLocations(final Result r) { - if (r == null) return Optional.empty(); - Optional regionInfo = getHRegionInfo(r, getRegionInfoColumn()); - if (!regionInfo.isPresent()) return Optional.empty(); - - List locations = new ArrayList(1); - NavigableMap> familyMap = r.getNoVersionMap(); - - locations.add(getRegionLocation(r, regionInfo.get(), 0)); - - NavigableMap infoMap = familyMap.get(getCatalogFamily()); - if (infoMap == null) return Optional.of(new RegionLocations(locations)); - - // iterate until all serverName columns are seen - int replicaId = 0; - byte[] serverColumn = getServerColumn(replicaId); - SortedMap serverMap = null; - serverMap = infoMap.tailMap(serverColumn, false); - - if (serverMap.isEmpty()) return Optional.of(new RegionLocations(locations)); - - for (Map.Entry entry : serverMap.entrySet()) { - replicaId = parseReplicaIdFromServerColumn(entry.getKey()); - if (replicaId < 0) { - break; - } - HRegionLocation location = getRegionLocation(r, regionInfo.get(), replicaId); - // In case the region replica is newly created, it's location might be null. We usually do not - // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. - if (location == null || location.getServerName() == null) { - locations.add(null); - } else { - locations.add(location); - } - } - - return Optional.of(new RegionLocations(locations)); - } - - /** - * Returns the HRegionLocation parsed from the given meta row Result - * for the given regionInfo and replicaId. The regionInfo can be the default region info - * for the replica. - * @param r the meta row result - * @param regionInfo RegionInfo for default replica - * @param replicaId the replicaId for the HRegionLocation - * @return HRegionLocation parsed from the given meta row Result for the given replicaId - */ - private static HRegionLocation getRegionLocation(final Result r, final RegionInfo regionInfo, - final int replicaId) { - Optional serverName = getServerName(r, replicaId); - long seqNum = getSeqNumDuringOpen(r, replicaId); - RegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId); - return new HRegionLocation(replicaInfo, serverName.orElse(null), seqNum); - } - - /** - * Returns a {@link ServerName} from catalog table {@link Result}. - * @param r Result to pull from - * @return A ServerName instance. - */ - private static Optional getServerName(final Result r, final int replicaId) { - byte[] serverColumn = getServerColumn(replicaId); - Cell cell = r.getColumnLatestCell(getCatalogFamily(), serverColumn); - if (cell == null || cell.getValueLength() == 0) return Optional.empty(); - String hostAndPort = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), - cell.getValueLength()); - byte[] startcodeColumn = getStartCodeColumn(replicaId); - cell = r.getColumnLatestCell(getCatalogFamily(), startcodeColumn); - if (cell == null || cell.getValueLength() == 0) return Optional.empty(); - try { - return Optional.of(ServerName.valueOf(hostAndPort, - Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()))); - } catch (IllegalArgumentException e) { - LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e); - return Optional.empty(); - } - } - - /** - * The latest seqnum that the server writing to meta observed when opening the region. - * E.g. the seqNum when the result of {@link #getServerName(Result, int)} was written. - * @param r Result to pull the seqNum from - * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written. - */ - private static long getSeqNumDuringOpen(final Result r, final int replicaId) { - Cell cell = r.getColumnLatestCell(getCatalogFamily(), getSeqNumColumn(replicaId)); - if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM; - return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + private static Optional getRegionLocations(Result r) { + return Optional.ofNullable(CatalogFamilyFormat.getRegionLocations(r)); } /** * @param tableName table we're working with * @return start row for scanning META according to query type */ - private static byte[] getTableStartRowForMeta(TableName tableName, QueryType type) { + public static byte[] getTableStartRowForMeta(TableName tableName, QueryType type) { if (tableName == null) { return null; } @@ -496,7 +466,7 @@ public class AsyncMetaTableAccessor { * @param tableName table we're working with * @return stop row for scanning META according to query type */ - private static byte[] getTableStopRowForMeta(TableName tableName, QueryType type) { + public static byte[] getTableStopRowForMeta(TableName tableName, QueryType type) { if (tableName == null) { return null; } @@ -522,107 +492,4 @@ public class AsyncMetaTableAccessor { } return stopRow; } - - /** - * Returns the RegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and - * qualifier of the catalog table result. - * @param r a Result object from the catalog table scan - * @param qualifier Column family qualifier - * @return An RegionInfo instance. - */ - private static Optional getHRegionInfo(final Result r, byte[] qualifier) { - Cell cell = r.getColumnLatestCell(getCatalogFamily(), qualifier); - if (cell == null) return Optional.empty(); - return Optional.ofNullable(RegionInfo.parseFromOrNull(cell.getValueArray(), - cell.getValueOffset(), cell.getValueLength())); - } - - /** - * Returns the column family used for meta columns. - * @return HConstants.CATALOG_FAMILY. - */ - private static byte[] getCatalogFamily() { - return HConstants.CATALOG_FAMILY; - } - - /** - * Returns the column family used for table columns. - * @return HConstants.TABLE_FAMILY. - */ - private static byte[] getTableFamily() { - return HConstants.TABLE_FAMILY; - } - - /** - * Returns the column qualifier for serialized region info - * @return HConstants.REGIONINFO_QUALIFIER - */ - private static byte[] getRegionInfoColumn() { - return HConstants.REGIONINFO_QUALIFIER; - } - - /** - * Returns the column qualifier for serialized table state - * @return HConstants.TABLE_STATE_QUALIFIER - */ - private static byte[] getStateColumn() { - return HConstants.TABLE_STATE_QUALIFIER; - } - - /** - * Returns the column qualifier for server column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for server column qualifier - */ - private static byte[] getServerColumn(int replicaId) { - return replicaId == 0 - ? HConstants.SERVER_QUALIFIER - : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for server start code column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for server start code column qualifier - */ - private static byte[] getStartCodeColumn(int replicaId) { - return replicaId == 0 - ? HConstants.STARTCODE_QUALIFIER - : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Returns the column qualifier for seqNum column for replicaId - * @param replicaId the replicaId of the region - * @return a byte[] for seqNum column qualifier - */ - private static byte[] getSeqNumColumn(int replicaId) { - return replicaId == 0 - ? HConstants.SEQNUM_QUALIFIER - : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER - + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); - } - - /** - * Parses the replicaId from the server column qualifier. See top of the class javadoc - * for the actual meta layout - * @param serverColumn the column qualifier - * @return an int for the replicaId - */ - private static int parseReplicaIdFromServerColumn(byte[] serverColumn) { - String serverStr = Bytes.toString(serverColumn); - - Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr); - if (matcher.matches() && matcher.groupCount() > 0) { - String group = matcher.group(1); - if (group != null && group.length() > 0) { - return Integer.parseInt(group.substring(1), 16); - } else { - return 0; - } - } - return -1; - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java index d36ffea9b6e..b2021688816 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java @@ -47,10 +47,10 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.commons.lang3.ObjectUtils; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -315,7 +315,7 @@ class AsyncNonMetaRegionLocator { // return whether we should stop the scan private boolean onScanNext(TableName tableName, LocateRequest req, Result result) { - RegionLocations locs = MetaTableAccessor.getRegionLocations(result); + RegionLocations locs = CatalogFamilyFormat.getRegionLocations(result); if (LOG.isDebugEnabled()) { LOG.debug("The fetched location of '{}', row='{}', locateType={} is {}", tableName, Bytes.toStringBinary(req.row), req.locateType, locs); @@ -476,7 +476,7 @@ class AsyncNonMetaRegionLocator { if (i < results.length) { TableCache tableCache = getTableCache(tableName); for (; i < results.length; i++) { - RegionLocations locs = MetaTableAccessor.getRegionLocations(results[i]); + RegionLocations locs = CatalogFamilyFormat.getRegionLocations(results[i]); if (locs == null) { continue; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncTableRegionLocatorImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncTableRegionLocatorImpl.java index fa3ea1ca4df..ad6a051208f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncTableRegionLocatorImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncTableRegionLocatorImpl.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.client; import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; @@ -58,8 +58,8 @@ class AsyncTableRegionLocatorImpl implements AsyncTableRegionLocator { return conn.registry.getMetaRegionLocations() .thenApply(locs -> Arrays.asList(locs.getRegionLocations())); } - return AsyncMetaTableAccessor.getTableHRegionLocations(conn.getTable(TableName.META_TABLE_NAME), - tableName); + return ClientMetaTableAccessor + .getTableHRegionLocations(conn.getTable(TableName.META_TABLE_NAME), tableName); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 8cc911ac58e..5ef955ddc43 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -46,16 +46,15 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.AsyncMetaTableAccessor; import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.CacheEvictionStatsAggregator; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.ClusterMetricsBuilder; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.MetaTableAccessor.QueryType; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.RegionMetrics; @@ -507,7 +506,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { if (TableName.isMetaTableName(tableName)) { return CompletableFuture.completedFuture(true); } - return AsyncMetaTableAccessor.tableExists(metaTable, tableName); + return ClientMetaTableAccessor.tableExists(metaTable, tableName); } @Override @@ -722,10 +721,11 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return CompletableFuture.completedFuture(true); } CompletableFuture future = new CompletableFuture<>(); - addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (tableState, error) -> { - completeCheckTableState(future, tableState.isPresent()? tableState.get(): null, error, - TableState.State.ENABLED, tableName); - }); + addListener(ClientMetaTableAccessor.getTableState(metaTable, tableName), + (tableState, error) -> { + completeCheckTableState(future, tableState.isPresent() ? tableState.get() : null, error, + TableState.State.ENABLED, tableName); + }); return future; } @@ -735,10 +735,11 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return CompletableFuture.completedFuture(false); } CompletableFuture future = new CompletableFuture<>(); - addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (tableState, error) -> { - completeCheckTableState(future, tableState.isPresent()? tableState.get(): null, error, - TableState.State.DISABLED, tableName); - }); + addListener(ClientMetaTableAccessor.getTableState(metaTable, tableName), + (tableState, error) -> { + completeCheckTableState(future, tableState.isPresent() ? tableState.get() : null, error, + TableState.State.DISABLED, tableName); + }); return future; } @@ -762,7 +763,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { future.complete(false); } else { addListener( - AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName), + ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName), (locations, error1) -> { if (error1 != null) { future.completeExceptionally(error1); @@ -887,7 +888,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { .thenApply(locs -> Stream.of(locs.getRegionLocations()).map(HRegionLocation::getRegion) .collect(Collectors.toList())); } else { - return AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName) + return ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName) .thenApply( locs -> locs.stream().map(HRegionLocation::getRegion).collect(Collectors.toList())); } @@ -1123,7 +1124,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return future; } else { // For non-meta table, we fetch all locations by scanning hbase:meta table - return AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName); + return ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName); } } @@ -1333,11 +1334,12 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { future.completeExceptionally(new TableNotFoundException(tableName)); return; } - addListener( - metaTable - .scanAll(new Scan().setReadType(ReadType.PREAD).addFamily(HConstants.CATALOG_FAMILY) - .withStartRow(MetaTableAccessor.getTableStartRowForMeta(tableName, QueryType.REGION)) - .withStopRow(MetaTableAccessor.getTableStopRowForMeta(tableName, QueryType.REGION))), + addListener(metaTable + .scanAll(new Scan().setReadType(ReadType.PREAD).addFamily(HConstants.CATALOG_FAMILY) + .withStartRow(ClientMetaTableAccessor.getTableStartRowForMeta(tableName, + ClientMetaTableAccessor.QueryType.REGION)) + .withStopRow(ClientMetaTableAccessor.getTableStopRowForMeta(tableName, + ClientMetaTableAccessor.QueryType.REGION))), (results, err2) -> { if (err2 != null) { future.completeExceptionally(err2); @@ -1346,10 +1348,10 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { if (results != null && !results.isEmpty()) { List> splitFutures = new ArrayList<>(); for (Result r : results) { - if (r.isEmpty() || MetaTableAccessor.getRegionInfo(r) == null) { + if (r.isEmpty() || CatalogFamilyFormat.getRegionInfo(r) == null) { continue; } - RegionLocations rl = MetaTableAccessor.getRegionLocations(r); + RegionLocations rl = CatalogFamilyFormat.getRegionLocations(r); if (rl != null) { for (HRegionLocation h : rl.getRegionLocations()) { if (h != null && h.getServerName() != null) { @@ -2377,12 +2379,12 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { .thenApply(locs -> Stream.of(locs.getRegionLocations()) .filter(loc -> loc.getRegion().getEncodedName().equals(encodedName)).findFirst()); } else { - future = AsyncMetaTableAccessor.getRegionLocationWithEncodedName(metaTable, + future = ClientMetaTableAccessor.getRegionLocationWithEncodedName(metaTable, regionNameOrEncodedRegionName); } } else { RegionInfo regionInfo = - MetaTableAccessor.parseRegionInfoFromRegionName(regionNameOrEncodedRegionName); + CatalogFamilyFormat.parseRegionInfoFromRegionName(regionNameOrEncodedRegionName); if (regionInfo.isMetaRegion()) { future = connection.registry.getMetaRegionLocations() .thenApply(locs -> Stream.of(locs.getRegionLocations()) @@ -2390,7 +2392,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { .findFirst()); } else { future = - AsyncMetaTableAccessor.getRegionLocation(metaTable, regionNameOrEncodedRegionName); + ClientMetaTableAccessor.getRegionLocation(metaTable, regionNameOrEncodedRegionName); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestCatalogFamilyFormat.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestCatalogFamilyFormat.java new file mode 100644 index 00000000000..78e0fdba301 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestCatalogFamilyFormat.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, SmallTests.class }) +public class TestCatalogFamilyFormat { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCatalogFamilyFormat.class); + + @Test + public void testParseReplicaIdFromServerColumn() { + String column1 = HConstants.SERVER_QUALIFIER_STR; + assertEquals(0, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column1))); + String column2 = column1 + CatalogFamilyFormat.META_REPLICA_ID_DELIMITER; + assertEquals(-1, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column2))); + String column3 = column2 + "00"; + assertEquals(-1, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column3))); + String column4 = column3 + "2A"; + assertEquals(42, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column4))); + String column5 = column4 + "2A"; + assertEquals(-1, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column5))); + String column6 = HConstants.STARTCODE_QUALIFIER_STR; + assertEquals(-1, CatalogFamilyFormat.parseReplicaIdFromServerColumn(Bytes.toBytes(column6))); + } + + @Test + public void testMetaReaderGetColumnMethods() { + assertArrayEquals(HConstants.SERVER_QUALIFIER, CatalogFamilyFormat.getServerColumn(0)); + assertArrayEquals( + Bytes.toBytes( + HConstants.SERVER_QUALIFIER_STR + CatalogFamilyFormat.META_REPLICA_ID_DELIMITER + "002A"), + CatalogFamilyFormat.getServerColumn(42)); + + assertArrayEquals(HConstants.STARTCODE_QUALIFIER, CatalogFamilyFormat.getStartCodeColumn(0)); + assertArrayEquals( + Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + + CatalogFamilyFormat.META_REPLICA_ID_DELIMITER + "002A"), + CatalogFamilyFormat.getStartCodeColumn(42)); + + assertArrayEquals(HConstants.SEQNUM_QUALIFIER, CatalogFamilyFormat.getSeqNumColumn(0)); + assertArrayEquals( + Bytes.toBytes( + HConstants.SEQNUM_QUALIFIER_STR + CatalogFamilyFormat.META_REPLICA_ID_DELIMITER + "002A"), + CatalogFamilyFormat.getSeqNumColumn(42)); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index 7567713883a..e991f3f8ef3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -31,6 +31,8 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -537,7 +539,7 @@ public class CatalogJanitor extends ScheduledChore { * generate more report. Report is NOT ready until after this visitor has been * {@link #close()}'d. */ - static class ReportMakingVisitor implements MetaTableAccessor.CloseableVisitor { + static class ReportMakingVisitor implements ClientMetaTableAccessor.CloseableVisitor { private final MasterServices services; private volatile boolean closed; @@ -615,10 +617,10 @@ public class CatalogJanitor extends ScheduledChore { // If locations is null, ensure the regioninfo is for sure empty before progressing. // If really empty, report as missing regioninfo! Otherwise, can run server check // and get RegionInfo from locations. - RegionLocations locations = MetaTableAccessor.getRegionLocations(metaTableRow); + RegionLocations locations = CatalogFamilyFormat.getRegionLocations(metaTableRow); if (locations == null) { - ri = MetaTableAccessor.getRegionInfo(metaTableRow, - MetaTableAccessor.getRegionInfoColumn()); + ri = CatalogFamilyFormat.getRegionInfo(metaTableRow, + HConstants.REGIONINFO_QUALIFIER); } else { ri = locations.getDefaultRegionLocation().getRegion(); checkServer(locations); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java index 3c23610ff47..984833fbbac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java @@ -26,6 +26,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; @@ -133,10 +135,10 @@ public class TableStateManager { Set getTablesInStates(TableState.State... states) throws IOException { // Only be called in region normalizer, will not use cache. final Set rv = Sets.newHashSet(); - MetaTableAccessor.fullScanTables(master.getConnection(), new MetaTableAccessor.Visitor() { + MetaTableAccessor.fullScanTables(master.getConnection(), new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result r) throws IOException { - TableState tableState = MetaTableAccessor.getTableState(r); + TableState tableState = CatalogFamilyFormat.getTableState(r); if (tableState != null && tableState.inStates(states)) { rv.add(tableState.getTableName()); } @@ -212,10 +214,10 @@ public class TableStateManager { throws IOException { Map states = new HashMap<>(); // NOTE: Full hbase:meta table scan! - MetaTableAccessor.fullScanTables(connection, new MetaTableAccessor.Visitor() { + MetaTableAccessor.fullScanTables(connection, new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result r) throws IOException { - TableState state = MetaTableAccessor.getTableState(r); + TableState state = CatalogFamilyFormat.getTableState(r); states.put(state.getTableName().getNameAsString(), state); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index f6fddebe453..ef6e30cc9fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -34,10 +34,10 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -1123,7 +1123,7 @@ public class AssignmentManager { */ private void closeRegionSilently(ServerName sn, byte [] regionName) { try { - RegionInfo ri = MetaTableAccessor.parseRegionInfoFromRegionName(regionName); + RegionInfo ri = CatalogFamilyFormat.parseRegionInfoFromRegionName(regionName); // Pass -1 for timeout. Means do not wait. ServerManager.closeRegionSilentlyAndWait(this.master.getAsyncClusterConnection(), sn, ri, -1); } catch (Exception e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index ffce38c39f0..63e74cc5e4c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -22,10 +22,11 @@ import java.util.Collections; import java.util.List; import java.util.SortedMap; import java.util.TreeMap; - +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -75,27 +76,28 @@ public class RegionStateStore { } public void visitMeta(final RegionStateVisitor visitor) throws IOException { - MetaTableAccessor.fullScanRegions(master.getConnection(), new MetaTableAccessor.Visitor() { - final boolean isDebugEnabled = LOG.isDebugEnabled(); + MetaTableAccessor.fullScanRegions(master.getConnection(), + new ClientMetaTableAccessor.Visitor() { + final boolean isDebugEnabled = LOG.isDebugEnabled(); - @Override - public boolean visit(final Result r) throws IOException { - if (r != null && !r.isEmpty()) { - long st = 0; - if (LOG.isTraceEnabled()) { - st = System.currentTimeMillis(); + @Override + public boolean visit(final Result r) throws IOException { + if (r != null && !r.isEmpty()) { + long st = 0; + if (LOG.isTraceEnabled()) { + st = System.currentTimeMillis(); + } + visitMetaEntry(visitor, r); + if (LOG.isTraceEnabled()) { + long et = System.currentTimeMillis(); + LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st)); + } + } else if (isDebugEnabled) { + LOG.debug("NULL result from meta - ignoring but this is strange."); } - visitMetaEntry(visitor, r); - if (LOG.isTraceEnabled()) { - long et = System.currentTimeMillis(); - LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st)); - } - } else if (isDebugEnabled) { - LOG.debug("NULL result from meta - ignoring but this is strange."); + return true; } - return true; - } - }); + }); } /** @@ -117,7 +119,7 @@ public class RegionStateStore { private void visitMetaEntry(final RegionStateVisitor visitor, final Result result) throws IOException { - final RegionLocations rl = MetaTableAccessor.getRegionLocations(result); + final RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result); if (rl == null) return; final HRegionLocation[] locations = rl.getRegionLocations(); @@ -176,7 +178,7 @@ public class RegionStateStore { long pid) throws IOException { long time = EnvironmentEdgeManager.currentTime(); final int replicaId = regionInfo.getReplicaId(); - final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo), time); + final Put put = new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), time); MetaTableAccessor.addRegionInfo(put, regionInfo); final StringBuilder info = new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=") @@ -199,7 +201,7 @@ public class RegionStateStore { put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) .setRow(put.getRow()) .setFamily(HConstants.CATALOG_FAMILY) - .setQualifier(MetaTableAccessor.getServerNameColumn(replicaId)) + .setQualifier(CatalogFamilyFormat.getServerNameColumn(replicaId)) .setTimestamp(put.getTimestamp()) .setType(Cell.Type.Put) .setValue(Bytes.toBytes(regionLocation.getServerName())) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java index 614e4a203e1..a71e467b881 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.procedure; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -184,7 +185,7 @@ public class EnableTableProcedure for (int i = 1; i < regionReplicaCount; i++) { // Since we have already added the entries to the META we will be getting only that here List columnCells = - r.getColumnCells(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i)); + r.getColumnCells(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i)); if (!columnCells.isEmpty()) { replicasFound++; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java index 9690c18405e..725a138ca91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java @@ -21,7 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; - +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.RegionLocations; @@ -112,10 +113,10 @@ public class HBCKServerCrashProcedure extends ServerCrashProcedure { } /** - * Visitor for hbase:meta that 'fixes' Unknown Server issues. Collects - * a List of Regions to reassign as 'result'. + * Visitor for hbase:meta that 'fixes' Unknown Server issues. Collects a List of Regions to + * reassign as 'result'. */ - private static class UnknownServerVisitor implements MetaTableAccessor.Visitor { + private static final class UnknownServerVisitor implements ClientMetaTableAccessor.Visitor { private final List reassigns = new ArrayList<>(); private final ServerName unknownServerName; private final Connection connection; @@ -127,7 +128,7 @@ public class HBCKServerCrashProcedure extends ServerCrashProcedure { @Override public boolean visit(Result result) throws IOException { - RegionLocations rls = MetaTableAccessor.getRegionLocations(result); + RegionLocations rls = CatalogFamilyFormat.getRegionLocations(result); if (rls == null) { return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/webapp/RegionReplicaInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/webapp/RegionReplicaInfo.java index 3e47d2ad2dd..fda3a70ae4f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/webapp/RegionReplicaInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/webapp/RegionReplicaInfo.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -89,7 +90,7 @@ public final class RegionReplicaInfo { return Collections.singletonList(null); } - final RegionLocations locations = MetaTableAccessor.getRegionLocations(result); + final RegionLocations locations = CatalogFamilyFormat.getRegionLocations(result); if (locations == null) { return Collections.singletonList(null); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index adb9415dbd4..c42ec703b62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -56,7 +56,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -68,8 +67,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -2052,12 +2053,12 @@ public class HBaseFsck extends Configured implements Closeable { if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication(); for (int i = 0; i < numReplicas; i++) { - get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i)); - get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i)); + get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i)); + get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i)); } } Result r = meta.get(get); - RegionLocations rl = MetaTableAccessor.getRegionLocations(r); + RegionLocations rl = CatalogFamilyFormat.getRegionLocations(r); if (rl == null) { LOG.warn("Unable to close region " + hi.getRegionNameAsString() + " since meta does not have handle to reach it"); @@ -2734,7 +2735,7 @@ public class HBaseFsck extends Configured implements Closeable { * @throws IOException if an error is encountered */ boolean loadMetaEntries() throws IOException { - MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { int countRecord = 1; // comparator to sort KeyValues with latest modtime @@ -2751,7 +2752,7 @@ public class HBaseFsck extends Configured implements Closeable { // record the latest modification of this META record long ts = Collections.max(result.listCells(), comp).getTimestamp(); - RegionLocations rl = MetaTableAccessor.getRegionLocations(result); + RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result); if (rl == null) { emptyRegionInfoQualifiers.add(result); errors.reportError(ERROR_CODE.EMPTY_META_CELL, @@ -3896,10 +3897,10 @@ public class HBaseFsck extends Configured implements Closeable { barrierScan.setCaching(100); barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY); barrierScan - .withStartRow(MetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable, - MetaTableAccessor.QueryType.REGION)) - .withStopRow(MetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable, - MetaTableAccessor.QueryType.REGION)); + .withStartRow(ClientMetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable, + ClientMetaTableAccessor.QueryType.REGION)) + .withStopRow(ClientMetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable, + ClientMetaTableAccessor.QueryType.REGION)); Result result; try (ResultScanner scanner = meta.getScanner(barrierScan)) { while ((result = scanner.next()) != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index e90b9e2aea3..157d2fde74b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2643,7 +2643,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { List rows = new ArrayList<>(); ResultScanner s = t.getScanner(new Scan()); for (Result result : s) { - RegionInfo info = MetaTableAccessor.getRegionInfo(result); + RegionInfo info = CatalogFamilyFormat.getRegionInfo(result); if (info == null) { LOG.error("No region info for row " + Bytes.toString(result.getRow())); // TODO figure out what to do for this new hosed case. @@ -3268,13 +3268,13 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { @Nullable public TableState findLastTableState(final TableName table) throws IOException { final AtomicReference lastTableState = new AtomicReference<>(null); - MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result r) throws IOException { if (!Arrays.equals(r.getRow(), table.getName())) { return false; } - TableState state = MetaTableAccessor.getTableState(r); + TableState state = CatalogFamilyFormat.getTableState(r); if (state != null) { lastTableState.set(state); } @@ -3282,7 +3282,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } }; MetaTableAccessor.scanMeta(asyncConnection.toConnection(), null, null, - MetaTableAccessor.QueryType.TABLE, Integer.MAX_VALUE, visitor); + ClientMetaTableAccessor.QueryType.TABLE, Integer.MAX_VALUE, visitor); return lastTableState.get(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index afc703eab9a..112fa01d376 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -38,7 +37,6 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; @@ -82,33 +80,35 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Sets; /** * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}. */ -@Category({MiscTests.class, MediumTests.class}) +@Category({ MiscTests.class, MediumTests.class }) @SuppressWarnings("deprecation") public class TestMetaTableAccessor { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetaTableAccessor.class); + HBaseClassTestRule.forClass(TestMetaTableAccessor.class); private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableAccessor.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static Connection connection; private Random random = new Random(); @Rule public TestName name = new TestName(); - @BeforeClass public static void beforeClass() throws Exception { + @BeforeClass + public static void beforeClass() throws Exception { UTIL.startMiniCluster(3); Configuration c = new Configuration(UTIL.getConfiguration()); // Tests to 4 retries every 5 seconds. Make it try every 1 second so more - // responsive. 1 second is default as is ten retries. + // responsive. 1 second is default as is ten retries. c.setLong("hbase.client.pause", 1000); c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10); connection = ConnectionFactory.createConnection(c); } - @AfterClass public static void afterClass() throws Exception { + @AfterClass + public static void afterClass() throws Exception { connection.close(); UTIL.shutdownMiniCluster(); } @@ -125,9 +125,9 @@ public class TestMetaTableAccessor { List regions = admin.getRegions(tn); assertEquals(4, regions.size()); admin.mergeRegionsAsync(regions.get(0).getRegionName(), regions.get(1).getRegionName(), false) - .get(60, TimeUnit.SECONDS); + .get(60, TimeUnit.SECONDS); admin.mergeRegionsAsync(regions.get(2).getRegionName(), regions.get(3).getRegionName(), false) - .get(60, TimeUnit.SECONDS); + .get(60, TimeUnit.SECONDS); List mergedRegions = admin.getRegions(tn); assertEquals(2, mergedRegions.size()); @@ -135,7 +135,7 @@ public class TestMetaTableAccessor { RegionInfo mergedRegion1 = mergedRegions.get(1); List mergeParents = - MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName()); + MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName()); assertTrue(mergeParents.contains(regions.get(0))); assertTrue(mergeParents.contains(regions.get(1))); mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion1.getRegionName()); @@ -160,17 +160,17 @@ public class TestMetaTableAccessor { Put put = new Put(Bytes.toBytes(this.name.getMethodName())); List ris = new ArrayList<>(); int limit = 10; - byte [] previous = HConstants.EMPTY_START_ROW; + byte[] previous = HConstants.EMPTY_START_ROW; for (int i = 0; i < limit; i++) { - RegionInfo ri = RegionInfoBuilder.newBuilder(tn). - setStartKey(previous).setEndKey(Bytes.toBytes(i)).build(); + RegionInfo ri = + RegionInfoBuilder.newBuilder(tn).setStartKey(previous).setEndKey(Bytes.toBytes(i)).build(); ris.add(ri); } put = MetaTableAccessor.addMergeRegions(put, ris); List cells = put.getFamilyCellMap().get(HConstants.CATALOG_FAMILY); String previousQualifier = null; assertEquals(limit, cells.size()); - for (Cell cell: cells) { + for (Cell cell : cells) { LOG.info(cell.toString()); String qualifier = Bytes.toString(cell.getQualifierArray()); assertTrue(qualifier.startsWith(HConstants.MERGE_QUALIFIER_PREFIX_STR)); @@ -195,12 +195,11 @@ public class TestMetaTableAccessor { } /** - * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write - * against hbase:meta while its hosted server is restarted to prove our retrying - * works. + * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write against hbase:meta + * while its hosted server is restarted to prove our retrying works. */ - @Test public void testRetrying() - throws IOException, InterruptedException { + @Test + public void testRetrying() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); LOG.info("Started " + tableName); Table t = UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY); @@ -209,8 +208,7 @@ public class TestMetaTableAccessor { regionCount = r.getStartKeys().length; } // Test it works getting a region from just made user table. - final List regions = - testGettingTableRegions(connection, tableName, regionCount); + final List regions = testGettingTableRegions(connection, tableName, regionCount); MetaTask reader = new MetaTask(connection, "reader") { @Override void metaTask() throws Throwable { @@ -229,8 +227,8 @@ public class TestMetaTableAccessor { writer.start(); // We're gonna check how it takes. If it takes too long, we will consider - // it as a fail. We can't put that in the @Test tag as we want to close - // the threads nicely + // it as a fail. We can't put that in the @Test tag as we want to close + // the threads nicely final long timeOut = 180000; long startTime = System.currentTimeMillis(); @@ -239,18 +237,17 @@ public class TestMetaTableAccessor { assertTrue(reader.isProgressing()); assertTrue(writer.isProgressing()); - // Kill server hosting meta -- twice . See if our reader/writer ride over the - // meta moves. They'll need to retry. + // Kill server hosting meta -- twice . See if our reader/writer ride over the + // meta moves. They'll need to retry. for (int i = 0; i < 2; i++) { LOG.info("Restart=" + i); UTIL.ensureSomeRegionServersAvailable(2); int index = -1; do { index = UTIL.getMiniHBaseCluster().getServerWithMeta(); - } while (index == -1 && - startTime + timeOut < System.currentTimeMillis()); + } while (index == -1 && startTime + timeOut < System.currentTimeMillis()); - if (index != -1){ + if (index != -1) { UTIL.getMiniHBaseCluster().abortRegionServer(index); UTIL.getMiniHBaseCluster().waitOnRegionServer(index); } @@ -288,8 +285,8 @@ public class TestMetaTableAccessor { @Override public void run() { try { - while(!this.stop) { - LOG.info("Before " + this.getName()+ ", count=" + this.count); + while (!this.stop) { + LOG.info("Before " + this.getName() + ", count=" + this.count); metaTask(); this.count += 1; LOG.info("After " + this.getName() + ", count=" + this.count); @@ -303,7 +300,7 @@ public class TestMetaTableAccessor { boolean isProgressing() throws InterruptedException { int currentCount = this.count; - while(currentCount == this.count) { + while (currentCount == this.count) { if (!isAlive()) return false; if (this.t != null) return false; Thread.sleep(10); @@ -313,8 +310,7 @@ public class TestMetaTableAccessor { @Override public String toString() { - return "count=" + this.count + ", t=" + - (this.t == null? "null": this.t.toString()); + return "count=" + this.count + ", t=" + (this.t == null ? "null" : this.t.toString()); } abstract void metaTask() throws Throwable; @@ -324,11 +320,11 @@ public class TestMetaTableAccessor { public void testGetRegionsFromMetaTable() throws IOException, InterruptedException { List regions = MetaTableLocator.getMetaRegions(UTIL.getZooKeeperWatcher()); assertTrue(regions.size() >= 1); - assertTrue( - MetaTableLocator.getMetaRegionsAndLocations(UTIL.getZooKeeperWatcher()).size() >= 1); + assertTrue(MetaTableLocator.getMetaRegionsAndLocations(UTIL.getZooKeeperWatcher()).size() >= 1); } - @Test public void testTableExists() throws IOException { + @Test + public void testTableExists() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); assertFalse(MetaTableAccessor.tableExists(connection, tableName)); UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); @@ -337,8 +333,7 @@ public class TestMetaTableAccessor { admin.disableTable(tableName); admin.deleteTable(tableName); assertFalse(MetaTableAccessor.tableExists(connection, tableName)); - assertTrue(MetaTableAccessor.tableExists(connection, - TableName.META_TABLE_NAME)); + assertTrue(MetaTableAccessor.tableExists(connection, TableName.META_TABLE_NAME)); UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); assertTrue(MetaTableAccessor.tableExists(connection, tableName)); admin.disableTable(tableName); @@ -346,7 +341,8 @@ public class TestMetaTableAccessor { assertFalse(MetaTableAccessor.tableExists(connection, tableName)); } - @Test public void testGetRegion() throws IOException, InterruptedException { + @Test + public void testGetRegion() throws IOException, InterruptedException { final String name = this.name.getMethodName(); LOG.info("Started " + name); // Test get on non-existent region. @@ -357,20 +353,18 @@ public class TestMetaTableAccessor { } // Test for the optimization made in HBASE-3650 - @Test public void testScanMetaForTable() - throws IOException, InterruptedException { + @Test + public void testScanMetaForTable() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); LOG.info("Started " + tableName); - /** Create 2 tables - - testScanMetaForTable - - testScanMetaForTablf - **/ + /** + * Create 2 tables - testScanMetaForTable - testScanMetaForTablf + **/ UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); // name that is +1 greater than the first one (e+1=f) - TableName greaterName = - TableName.valueOf("testScanMetaForTablf"); + TableName greaterName = TableName.valueOf("testScanMetaForTablf"); UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY); // Now make sure we only get the regions from 1 of the tables at a time @@ -380,66 +374,20 @@ public class TestMetaTableAccessor { } private static List testGettingTableRegions(final Connection connection, - final TableName name, final int regionCount) - throws IOException, InterruptedException { + final TableName name, final int regionCount) throws IOException, InterruptedException { List regions = MetaTableAccessor.getTableRegions(connection, name); assertEquals(regionCount, regions.size()); Pair pair = MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName()); - assertEquals(regions.get(0).getEncodedName(), - pair.getFirst().getEncodedName()); + assertEquals(regions.get(0).getEncodedName(), pair.getFirst().getEncodedName()); return regions; } - private static void testGetRegion(final Connection connection, - final RegionInfo region) - throws IOException, InterruptedException { + private static void testGetRegion(final Connection connection, final RegionInfo region) + throws IOException, InterruptedException { Pair pair = MetaTableAccessor.getRegion(connection, region.getRegionName()); - assertEquals(region.getEncodedName(), - pair.getFirst().getEncodedName()); - } - - @Test - public void testParseReplicaIdFromServerColumn() { - String column1 = HConstants.SERVER_QUALIFIER_STR; - assertEquals(0, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column1))); - String column2 = column1 + MetaTableAccessor.META_REPLICA_ID_DELIMITER; - assertEquals(-1, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column2))); - String column3 = column2 + "00"; - assertEquals(-1, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column3))); - String column4 = column3 + "2A"; - assertEquals(42, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column4))); - String column5 = column4 + "2A"; - assertEquals(-1, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column5))); - String column6 = HConstants.STARTCODE_QUALIFIER_STR; - assertEquals(-1, - MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column6))); - } - - @Test - public void testMetaReaderGetColumnMethods() { - assertArrayEquals(HConstants.SERVER_QUALIFIER, MetaTableAccessor.getServerColumn(0)); - assertArrayEquals(Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR - + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"), - MetaTableAccessor.getServerColumn(42)); - - assertArrayEquals(HConstants.STARTCODE_QUALIFIER, - MetaTableAccessor.getStartCodeColumn(0)); - assertArrayEquals(Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR - + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"), - MetaTableAccessor.getStartCodeColumn(42)); - - assertArrayEquals(HConstants.SEQNUM_QUALIFIER, - MetaTableAccessor.getSeqNumColumn(0)); - assertArrayEquals(Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR - + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"), - MetaTableAccessor.getSeqNumColumn(42)); + assertEquals(region.getEncodedName(), pair.getFirst().getEncodedName()); } @Test @@ -450,26 +398,14 @@ public class TestMetaTableAccessor { long regionId = System.currentTimeMillis(); RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo replica1 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(1) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(1).build(); RegionInfo replica100 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(100) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(100).build(); long seqNum0 = random.nextLong(); long seqNum1 = random.nextLong(); @@ -502,31 +438,31 @@ public class TestMetaTableAccessor { } } - public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName, - long seqNum, int replicaId, boolean checkSeqNum) throws IOException { + public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName, long seqNum, + int replicaId, boolean checkSeqNum) throws IOException { Get get = new Get(row); Result result = meta.get(get); assertTrue(Bytes.equals( - result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(replicaId)), + result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(replicaId)), Bytes.toBytes(serverName.getAddress().toString()))); assertTrue(Bytes.equals( - result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(replicaId)), + result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(replicaId)), Bytes.toBytes(serverName.getStartcode()))); if (checkSeqNum) { assertTrue(Bytes.equals( - result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(replicaId)), + result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(replicaId)), Bytes.toBytes(seqNum))); } } public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId) - throws IOException { + throws IOException { Get get = new Get(row); Result result = meta.get(get); Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(replicaId)); + CatalogFamilyFormat.getServerColumn(replicaId)); Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(replicaId)); + CatalogFamilyFormat.getStartCodeColumn(replicaId)); assertNotNull(serverCell); assertNotNull(startCodeCell); assertEquals(0, serverCell.getValueLength()); @@ -537,8 +473,8 @@ public class TestMetaTableAccessor { public void testMetaLocationForRegionReplicasIsRemovedAtTableDeletion() throws IOException { long regionId = System.currentTimeMillis(); RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) - .setRegionId(regionId).setReplicaId(0).build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); Table meta = MetaTableAccessor.getMetaHTable(connection); try { @@ -550,13 +486,13 @@ public class TestMetaTableAccessor { Result result = meta.get(get); for (int replicaId = 0; replicaId < 3; replicaId++) { Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(replicaId)); + CatalogFamilyFormat.getServerColumn(replicaId)); Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(replicaId)); + CatalogFamilyFormat.getStartCodeColumn(replicaId)); Cell stateCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getRegionStateColumn(replicaId)); + CatalogFamilyFormat.getRegionStateColumn(replicaId)); Cell snCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerNameColumn(replicaId)); + CatalogFamilyFormat.getServerNameColumn(replicaId)); if (replicaId == 0) { assertNotNull(stateCell); } else { @@ -575,12 +511,8 @@ public class TestMetaTableAccessor { public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException { long regionId = System.currentTimeMillis(); RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); Table meta = MetaTableAccessor.getMetaHTable(connection); try { @@ -599,27 +531,15 @@ public class TestMetaTableAccessor { long regionId = System.currentTimeMillis(); ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(Bytes.toBytes("a")) - .setSplit(false) - .setRegionId(regionId + 1) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false) + .setRegionId(regionId + 1).setReplicaId(0).build(); RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(Bytes.toBytes("a")) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId + 1) - .setReplicaId(0) - .build(); + .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId + 1).setReplicaId(0).build(); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) { List regionInfos = Lists.newArrayList(parent); @@ -640,41 +560,29 @@ public class TestMetaTableAccessor { ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); RegionInfo parentA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(Bytes.toBytes("a")) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo parentB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(Bytes.toBytes("a")) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo merged = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId + 1) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId + 1).setReplicaId(0).build(); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) { List regionInfos = Lists.newArrayList(parentA, parentB); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); MetaTableAccessor.mergeRegions(connection, merged, getMapOfRegionsToSeqNum(parentA, parentB), - serverName0, 3); + serverName0, 3); assertEmptyMetaLocation(meta, merged.getRegionName(), 1); assertEmptyMetaLocation(meta, merged.getRegionName(), 2); } } - private Map getMapOfRegionsToSeqNum(RegionInfo ... regions) { + private Map getMapOfRegionsToSeqNum(RegionInfo... regions) { Map mids = new HashMap<>(regions.length); - for (RegionInfo region: regions) { + for (RegionInfo region : regions) { mids.put(region, -1L); } return mids; @@ -687,15 +595,14 @@ public class TestMetaTableAccessor { final TableName tableName = TableName.valueOf(name.getMethodName()); final byte[] FAMILY = Bytes.toBytes("family"); final byte[][] SPLIT_KEYS = - new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") }; + new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") }; UTIL.createTable(tableName, FAMILY, SPLIT_KEYS); Table table = connection.getTable(tableName); // Make sure all the regions are deployed UTIL.countRows(table); - MetaTableAccessor.Visitor visitor = - mock(MetaTableAccessor.Visitor.class); + ClientMetaTableAccessor.Visitor visitor = mock(ClientMetaTableAccessor.Visitor.class); doReturn(true).when(visitor).visit((Result) anyObject()); // Scanning the entire table should give us three rows @@ -731,12 +638,8 @@ public class TestMetaTableAccessor { public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException { long regionId = System.currentTimeMillis(); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); ServerName sn = ServerName.valueOf("bar", 0, 0); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) { @@ -749,11 +652,11 @@ public class TestMetaTableAccessor { Get get = new Get(regionInfo.getRegionName()); Result result = meta.get(get); Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(0)); + CatalogFamilyFormat.getServerColumn(0)); Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(0)); + CatalogFamilyFormat.getStartCodeColumn(0)); Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getSeqNumColumn(0)); + CatalogFamilyFormat.getSeqNumColumn(0)); assertNotNull(serverCell); assertNotNull(startCodeCell); assertNotNull(seqNumCell); @@ -771,27 +674,16 @@ public class TestMetaTableAccessor { long regionId = System.currentTimeMillis(); RegionInfo regionInfoA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(new byte[] {'a'}) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(new byte[] { 'a' }).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo regionInfoB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(new byte[] {'a'}) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); - RegionInfo mergedRegionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(new byte[] { 'a' }).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); + RegionInfo mergedRegionInfo = + RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); ServerName sn = ServerName.valueOf("bar", 0, 0); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) { @@ -801,7 +693,7 @@ public class TestMetaTableAccessor { // write the serverName column with a big current time, but set the masters time as even // bigger. When region merge deletes the rows for regionA and regionB, the serverName columns // should not be seen by the following get - long serverNameTime = EnvironmentEdgeManager.currentTime() + 100000000; + long serverNameTime = EnvironmentEdgeManager.currentTime() + 100000000; long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789; // write the serverName columns @@ -811,7 +703,7 @@ public class TestMetaTableAccessor { Get get = new Get(mergedRegionInfo.getRegionName()); Result result = meta.get(get); Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(0)); + CatalogFamilyFormat.getServerColumn(0)); assertNotNull(serverCell); assertEquals(serverNameTime, serverCell.getTimestamp()); @@ -821,19 +713,18 @@ public class TestMetaTableAccessor { try { // now merge the regions, effectively deleting the rows for region a and b. MetaTableAccessor.mergeRegions(connection, mergedRegionInfo, - getMapOfRegionsToSeqNum(regionInfoA, regionInfoB), sn, 1); + getMapOfRegionsToSeqNum(regionInfoA, regionInfoB), sn, 1); } finally { EnvironmentEdgeManager.reset(); } - result = meta.get(get); serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(0)); + CatalogFamilyFormat.getServerColumn(0)); Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(0)); + CatalogFamilyFormat.getStartCodeColumn(0)); Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getSeqNumColumn(0)); + CatalogFamilyFormat.getSeqNumColumn(0)); assertNull(serverCell); assertNull(startCodeCell); assertNull(seqNumCell); @@ -880,7 +771,7 @@ public class TestMetaTableAccessor { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Admin admin = connection.getAdmin(); - RegionLocator rl = connection.getRegionLocator(tableName)) { + RegionLocator rl = connection.getRegionLocator(tableName)) { // create a table and prepare for a manual split UTIL.createTable(tableName, "cf1"); @@ -889,18 +780,11 @@ public class TestMetaTableAccessor { RegionInfo parent = loc.getRegion(); long rid = 1000; byte[] splitKey = Bytes.toBytes("a"); - RegionInfo splitA = RegionInfoBuilder.newBuilder(parent.getTable()) - .setStartKey(parent.getStartKey()) - .setEndKey(splitKey) - .setSplit(false) - .setRegionId(rid) - .build(); - RegionInfo splitB = RegionInfoBuilder.newBuilder(parent.getTable()) - .setStartKey(splitKey) - .setEndKey(parent.getEndKey()) - .setSplit(false) - .setRegionId(rid) - .build(); + RegionInfo splitA = + RegionInfoBuilder.newBuilder(parent.getTable()).setStartKey(parent.getStartKey()) + .setEndKey(splitKey).setSplit(false).setRegionId(rid).build(); + RegionInfo splitB = RegionInfoBuilder.newBuilder(parent.getTable()).setStartKey(splitKey) + .setEndKey(parent.getEndKey()).setSplit(false).setRegionId(rid).build(); // find the meta server MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster(); @@ -926,49 +810,36 @@ public class TestMetaTableAccessor { long regionId = System.currentTimeMillis(); ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo")) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId).setReplicaId(0).build(); RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo")) - .setStartKey(HConstants.EMPTY_START_ROW) - .setEndKey(Bytes.toBytes("a")) - .setSplit(false) - .setRegionId(regionId + 1) - .setReplicaId(0) - .build(); + .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false) + .setRegionId(regionId + 1).setReplicaId(0).build(); RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo")) - .setStartKey(Bytes.toBytes("a")) - .setEndKey(HConstants.EMPTY_END_ROW) - .setSplit(false) - .setRegionId(regionId + 1) - .setReplicaId(0) - .build(); + .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false) + .setRegionId(regionId + 1).setReplicaId(0).build(); Table meta = MetaTableAccessor.getMetaHTable(connection); try { List regionInfos = Lists.newArrayList(parent); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); - MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, - serverName0, 3); + MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, serverName0, 3); Get get1 = new Get(splitA.getRegionName()); Result resultA = meta.get(get1); Cell serverCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(splitA.getReplicaId())); + CatalogFamilyFormat.getServerColumn(splitA.getReplicaId())); Cell startCodeCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(splitA.getReplicaId())); + CatalogFamilyFormat.getStartCodeColumn(splitA.getReplicaId())); assertNull(serverCellA); assertNull(startCodeCellA); Get get2 = new Get(splitA.getRegionName()); Result resultB = meta.get(get2); Cell serverCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(splitB.getReplicaId())); + CatalogFamilyFormat.getServerColumn(splitB.getReplicaId())); Cell startCodeCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(splitB.getReplicaId())); + CatalogFamilyFormat.getStartCodeColumn(splitB.getReplicaId())); assertNull(serverCellB); assertNull(startCodeCellB); } finally { @@ -984,8 +855,8 @@ public class TestMetaTableAccessor { UTIL.createTable(tableName, "cf"); final List regions = UTIL.getHBaseCluster().getRegions(tableName); final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); - final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), - encodedName); + final Result result = + MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); assertNotNull(result); assertTrue(result.advance()); final String resultingRowKey = CellUtil.getCellKeyAsString(result.current()); @@ -996,9 +867,8 @@ public class TestMetaTableAccessor { @Test public void testScanByRegionEncodedNameNonExistingRegion() throws Exception { final String encodedName = "nonexistingregion"; - final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), - encodedName); + final Result result = + MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); assertNull(result); } } - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java index ef73b20850b..1cbde2fa49c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java @@ -31,7 +31,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -85,7 +85,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); List regionLocations = - AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); + ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); int originalCount = regionLocations.size(); initSplitMergeSwitch(); @@ -116,7 +116,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); List regionLocations = - AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); + ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); int originalCount = regionLocations.size(); initSplitMergeSwitch(); @@ -160,7 +160,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { createTableWithDefaultConf(tableName, splitRows); AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); - List regionLocations = AsyncMetaTableAccessor + List regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); RegionInfo regionA; RegionInfo regionB; @@ -174,7 +174,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { regionC = regionLocations.get(2).getRegion(); admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get(); - regionLocations = AsyncMetaTableAccessor + regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); assertEquals(2, regionLocations.size()); @@ -198,7 +198,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { admin.mergeRegions(regionC.getRegionName(), mergedChildRegion.getRegionName(), false).get(); - regionLocations = AsyncMetaTableAccessor + regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); assertEquals(1, regionLocations.size()); } @@ -243,7 +243,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { createTableWithDefaultConf(tableName); AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); - List regionLocations = AsyncMetaTableAccessor + List regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); assertEquals(1, regionLocations.size()); @@ -273,7 +273,7 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { int count = 0; for (int i = 0; i < 45; i++) { try { - regionLocations = AsyncMetaTableAccessor + regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); count = regionLocations.size(); if (count >= 2) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 5b03591d69f..52ebc168674 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletionException; -import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -74,7 +74,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { } static TableState.State getStateFromMeta(TableName table) throws Exception { - Optional state = AsyncMetaTableAccessor + Optional state = ClientMetaTableAccessor .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get(); assertTrue(state.isPresent()); return state.get().getState(); @@ -85,20 +85,20 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); createTableWithDefaultConf(tableName); - List regionLocations = AsyncMetaTableAccessor + List regionLocations = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); assertEquals("Table should have only 1 region", 1, regionLocations.size()); final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2"); createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } }); - regionLocations = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get(); + regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get(); assertEquals("Table should have only 2 region", 2, regionLocations.size()); final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3"); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 3).join(); - regionLocations = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get(); + regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get(); assertEquals("Table should have only 3 region", 3, regionLocations.size()); final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4"); @@ -115,7 +115,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { builder = TableDescriptorBuilder.newBuilder(tableName5); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join(); - regionLocations = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get(); + regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get(); assertEquals("Table should have 16 region", 16, regionLocations.size()); } @@ -132,7 +132,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); - List regions = AsyncMetaTableAccessor + List regions = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); Iterator hris = regions.iterator(); @@ -191,7 +191,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); admin.createTable(builder.build(), startKey, endKey, expectedRegions).join(); - regions = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get(); + regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); @@ -243,7 +243,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); admin.createTable(builder.build(), startKey, endKey, expectedRegions).join(); - regions = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3) + regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3) .get(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index 344beace42a..6ba960bef91 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -30,7 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; -import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; @@ -283,7 +283,7 @@ public class TestAsyncTableAdminApi3 extends TestAsyncAdminBase { createTableWithDefaultConf(tableName, splitKeys); AsyncTable metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); - List regions = AsyncMetaTableAccessor + List regions = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), @@ -294,7 +294,7 @@ public class TestAsyncTableAdminApi3 extends TestAsyncAdminBase { // Enable table, use retain assignment to assign regions. admin.enableTable(tableName).join(); - List regions2 = AsyncMetaTableAccessor + List regions2 = ClientMetaTableAccessor .getTableHRegionLocations(metaTable, tableName).get(); // Check the assignment. assertEquals(regions.size(), regions2.size()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaTableAccessorNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaTableAccessorNoCluster.java index dd1965d3c5b..e160fb313d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaTableAccessorNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaTableAccessorNoCluster.java @@ -24,12 +24,12 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.After; @@ -63,26 +63,26 @@ public class TestMetaTableAccessorNoCluster { @Test public void testGetHRegionInfo() throws IOException { - assertNull(MetaTableAccessor.getRegionInfo(new Result())); + assertNull(CatalogFamilyFormat.getRegionInfo(new Result())); List kvs = new ArrayList<>(); Result r = Result.create(kvs); - assertNull(MetaTableAccessor.getRegionInfo(r)); + assertNull(CatalogFamilyFormat.getRegionInfo(r)); byte[] f = HConstants.CATALOG_FAMILY; // Make a key value that doesn't have the expected qualifier. kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, HConstants.SERVER_QUALIFIER, f)); r = Result.create(kvs); - assertNull(MetaTableAccessor.getRegionInfo(r)); + assertNull(CatalogFamilyFormat.getRegionInfo(r)); // Make a key that does not have a regioninfo value. kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, HConstants.REGIONINFO_QUALIFIER, f)); - RegionInfo hri = MetaTableAccessor.getRegionInfo(Result.create(kvs)); + RegionInfo hri = CatalogFamilyFormat.getRegionInfo(Result.create(kvs)); assertTrue(hri == null); // OK, give it what it expects kvs.clear(); kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, HConstants.REGIONINFO_QUALIFIER, RegionInfo.toByteArray(RegionInfoBuilder.FIRST_META_REGIONINFO))); - hri = MetaTableAccessor.getRegionInfo(Result.create(kvs)); + hri = CatalogFamilyFormat.getRegionInfo(Result.create(kvs)); assertNotNull(hri); assertTrue(RegionInfo.COMPARATOR.compare(hri, RegionInfoBuilder.FIRST_META_REGIONINFO) == 0); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorCluster.java index 83e74e88be3..f0291e4eb31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorCluster.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Arrays; import java.util.List; - +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -145,8 +145,8 @@ public class TestCatalogJanitorCluster { List t1Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T1); RegionInfo t1Ri1 = t1Ris.get(1); Put pServer = new Put(t1Ri1.getRegionName()); - pServer.addColumn(MetaTableAccessor.getCatalogFamily(), - MetaTableAccessor.getServerColumn(0), Bytes.toBytes("bad.server.example.org:1234")); + pServer.addColumn(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getServerColumn(0), Bytes.toBytes("bad.server.example.org:1234")); MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pServer)); gc = janitor.scan(); report = janitor.getLastReport(); @@ -157,8 +157,8 @@ public class TestCatalogJanitorCluster { // break if this happens. LOG.info("Make null info:server"); Put emptyInfoServerPut = new Put(t1Ri1.getRegionName()); - emptyInfoServerPut.addColumn(MetaTableAccessor.getCatalogFamily(), - MetaTableAccessor.getServerColumn(0), Bytes.toBytes("")); + emptyInfoServerPut.addColumn(HConstants.CATALOG_FAMILY, + CatalogFamilyFormat.getServerColumn(0), Bytes.toBytes("")); MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(emptyInfoServerPut)); janitor.scan(); report = janitor.getLastReport(); @@ -166,8 +166,8 @@ public class TestCatalogJanitorCluster { // Mke an empty regioninfo in t1. RegionInfo t1Ri2 = t1Ris.get(2); Put pEmptyRI = new Put(t1Ri2.getRegionName()); - pEmptyRI.addColumn(MetaTableAccessor.getCatalogFamily(), - MetaTableAccessor.getRegionInfoColumn(), HConstants.EMPTY_BYTE_ARRAY); + pEmptyRI.addColumn(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pEmptyRI)); janitor.scan(); report = janitor.getLastReport(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java index 77cf97d1fd9..b5b5e63ccd8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionLocation; @@ -174,7 +175,7 @@ public class TestCatalogJanitorInMemoryStates { if (result == null) { break; } - region = MetaTableAccessor.getRegionInfo(result); + region = CatalogFamilyFormat.getRegionInfo(result); if (region.isSplitParent()) { LOG.debug(region.toString() + " IS a parent!"); pair = MetaTableAccessor.getDaughterRegions(result); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java index bdcc602dfe5..e653234cb97 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java @@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -106,14 +108,14 @@ public class TestMaster { byte[] rowKey) throws IOException { final AtomicReference> result = new AtomicReference<>(null); - MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result data) throws IOException { if (data == null || data.size() <= 0) { return true; } - Pair pair = new Pair<>(MetaTableAccessor.getRegionInfo(data), - MetaTableAccessor.getServerName(data, 0)); + Pair pair = new Pair<>(CatalogFamilyFormat.getRegionInfo(data), + CatalogFamilyFormat.getServerName(data, 0)); if (!pair.getFirst().getTable().equals(tableName)) { return false; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java index bb2fbfad3dc..01d482c095f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java @@ -33,16 +33,16 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.MetaTableAccessor.Visitor; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.StartMiniClusterOption; @@ -162,7 +162,7 @@ public class TestMasterOperationsForRegionReplicas { List metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection()); int numRows = 0; for (Result result : metaRows) { - RegionLocations locations = MetaTableAccessor.getRegionLocations(result); + RegionLocations locations = CatalogFamilyFormat.getRegionLocations(result); RegionInfo hri = locations.getRegionLocation().getRegion(); if (!hri.getTable().equals(tableName)) continue; numRows += 1; @@ -310,11 +310,11 @@ public class TestMasterOperationsForRegionReplicas { for (byte[] row : tableRows) { Delete deleteOneReplicaLocation = new Delete(row); deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getServerColumn(1)); + CatalogFamilyFormat.getServerColumn(1)); deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getSeqNumColumn(1)); + CatalogFamilyFormat.getSeqNumColumn(1)); deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, - MetaTableAccessor.getStartCodeColumn(1)); + CatalogFamilyFormat.getStartCodeColumn(1)); metaTable.delete(deleteOneReplicaLocation); } metaTable.close(); @@ -337,10 +337,12 @@ public class TestMasterOperationsForRegionReplicas { Connection connection) throws IOException { assert (ADMIN.tableExists(table)); final AtomicInteger count = new AtomicInteger(); - Visitor visitor = new Visitor() { + ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result r) throws IOException { - if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet(); + if (CatalogFamilyFormat.getRegionInfo(r).getTable().equals(table)) { + count.incrementAndGet(); + } return true; } }; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java index 77c5c58a77d..d13b36b90fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java @@ -18,10 +18,10 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; @@ -498,7 +498,7 @@ public class TestMasterTransitions { scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); ResultScanner s = meta.getScanner(scan); for (Result r = null; (r = s.next()) != null;) { - RegionInfo hri = MetaTableAccessor.getRegionInfo(r); + RegionInfo hri = CatalogFamilyFormat.getRegionInfo(r); if (hri == null) break; if (!hri.getTable().equals(TABLENAME)) { continue; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java index f03e99a2bc8..e9e4a391cf1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -473,12 +475,12 @@ public class TestRegionPlacement { final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0); final AtomicInteger totalRegionNum = new AtomicInteger(0); LOG.info("The start of region placement verification"); - MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result result) throws IOException { try { @SuppressWarnings("deprecation") - RegionInfo info = MetaTableAccessor.getRegionInfo(result); + RegionInfo info = CatalogFamilyFormat.getRegionInfo(result); if(info.getTable().getNamespaceAsString() .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { return true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index ea409b00868..0cc625fd815 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -243,10 +245,10 @@ public class MasterProcedureTestingUtility { private static int countMetaRegions(final HMaster master, final TableName tableName) throws IOException { final AtomicInteger actualRegCount = new AtomicInteger(0); - final MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + final ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() { @Override public boolean visit(Result rowResult) throws IOException { - RegionLocations list = MetaTableAccessor.getRegionLocations(rowResult); + RegionLocations list = CatalogFamilyFormat.getRegionLocations(rowResult); if (list == null) { LOG.warn("No serialized RegionInfo in " + rowResult); return true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java index a4d251f64f5..106936dd524 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.List; - +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -103,7 +103,7 @@ public class TestHBCKSCP extends TestSCPBase { // Assert region is OPEN. assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER))); - ServerName serverName = MetaTableAccessor.getServerName(r, 0); + ServerName serverName = CatalogFamilyFormat.getServerName(r, 0); assertEquals(rsServerName, serverName); // moveFrom adds to dead servers and adds it to processing list only we will // not be processing this server 'normally'. Remove it from processing by @@ -129,7 +129,7 @@ public class TestHBCKSCP extends TestSCPBase { r = MetaTableAccessor.getRegionResult(master.getConnection(), rsRI.getRegionName()); assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER))); - serverName = MetaTableAccessor.getServerName(r, 0); + serverName = CatalogFamilyFormat.getServerName(r, 0); assertNotNull(cluster.getRegionServer(serverName)); assertEquals(rsServerName, serverName); @@ -149,7 +149,7 @@ public class TestHBCKSCP extends TestSCPBase { r = MetaTableAccessor.getRegionResult(master.getConnection(), rsRI.getRegionName()); assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER))); - serverName = MetaTableAccessor.getServerName(r, 0); + serverName = CatalogFamilyFormat.getServerName(r, 0); assertNotNull(cluster.getRegionServer(serverName)); assertNotEquals(rsServerName, serverName); // Make sure no mention of old server post SCP. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 84a1d73cf32..c35703805af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -31,6 +31,7 @@ import java.util.TreeSet; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -443,7 +444,7 @@ public class TestEndToEndSplitTransaction { break; } - region = MetaTableAccessor.getRegionInfo(result); + region = CatalogFamilyFormat.getRegionInfo(result); if (region.isSplitParent()) { log("found parent region: " + region.toString()); PairOfSameType pair = MetaTableAccessor.getDaughterRegions(result); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java index 21ad20c0568..8c218d6cee8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; import java.util.List; - +import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -113,10 +113,10 @@ public class TestHBaseFsckCleanReplicationBarriers { barrierScan.setCaching(100); barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY); barrierScan - .withStartRow( - MetaTableAccessor.getTableStartRowForMeta(tableName, MetaTableAccessor.QueryType.REGION)) - .withStopRow( - MetaTableAccessor.getTableStopRowForMeta(tableName, MetaTableAccessor.QueryType.REGION)); + .withStartRow(ClientMetaTableAccessor.getTableStartRowForMeta(tableName, + ClientMetaTableAccessor.QueryType.REGION)) + .withStopRow(ClientMetaTableAccessor.getTableStopRowForMeta(tableName, + ClientMetaTableAccessor.QueryType.REGION)); Result result; try (ResultScanner scanner = MetaTableAccessor.getMetaHTable(UTIL.getConnection()).getScanner(barrierScan)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java index ae7d12af47d..d57cc9c8b37 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -238,7 +239,7 @@ public class OfflineMetaRebuildTestCore { List dels = new ArrayList<>(); for (Result r : scanner) { RegionInfo info = - MetaTableAccessor.getRegionInfo(r); + CatalogFamilyFormat.getRegionInfo(r); if(info != null && !info.getTable().getNamespaceAsString() .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { Delete d = new Delete(r.getRow()); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftHBaseServiceHandler.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftHBaseServiceHandler.java index 32706a8ee8c..ecc2cccab31 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftHBaseServiceHandler.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftHBaseServiceHandler.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilder; import org.apache.hadoop.hbase.CellBuilderFactory; @@ -41,7 +42,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; @@ -1130,7 +1130,7 @@ public class ThriftHBaseServiceHandler extends HBaseServiceHandler implements Hb } // find region start and end keys - RegionInfo regionInfo = MetaTableAccessor.getRegionInfo(startRowResult); + RegionInfo regionInfo = CatalogFamilyFormat.getRegionInfo(startRowResult); if (regionInfo == null) { throw new IOException("RegionInfo REGIONINFO was null or " + " empty in Meta for row=" @@ -1144,7 +1144,7 @@ public class ThriftHBaseServiceHandler extends HBaseServiceHandler implements Hb region.version = HREGION_VERSION; // version not used anymore, PB encoding used. // find region assignment to server - ServerName serverName = MetaTableAccessor.getServerName(startRowResult, 0); + ServerName serverName = CatalogFamilyFormat.getServerName(startRowResult, 0); if (serverName != null) { region.setServerName(Bytes.toBytes(serverName.getHostname())); region.port = serverName.getPort();