HBASE-21464 Splitting blocked with meta NSRE during split transaction

When looking up the locations of hbase:meta with useCache false, clear all previous
cache entries for it first

Fix Table reference leaks in MetaTableAccessor with try-with-resources

Signed-off-by: Allan Yang <allan163@apache.org>

hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java

@@ -175,8 +175,7 @@ public class MetaTableAccessor {
    * @return An {@link Table} for <code>hbase:meta</code>
    * @throws IOException
    */
-  static Table getMetaHTable(final Connection connection)
+  static Table getMetaHTable(final Connection connection) throws IOException {
-  throws IOException {
     // We used to pass whole CatalogTracker in here, now we just pass in Connection
     if (connection == null) {
       throw new NullPointerException("No connection");
@@ -248,11 +247,13 @@ public class MetaTableAccessor {
     }
     Get get = new Get(row);
     get.addFamily(HConstants.CATALOG_FAMILY);
-    Result r = get(getMetaHTable(connection), get);
+    try (Table metaTable = getMetaHTable(connection)) {
-    RegionLocations locations = getRegionLocations(r);
+      Result r = get(metaTable, get);
-    return locations == null
+      RegionLocations locations = getRegionLocations(r);
-      ? null
+      return locations == null
-      : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+          ? null
+          : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+    }
   }
 
   /**
@@ -267,8 +268,10 @@ public class MetaTableAccessor {
     byte[] row = getMetaKeyForRegion(regionInfo);
     Get get = new Get(row);
     get.addFamily(HConstants.CATALOG_FAMILY);
-    Result r = get(getMetaHTable(connection), get);
+    try (Table metaTable = getMetaHTable(connection)) {
-    return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+      Result r = get(metaTable, get);
+      return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+    }
   }
 
   /** Returns the row key to use for this regionInfo */
@@ -300,7 +303,9 @@ public class MetaTableAccessor {
       byte[] regionName) throws IOException {
     Get get = new Get(regionName);
     get.addFamily(HConstants.CATALOG_FAMILY);
-    return get(getMetaHTable(connection), get);
+    try (Table metaTable = getMetaHTable(connection)) {
+      return get(metaTable, get);
+    }
   }
 
   /**
@@ -631,19 +636,19 @@ public class MetaTableAccessor {
       scan.setCaching(caching);
     }
     scan.addFamily(HConstants.CATALOG_FAMILY);
-    Table metaTable = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    ResultScanner scanner = null;
+      try (ResultScanner scanner = metaTable.getScanner(scan)) {
-    try {
+        Result data;
-      scanner = metaTable.getScanner(scan);
+        while ((data = scanner.next()) != null) {
-      Result data;
+          if (data.isEmpty()) {
-      while((data = scanner.next()) != null) {
+            continue;
-        if (data.isEmpty()) continue;
+          }
-        // Break if visit returns false.
+          // Break if visit returns false.
-        if (!visitor.visit(data)) break;
+          if (!visitor.visit(data)) {
+            break;
+          }
+        }
       }
-    } finally {
-      if (scanner != null) scanner.close();
-      metaTable.close();
     }
   }
 
@@ -1020,7 +1025,9 @@ public class MetaTableAccessor {
    */
   static void putToMetaTable(final Connection connection, final Put p)
     throws IOException {
-    put(getMetaHTable(connection), p);
+    try (Table metaTable = getMetaHTable(connection)) {
+      put(metaTable, p);
+    }
   }
 
   /**
@@ -1044,11 +1051,8 @@ public class MetaTableAccessor {
    */
   public static void putsToMetaTable(final Connection connection, final List<Put> ps)
     throws IOException {
-    Table t = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
+      metaTable.put(ps);
-      t.put(ps);
-    } finally {
-      t.close();
     }
   }
 
@@ -1073,11 +1077,8 @@ public class MetaTableAccessor {
    */
   public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
     throws IOException {
-    Table t = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
+      metaTable.delete(deletes);
-      t.delete(deletes);
-    } finally {
-      t.close();
     }
   }
 
@@ -1116,15 +1117,12 @@ public class MetaTableAccessor {
   public static void mutateMetaTable(final Connection connection,
                                      final List<Mutation> mutations)
     throws IOException {
-    Table t = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
+      metaTable.batch(mutations);
-      t.batch(mutations);
     } catch (InterruptedException e) {
       InterruptedIOException ie = new InterruptedIOException(e.getMessage());
       ie.initCause(e);
       throw ie;
-    } finally {
-      t.close();
     }
   }
 
@@ -1188,11 +1186,8 @@ public class MetaTableAccessor {
    */
   public static void addRegionToMeta(Connection connection, HRegionInfo regionInfo,
                                      HRegionInfo splitA, HRegionInfo splitB) throws IOException {
-    Table meta = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
+      addRegionToMeta(metaTable, regionInfo, splitA, splitB);
-      addRegionToMeta(meta, regionInfo, splitA, splitB);
-    } finally {
-      meta.close();
     }
   }
 
@@ -1269,8 +1264,7 @@ public class MetaTableAccessor {
       HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication,
       long masterSystemTime)
           throws IOException {
-    Table meta = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
       HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
 
       // use the maximum of what master passed us vs local time.
@@ -1298,9 +1292,7 @@ public class MetaTableAccessor {
 
       byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
         + HConstants.DELIMITER);
-      multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
+      multiMutate(metaTable, tableRow, putOfMerged, deleteA, deleteB);
-    } finally {
-      meta.close();
     }
   }
 
@@ -1318,8 +1310,7 @@ public class MetaTableAccessor {
   public static void splitRegion(final Connection connection,
                                  HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
                                  ServerName sn, int regionReplication) throws IOException {
-    Table meta = getMetaHTable(connection);
+    try (Table metaTable = getMetaHTable(connection)) {
-    try {
       HRegionInfo copyOfParent = new HRegionInfo(parent);
       copyOfParent.setOffline(true);
       copyOfParent.setSplit(true);
@@ -1343,9 +1334,7 @@ public class MetaTableAccessor {
       }
 
       byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
-      multiMutate(meta, tableRow, putParent, putA, putB);
+      multiMutate(metaTable, tableRow, putParent, putA, putB);
-    } finally {
-      meta.close();
     }
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java

@@ -1236,10 +1236,9 @@ class ConnectionManager {
       // HBASE-10785: We cache the location of the META itself, so that we are not overloading
       // zookeeper with one request for every region lookup. We cache the META with empty row
       // key in MetaCache.
-      byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta
       RegionLocations locations = null;
       if (useCache) {
-        locations = getCachedLocation(tableName, metaCacheKey);
+        locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
         if (locations != null && locations.getRegionLocation(replicaId) != null) {
           return locations;
         }
@@ -1250,10 +1249,15 @@ class ConnectionManager {
         // Check the cache again for a hit in case some other thread made the
         // same query while we were waiting on the lock.
         if (useCache) {
-          locations = getCachedLocation(tableName, metaCacheKey);
+          locations = getCachedLocation(tableName, HRegionInfo.FIRST_META_REGIONINFO.getStartKey());
           if (locations != null && locations.getRegionLocation(replicaId) != null) {
             return locations;
           }
+        } else {
+          // Don't keep stale entries in cache when relocating meta. Cache state should
+          // reflect whatever is up in zookeeper (which could be the case where no region
+          // is deployed yet).
+          clearRegionCache(tableName);
         }
 
         // Look up from zookeeper