HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1866,6 +1866,26 @@ public class HRegionServer extends HasThread implements
     return wal;
   }
 
+  @Override
+  public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+    if (regionInfo != null && regionInfo.isMetaTable() &&
+        regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+
+      walFactory.closeMetaWAL(regionInfo.getEncodedNameAsBytes());
+
+      LogRoller roller;
+      if (regionInfo != null && regionInfo.isMetaTable() &&
+          regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+        roller = metawalRoller.get();
+        if (roller != null) {
+          roller.removeWAL(wal); // only do this for meta WAL
+        }
+
+        // TODO: meta wal roller is left running. Should be fine.
+      }
+    }
+  }
+
   @Override
   public ClusterConnection getConnection() {
     return this.clusterConnection;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java

@@ -78,6 +78,10 @@ public class LogRoller extends HasThread {
     }
   }
 
+  public void removeWAL(final WAL wal) {
+    walNeedsRoll.remove(wal);
+  }
+
   public void requestRollAll() {
     for (WAL wal : walNeedsRoll.keySet()) {
       walNeedsRoll.put(wal, Boolean.TRUE);

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

@@ -1669,7 +1669,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         return response;
       }
 
-      HRegion.warmupHRegion(region, htd, regionServer.getWAL(region),
+      HRegion.warmupHRegion(region, htd, null,
           regionServer.getConfiguration(), regionServer, null);
 
     } catch (IOException ie) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java

@@ -54,6 +54,12 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi
    * default (common) WAL */
   WAL getWAL(HRegionInfo regionInfo) throws IOException;
 
+  /**
+   * Releases the dependency of this region to the WAL previously obtained from
+   * {@link #getWAL(HRegionInfo)}.
+   */
+  void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException;
+
   /**
    * @return Implementation of {@link CompactionRequestor} or null.
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java

@@ -19,6 +19,9 @@
 package org.apache.hadoop.hbase.regionserver.handler;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+import java.io.IOException;
+
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventType;
@@ -37,4 +40,9 @@ public class CloseMetaHandler extends CloseRegionHandler {
       final boolean abort) {
     super(server, rsServices, regionInfo, abort, EventType.M_RS_CLOSE_META, null);
   }
+
+  @Override
+  protected void releaseWALIfNeeded() throws IOException {
+    rsServices.releaseWAL(regionInfo, region.getWAL());
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java

@@ -44,8 +44,9 @@ public class CloseRegionHandler extends EventHandler {
   // have a running queue of user regions to close?
   private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class);
 
-  private final RegionServerServices rsServices;
+  protected final RegionServerServices rsServices;
-  private final HRegionInfo regionInfo;
+  protected final HRegionInfo regionInfo;
+  protected HRegion region;
 
   // If true, the hosting server is aborting.  Region close process is different
   // when we are aborting.
@@ -90,7 +91,7 @@ public class CloseRegionHandler extends EventHandler {
       LOG.debug("Processing close of " + name);
       String encodedRegionName = regionInfo.getEncodedName();
       // Check that this region is being served here
-      HRegion region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
+      region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
       if (region == null) {
         LOG.warn("Received CLOSE for region " + name + " but currently not serving - ignoring");
         // TODO: do better than a simple warning
@@ -106,6 +107,10 @@ public class CloseRegionHandler extends EventHandler {
             regionInfo.getRegionNameAsString());
           return;
         }
+
+        if (!abort) {
+          releaseWALIfNeeded();
+        }
       } catch (IOException ioe) {
         // An IOException here indicates that we couldn't successfully flush the
         // memstore before closing. So, we need to abort the server and allow
@@ -125,4 +130,8 @@ public class CloseRegionHandler extends EventHandler {
         remove(this.regionInfo.getEncodedNameAsBytes(), Boolean.FALSE);
     }
   }
+
+  protected void releaseWALIfNeeded() throws IOException {
+    // release the WAL if needed. Only meta does this for now.
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java

@@ -250,6 +250,21 @@ public class WALFactory {
     return metaProvider.getWAL(identifier, null);
   }
 
+  /**
+   * Closes the meta WAL and meta WALProvider
+   */
+  public void closeMetaWAL(final byte[] identifier) throws IOException {
+    // NOTE: this assumes single META region. The close of WAL does not do ref-counting for the
+    // number of regions depending on the meta WAL
+    WALProvider metaProvider = this.metaProvider.get();
+    if (metaProvider != null) {
+      if (this.metaProvider.compareAndSet(metaProvider, null)) {
+        // close the metaProvider
+        metaProvider.close();
+      }
+    }
+  }
+
   public Reader createReader(final FileSystem fs, final Path path) throws IOException {
     return createReader(fs, path, (CancelableProgressable)null);
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java

@@ -306,4 +306,8 @@ public class MockRegionServerServices implements RegionServerServices {
   public double getCompactionPressure() {
     return 0;
   }
+
+  @Override
+  public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java

@@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
@@ -80,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfiguratio
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
@@ -651,4 +651,8 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
   public double getCompactionPressure() {
     return 0;
   }
-}
+
+  @Override
+  public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+  }
+}