HBASE-8097 MetaServerShutdownHandler may potentially keep bumping up DeadServer.numProcessing (Jeffrey Zhong)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1457932 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java

@@ -98,9 +98,15 @@ public class DeadServer {
     return clone;
   }
 
-  public synchronized boolean add(ServerName e) {
+  /**
+   * Adds the server to the dead server list if it's not there already.
+   * @param sn the server name
+   */
+  public synchronized void add(ServerName sn) {
     this.numProcessing++;
-    return deadServers.put(e, EnvironmentEdgeManager.currentTimeMillis()) != null;
+    if (!deadServers.containsKey(sn)){
+      deadServers.put(sn, EnvironmentEdgeManager.currentTimeMillis());
+    }
   }
 
   @SuppressWarnings("UnusedParameters")

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -487,7 +487,7 @@ public class ServerManager {
     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
     if (carryingMeta) {
       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
-        this.services, this.deadservers, serverName, carryingMeta));
+        this.services, this.deadservers, serverName));
     } else {
       this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
         this.services, this.deadservers, serverName, true));

hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java

@@ -32,49 +32,51 @@ import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.zookeeper.KeeperException;
 
 /**
- * Shutdown handler for the server hosting <code>-ROOT-</code>,
+ * Shutdown handler for the server hosting <code>.META.</code>
- * <code>.META.</code>, or both.
  */
 @InterfaceAudience.Private
 public class MetaServerShutdownHandler extends ServerShutdownHandler {
-  private final boolean carryingMeta;
   private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
   public MetaServerShutdownHandler(final Server server,
       final MasterServices services,
-      final DeadServer deadServers, final ServerName serverName,
+      final DeadServer deadServers, final ServerName serverName) {
-      final boolean carryingMeta) {
     super(server, services, deadServers, serverName,
       EventType.M_META_SERVER_SHUTDOWN, true);
-    this.carryingMeta = carryingMeta;
   }
 
   @Override
   public void process() throws IOException {
-    try {
+    boolean gotException = true; 
-      LOG.info("Splitting META logs for " + serverName);
+    try{
-      if (this.shouldSplitHlog) {
+      try {
-        this.services.getMasterFileSystem().splitMetaLog(serverName);
+        LOG.info("Splitting META logs for " + serverName);
+        if (this.shouldSplitHlog) {
+          this.services.getMasterFileSystem().splitMetaLog(serverName);
+        }
+      } catch (IOException ioe) {
+        this.services.getExecutorService().submit(this);
+        this.deadServers.add(serverName);
+        throw new IOException("failed log splitting for " +
+            serverName + ", will retry", ioe);
       }
-    } catch (IOException ioe) {
+  
-      this.services.getExecutorService().submit(this);
+      // Assign meta if we were carrying it.
-      this.deadServers.add(serverName);
-      throw new IOException("failed log splitting for " +
-          serverName + ", will retry", ioe);
-    }
-
-    // Assign root and meta if we were carrying them.
-    if (isCarryingMeta()) { // .META.
       // Check again: region may be assigned to other where because of RIT
       // timeout
       if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
-        LOG.info("Server " + serverName
+        LOG.info("Server " + serverName + " was carrying META. Trying to assign.");
-            + " was carrying META. Trying to assign.");
+        this.services.getAssignmentManager().regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
-        this.services.getAssignmentManager().regionOffline(
-            HRegionInfo.FIRST_META_REGIONINFO);
         verifyAndAssignMetaWithRetries();
       } else {
         LOG.info("META has been assigned to otherwhere, skip assigning.");
       }
+      
+      gotException = false;
+    } finally {
+      if (gotException){
+        // If we had an exception, this.deadServers.finish will be skipped in super.process()
+        this.deadServers.finish(serverName);
+      }     
     }
     super.process();
   }
@@ -97,10 +99,10 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler {
     if (!this.server.getCatalogTracker().verifyMetaRegionLocation(timeout)) {
       this.services.getAssignmentManager().assignMeta();
     } else if (serverName.equals(server.getCatalogTracker().getMetaLocation())) {
-      throw new IOException("-ROOT- is onlined on the dead server "
+      throw new IOException(".META. is onlined on the dead server "
           + serverName);
     } else {
-      LOG.info("Skip assigning -ROOT-, because it is online on the "
+      LOG.info("Skip assigning .META., because it is online on the "
           + server.getCatalogTracker().getMetaLocation());
     }
   }
@@ -142,10 +144,6 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler {
     }
   }
 
-  boolean isCarryingMeta() {
-    return this.carryingMeta;
-  }
-  
   @Override
   public String toString() {
     String name = "UnknownServerName";