HADOOP-2417 Fix critical shutdown problem introduced by HADOOP-2338

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@604011 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -77,6 +77,7 @@ Trunk (unreleased changes)
    HADOOP-2396 NPE in HMaster.cancelLease
    HADOOP-2397 The only time that a meta scanner should try to recover a log is
                when the master is starting
+   HADOOP-2417 Fix critical shutdown problem introduced by HADOOP-2338
    
   IMPROVEMENTS
    HADOOP-2401 Add convenience put method that takes writable

src/java/org/apache/hadoop/hbase/HMaster.java

@@ -181,16 +181,14 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
    */
   abstract class BaseScanner extends Chore {
     protected boolean rootRegion;
-    protected final Text tableName;
 
     protected abstract boolean initialScan();
     protected abstract void maintenanceScan();
 
-    BaseScanner(final Text tableName, final int period,
+    BaseScanner(final boolean rootRegion, final int period,
         final AtomicBoolean stop) {
       super(period, stop);
-      this.tableName = tableName;
+      this.rootRegion = rootRegion;
-      this.rootRegion = tableName.equals(ROOT_TABLE_NAME);
     }
     
     @Override
@@ -506,7 +504,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
   class RootScanner extends BaseScanner {
     /** Constructor */
     public RootScanner() {
-      super(HConstants.ROOT_TABLE_NAME, metaRescanInterval, closed);
+      super(true, metaRescanInterval, closed);
     }
 
     private boolean scanRoot() {
@@ -671,7 +669,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
     
     /** Constructor */
     public MetaScanner() {
-      super(HConstants.META_TABLE_NAME, metaRescanInterval, closed);
+      super(false, metaRescanInterval, closed);
     }
 
     private boolean scanOneMetaRegion(MetaRegion region) {
@@ -1182,16 +1180,25 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
    * regions can shut down.
    */
   private void stopScanners() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("telling root scanner to stop");
+    }
     synchronized(rootScannerLock) {
       if (rootScannerThread.isAlive()) {
         rootScannerThread.interrupt();  // Wake root scanner
       }
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("telling meta scanner to stop");
+    }
     synchronized(metaScannerLock) {
       if (metaScannerThread.isAlive()) {
         metaScannerThread.interrupt();  // Wake meta scanner
       }
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("meta and root scanners notified");
+    }
   }
 
   /*
@@ -1341,7 +1348,11 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         }
       } else if (msgs[0].getMsg() == HMsg.MSG_REPORT_QUIESCED) {
         LOG.info("Region server " + serverName + " quiesced");
-        if(quiescedMetaServers.incrementAndGet() == serversToServerInfo.size()) {
+        quiescedMetaServers.incrementAndGet();
+      }
+    }
+
+    if(quiescedMetaServers.get() >= serversToServerInfo.size()) {
       // If the only servers we know about are meta servers, then we can
       // proceed with shutdown
       LOG.info("All user tables quiesced. Proceeding with shutdown");
@@ -1352,7 +1363,8 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         delayedToDoQueue.clear();                  // Empty shut down queue
         toDoQueue.notifyAll();                     // Wake main thread
       }
-        }
+      synchronized (serversToServerInfo) {
+        serversToServerInfo.notifyAll();
       }
     }
 
@@ -1638,7 +1650,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
             " split. New regions are: " + newRegionA.getRegionName() + ", " +
             newRegionB.getRegionName());
 
-        if (region.getTableDesc().getName().equals(META_TABLE_NAME)) {
+        if (region.isMetaTable()) {
           // A meta region has split.
 
           onlineMetaRegions.remove(region.getStartKey());
@@ -2028,7 +2040,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
               serverName + "> (or server is null). Marking unassigned if " +
           "meta and clearing pendingRegions");
 
-          if (info.getTableDesc().getName().equals(META_TABLE_NAME)) {
+          if (info.isMetaTable()) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("removing meta region " + info.getRegionName() +
                   " from online meta regions");

src/java/org/apache/hadoop/hbase/HRegion.java

@@ -225,6 +225,7 @@ public class HRegion implements HConstants {
   protected final long threadWakeFrequency;
   private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
   private final Integer updateLock = new Integer(0);
+  private final Integer splitLock = new Integer(0);
   private final long desiredMaxFileSize;
   private final long minSequenceId;
   private final String encodedRegionName;
@@ -381,6 +382,7 @@ public class HRegion implements HConstants {
       LOG.info("region " + this.regionInfo.getRegionName() + " already closed");
       return null;
     }
+    synchronized (splitLock) {
       lock.writeLock().lock();
       try {
         synchronized (writestate) {
@@ -431,6 +433,7 @@ public class HRegion implements HConstants {
         lock.writeLock().unlock();
       }
     }
+  }
   
   //////////////////////////////////////////////////////////////////////////////
   // HRegion accessors
@@ -541,8 +544,9 @@ public class HRegion implements HConstants {
   HRegion[] splitRegion(final RegionUnavailableListener listener)
     throws IOException {
 
+    synchronized (splitLock) {
       Text midKey = new Text();
-    if (!needsSplit(midKey)) {
+      if (closed.get() || !needsSplit(midKey)) {
         return null;
       }
       long startTime = System.currentTimeMillis();
@@ -625,6 +629,7 @@ public class HRegion implements HConstants {
           StringUtils.formatTimeDiff(System.currentTimeMillis(), startTime));
       return regions;
     }
+  }
   
   /*
    * Iterates through all the HStores and finds the one with the largest
@@ -1030,6 +1035,7 @@ public class HRegion implements HConstants {
    * avoid a bunch of disk activity.
    *
    * @param row
+   * @param ts
    * @return Map<columnName, byte[]> values
    * @throws IOException
    */
@@ -1282,6 +1288,7 @@ public class HRegion implements HConstants {
    * @param row The row to operate on
    * @param family The column family to match
    * @param timestamp Timestamp to match
+   * @throws IOException
    */
   public void deleteFamily(Text row, Text family, long timestamp)
   throws IOException{

src/java/org/apache/hadoop/hbase/HRegionServer.java

@@ -79,9 +79,9 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
   // of HRegionServer in isolation. We use AtomicBoolean rather than
   // plain boolean so we can pass a reference to Chore threads.  Otherwise,
   // Chore threads need to know about the hosting class.
-  protected final AtomicBoolean stopRequested = new AtomicBoolean(false);
+  protected volatile AtomicBoolean stopRequested = new AtomicBoolean(false);
   
-  protected final AtomicBoolean quiesced = new AtomicBoolean(false);
+  protected volatile AtomicBoolean quiesced = new AtomicBoolean(false);
   
   // Go down hard.  Used if file system becomes unavailable and also in
   // debugging and unit tests.
@@ -95,13 +95,13 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
   private final Random rand = new Random();
   
   // region name -> HRegion
-  protected final SortedMap<Text, HRegion> onlineRegions =
+  protected volatile SortedMap<Text, HRegion> onlineRegions =
     Collections.synchronizedSortedMap(new TreeMap<Text, HRegion>());
-  protected final Map<Text, HRegion> retiringRegions =
+  protected volatile Map<Text, HRegion> retiringRegions =
     new ConcurrentHashMap<Text, HRegion>();
  
   protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final List<HMsg> outboundMsgs =
+  private volatile List<HMsg> outboundMsgs =
     Collections.synchronizedList(new ArrayList<HMsg>());
 
   final int numRetries;
@@ -120,7 +120,7 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
   private final Leases leases;
   
   // Request counter
-  private final AtomicInteger requestCount = new AtomicInteger();
+  private volatile AtomicInteger requestCount = new AtomicInteger();
   
   // A sleeper that sleeps for msgInterval.
   private final Sleeper sleeper;
@@ -296,7 +296,7 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       // splitting a 'normal' region, and the ROOT table needs to be
       // updated if we are splitting a META region.
       HTable t = null;
-      if (region.getRegionInfo().getTableDesc().getName().equals(META_TABLE_NAME)) {
+      if (region.getRegionInfo().isMetaTable()) {
         // We need to update the root region
         if (this.root == null) {
           this.root = new HTable(conf, ROOT_TABLE_NAME);