HBASE-2774 Spin in ReadWriteConsistencyControl eating CPU (load > 40) and no progress running YCSB on clean cluster startup

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@957411 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -417,6 +417,8 @@ Release 0.21.0 - Unreleased
                average - slop; endless churn
    HBASE-2729  Interrupted or failed memstore flushes should not corrupt the
                region
+   HBASE-2774  Spin in ReadWriteConsistencyControl eating CPU (load > 40) and
+               no progress running YCSB on clean cluster startup                     
 
   IMPROVEMENTS
    HBASE-1760  Cleanup TODOs in HTable

src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java

@@ -10,8 +10,11 @@ import java.util.concurrent.atomic.AtomicLong;
  * the new writes for readers to read (thus forming atomic transactions).
  */
 public class ReadWriteConsistencyControl {
-  private final AtomicLong memstoreRead = new AtomicLong();
-  private final AtomicLong memstoreWrite = new AtomicLong();
+  private volatile long memstoreRead = 0;
+  private volatile long memstoreWrite = 0;
+
+  private final Object readWaiters = new Object();
+
   // This is the pending queue of writes.
   private final LinkedList<WriteEntry> writeQueue =
       new LinkedList<WriteEntry>();
@@ -34,12 +37,13 @@ public class ReadWriteConsistencyControl {
 
   public WriteEntry beginMemstoreInsert() {
     synchronized (writeQueue) {
-      long nextWriteNumber = memstoreWrite.incrementAndGet();
+      long nextWriteNumber = ++memstoreWrite;
       WriteEntry e = new WriteEntry(nextWriteNumber);
       writeQueue.add(e);
       return e;
     }
   }
+
   public void completeMemstoreInsert(WriteEntry e) {
     synchronized (writeQueue) {
       e.markCompleted();
@@ -70,24 +74,32 @@ public class ReadWriteConsistencyControl {
       }
 
       if (nextReadValue > 0) {
-        memstoreRead.set(nextReadValue);
+        memstoreRead = nextReadValue;
+
+        synchronized (readWaiters) {
+          readWaiters.notifyAll();
+        }
+
       }
     }
 
-    // Spin until any other concurrent puts have finished. This makes sure that
-    // if we move on to construct a scanner, we'll get read-your-own-writes
-    // consistency. We anticipate that since puts to the memstore are very fast,
-    // this will be on the order of microseconds - so spinning should be faster
-    // than a condition variable.
-    int spun = 0;
-    while (memstoreRead.get() < e.getWriteNumber()) {
-      spun++;
+    boolean interrupted = false;
+    while (memstoreRead < e.getWriteNumber()) {
+      synchronized (readWaiters) {
+        try {
+          readWaiters.wait(0);
+        } catch (InterruptedException e) {
+          // We were interrupted... finish the loop -- i.e. cleanup --and then
+          // on our way out, reset the interrupt flag.
+          interrupted = true;
+        }
+      }
     }
-    // Could potentially expose spun as a metric
+    if (interrupted) Thread.currentThread.interrupt();
   }
 
   public long memstoreReadPoint() {
-    return memstoreRead.get();
+    return memstoreRead;
   }