HBASE-10329 Fail the writes rather than proceeding silently to prevent data loss when AsyncSyncer encounters null writer and its writes aren't synced by other Asyncer

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1557839 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java

@@ -1206,11 +1206,36 @@ class FSHLog implements HLog, Syncable {
           // 2. do 'sync' to HDFS to provide durability
           long now = EnvironmentEdgeManager.currentTimeMillis();
           try {
-            this.isSyncing = true;
+            if (writer == null) {
-            if (writer != null) {
+              // the only possible case where writer == null is as below:
-              writer.sync();
+              // 1. t1: AsyncWriter append writes to hdfs,
+              //        envokes AsyncSyncer 1 with writtenTxid==100
+              // 2. t2: AsyncWriter append writes to hdfs,
+              //        envokes AsyncSyncer 2 with writtenTxid==200
+              // 3. t3: rollWriter starts, it grabs the updateLock which
+              //        prevents further writes entering pendingWrites and
+              //        wait for all items(200) in pendingWrites to append/sync
+              //        to hdfs
+              // 4. t4: AsyncSyncer 2 finishes, now syncedTillHere==200
+              // 5. t5: rollWriter close writer, set writer=null...
+              // 6. t6: AsyncSyncer 1 starts to use writer to do sync... before
+              //        rollWriter set writer to the newly created Writer
+              //
+              // So when writer == null here:
+              // 1. if txidToSync <= syncedTillHere, can safely ignore sync here;
+              // 2. if txidToSync > syncedTillHere, we need fail all the writes with
+              //    txid <= txidToSync to avoid 'data loss' where user get successful
+              //    write response but can't read the writes!
+              if (this.txidToSync > syncedTillHere.get()) {
+                LOG.fatal("should never happen: has unsynced writes but writer is null!");
+                asyncIOE = new IOException("has unsynced writes but writer is null!");
+                failedTxid.set(this.txidToSync);
               }
+            } else {
+              this.isSyncing = true;
+              writer.sync();
               this.isSyncing = false;
+            }
             postSync();
           } catch (IOException e) {
             LOG.fatal("Error while AsyncSyncer sync, request close of hlog ", e);
@@ -1218,6 +1243,8 @@ class FSHLog implements HLog, Syncable {
 
             asyncIOE = e;
             failedTxid.set(this.txidToSync);
+
+            this.isSyncing = false;
           }
           metrics.finishSync(EnvironmentEdgeManager.currentTimeMillis() - now);