[RECOVERY] Increment Store refcount on RecoveryTarget

We should make sure we have incremented the store refcount
before we start the recovery on the recovyer target.

Closes #6844

src/main/java/org/elasticsearch/index/store/Store.java

@@ -169,16 +169,36 @@ public class Store extends AbstractIndexShardComponent implements CloseableIndex
      *
      * Note: Close can safely be called multiple times.
      * @see #decRef
+     * @see #tryIncRef()
+     * @throws AlreadyClosedException iff the reference counter can not be incremented.
      */
     public final void incRef() {
+        if (tryIncRef() == false) {
+            throw new AlreadyClosedException("Store is already closed can't increment refCount current count [" + refCount.get() + "]");
+        }
+    }
+
+    /**
+     * Tries to increment the refCount of this Store instance. This method will return <tt>true</tt> iff the refCount was
+     * incremented successfully otherwise <tt>false</tt>. RefCounts are used to determine when a
+     * Store can be closed safely, i.e. as soon as there are no more references. Be sure to always call a
+     * corresponding {@link #decRef}, in a finally clause; otherwise the store may never be closed.  Note that
+     * {@link #close} simply calls decRef(), which means that the Store will not really be closed until {@link
+     * #decRef} has been called for all outstanding references.
+     *
+     * Note: Close can safely be called multiple times.
+     * @see #decRef()
+     * @see #incRef()
+     */
+    public final boolean tryIncRef() {
         do {
             int i = refCount.get();
             if (i > 0) {
                 if (refCount.compareAndSet(i, i + 1)) {
-                    return;
+                    return true;
                 }
             } else {
-                throw new AlreadyClosedException("Store is already closed can't increment refCount current count [" + i + "]");
+                return false;
             }
         } while (true);
     }

src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java

@@ -188,7 +188,6 @@ public class RecoveryTarget extends AbstractComponent {
 
     private void doRecovery(final StartRecoveryRequest request, final RecoveryStatus recoveryStatus, final RecoveryListener listener) {
         assert request.sourceNode() != null : "can't do a recovery without a source node";
-
         final InternalIndexShard shard = recoveryStatus.indexShard;
         if (shard == null) {
             listener.onIgnoreRecovery(false, "shard missing locally, stop recovery");
@@ -205,7 +204,7 @@ public class RecoveryTarget extends AbstractComponent {
         }
 
         recoveryStatus.recoveryThread = Thread.currentThread();
-
+        if (shard.store().tryIncRef()) {
             try {
                 logger.trace("[{}][{}] starting recovery from {}", request.shardId().index().name(), request.shardId().id(), request.sourceNode());
 
@@ -305,6 +304,11 @@ public class RecoveryTarget extends AbstractComponent {
 
                 logger.warn("[{}][{}] recovery from [{}] failed", e, request.shardId().index().name(), request.shardId().id(), request.sourceNode());
                 listener.onRecoveryFailure(new RecoveryFailedException(request, e), true);
+            } finally {
+                shard.store().decRef();
+            }
+        } else {
+            listener.onIgnoreRecovery(false, "local store closed, stop recovery");
         }
     }