Allow double-closing of FSTranslog

the translog might be reused across engines which is currently a problem
in the design such that we have to allow calls to `close` more than once.
This moves the closed check for snapshot on the actual file to exit the loop.

Relates to #10807

src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java

@@ -239,6 +239,11 @@ public class BufferingFsTranslogFile implements FsTranslogFile {
         return channelReference.file();
     }
 
+    @Override
+    public boolean closed() {
+        return this.closed.get();
+    }
+
     class WrapperOutputStream extends OutputStream {
 
         @Override

src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java

@@ -94,8 +94,6 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
 
     private final ApplySettings applySettings = new ApplySettings();
 
-    private final AtomicBoolean closed = new AtomicBoolean(false);
-
     @Inject
     public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService,
                       BigArrays bigArrays, ShardPath shardPath) throws IOException {
@@ -141,7 +139,6 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
 
     @Override
     public void close() throws IOException {
-        if (closed.compareAndSet(false, true)) {
         if (indexSettingsService != null) {
             indexSettingsService.removeListener(applySettings);
         }
@@ -152,7 +149,6 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
             rwl.writeLock().unlock();
         }
     }
-    }
 
     @Override
     public Path location() {
@@ -358,13 +354,15 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
     @Override
     public FsChannelSnapshot snapshot() throws TranslogException {
         while (true) {
-            if (closed.get()) {
+            FsTranslogFile current = this.current;
-                throw new TranslogException(shardId, "translog is already closed");
-            }
             FsChannelSnapshot snapshot = current.snapshot();
             if (snapshot != null) {
                 return snapshot;
             }
+            if (current.closed() && this.current == current) {
+                // check if we are closed and if we are still current - then this translog is closed and we can exit
+                throw new TranslogException(shardId, "current translog is already closed");
+            }
             Thread.yield();
         }
     }

src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java

@@ -82,4 +82,6 @@ public interface FsTranslogFile extends Closeable {
     TranslogStream getStream();
 
     public Path getPath();
+
+    public boolean closed();
 }

src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java

@@ -182,4 +182,10 @@ public class SimpleFsTranslogFile implements FsTranslogFile {
     public void updateBufferSize(int bufferSize) throws TranslogException {
         // nothing to do here...
     }
+
+    @Override
+    public boolean closed() {
+        return this.closed.get();
+    }
+
 }

src/test/java/org/elasticsearch/index/translog/AbstractSimpleTranslogTests.java

@@ -340,7 +340,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
             Translog.Snapshot snapshot = translog.snapshot();
             fail("translog is closed");
         } catch (TranslogException ex) {
-            assertEquals(ex.getMessage(), "translog is already closed");
+            assertEquals(ex.getMessage(), "current translog is already closed");
         }
     }