diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java
index 59f7eebd503..d81c1c5fa6c 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java
@@ -138,12 +138,12 @@ public class CacheFile {
             try {
                 synchronized (listeners) {
                     ensureOpen();
-                    final boolean added = listeners.add(listener);
-                    assert added : "listener already exists " + listener;
-                    if (listeners.size() == 1) {
+                    if (listeners.isEmpty()) {
                         assert channelRef == null;
                         channelRef = new FileChannelReference();
                     }
+                    final boolean added = listeners.add(listener);
+                    assert added : "listener already exists " + listener;
                 }
                 success = true;
             } finally {