diff --git a/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index b4b9e13f7e0..713563eb111 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -129,15 +129,12 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
             return true;
         }
         synchronized (this) {
-            List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
-            if (listeners == null) {
-                if (closed) {
-                    throw new IllegalStateException("can't wait for refresh on a closed index");
-                }
-                listeners = new ArrayList<>();
-                refreshListeners = listeners;
+            if (closed) {
+                throw new IllegalStateException("can't wait for refresh on a closed index");
             }
-            if (refreshForcers == 0 && listeners.size() < getMaxRefreshListeners.getAsInt()) {
+            List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
+            final int maxRefreshes = getMaxRefreshListeners.getAsInt();
+            if (refreshForcers == 0 && maxRefreshes > 0 && (listeners == null || listeners.size() < maxRefreshes)) {
                 ThreadContext.StoredContext storedContext = threadContext.newStoredContext(true);
                 Consumer<Boolean> contextPreservingListener = forced -> {
                     try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
@@ -145,8 +142,12 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
                         listener.accept(forced);
                     }
                 };
+                if (listeners == null) {
+                    listeners = new ArrayList<>();
+                }
                 // We have a free slot so register the listener
                 listeners.add(new Tuple<>(location, contextPreservingListener));
+                refreshListeners = listeners;
                 return false;
             }
         }