diff --git a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java
index 04bd16eaff6..0feaed21782 100644
--- a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java
+++ b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java
@@ -56,6 +56,7 @@ import com.metamx.emitter.service.ServiceMetricEvent;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -191,7 +192,12 @@ public class ServerManager implements QuerySegmentWalker
           dataSourceCounts.add(dataSource, -1L);
         }
 
-        oldQueryable.decrement();
+        try {
+          oldQueryable.close();
+        }
+        catch (IOException e) {
+          throw new SegmentLoadingException(e, "Unable to close segment %s", segment.getIdentifier());
+        }
       } else {
         log.info(
             "Told to delete a queryable on dataSource[%s] for interval[%s] and version [%s] that I don't have.",
diff --git a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java b/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java
index fb9a4bc8cfa..6a7b16efbf3 100644
--- a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java
+++ b/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java
@@ -112,8 +112,12 @@ public class ReferenceCountingSegment implements Segment
   {
     synchronized (lock) {
       if (!isClosed) {
-        baseSegment.close();
-        isClosed = true;
+        if (numReferences > 0) {
+          decrement();
+        } else {
+          baseSegment.close();
+          isClosed = true;
+        }
       }
     }
   }
@@ -140,7 +144,7 @@ public class ReferenceCountingSegment implements Segment
     }
   }
 
-  public void decrement()
+  private void decrement()
   {
     synchronized (lock) {
       if (!isClosed) {