diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
index 43aeeb8405a..7345fe725e0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
@@ -214,7 +214,7 @@ public class MultiDocValues {
     if (!anyReal) {
       return null;
     } else {
-      OrdinalMap mapping = new OrdinalMap(values);
+      OrdinalMap mapping = new OrdinalMap(r.getCoreCacheKey(), values);
       return new MultiSortedDocValues(values, starts, mapping);
     }
   }
@@ -222,6 +222,8 @@ public class MultiDocValues {
   /** maps per-segment ordinals to/from global ordinal space */
   // TODO: use more efficient packed ints structures (these are all positive values!)
   static class OrdinalMap {
+    // cache key of whoever asked for this aweful thing
+    final Object owner;
     // globalOrd -> (globalOrd - segmentOrd)
     final AppendingLongBuffer globalOrdDeltas;
     // globalOrd -> sub index
@@ -229,9 +231,10 @@ public class MultiDocValues {
     // segmentOrd -> (globalOrd - segmentOrd)
     final AppendingLongBuffer ordDeltas[];
     
-    OrdinalMap(SortedDocValues subs[]) throws IOException {
+    OrdinalMap(Object owner, SortedDocValues subs[]) throws IOException {
       // create the ordinal mappings by pulling a termsenum over each sub's 
       // unique terms, and walking a multitermsenum over those
+      this.owner = owner;
       globalOrdDeltas = new AppendingLongBuffer();
       subIndexes = new AppendingLongBuffer();
       ordDeltas = new AppendingLongBuffer[subs.length];
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
index 690480dc346..32380f2f56b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
@@ -106,14 +106,13 @@ public final class SlowCompositeReaderWrapper extends AtomicReader {
         SortedDocValues dv = MultiDocValues.getSortedValues(in, field);
         if (dv instanceof MultiSortedDocValues) {
           map = ((MultiSortedDocValues)dv).mapping;
-          cachedOrdMaps.put(field, map);
+          if (map.owner == getCoreCacheKey()) {
+            cachedOrdMaps.put(field, map);
+          }
         }
         return dv;
       }
     }
-    if (true) { // nocommit
-      return MultiDocValues.getSortedValues(in, field);
-    }
     // cached multi dv
     assert map != null;
     int size = in.leaves().size();