LUCENE-4790: FieldCache.getDocTermOrds back to the future bug

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1448368 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -238,6 +238,10 @@ Bug Fixes
   is enhanced to roll up beyond the bottom cell level. (David Smiley,
   Florian Schilling)
 
+* LUCENE-4790: Fix FieldCacheImpl.getDocTermOrds to not bake deletes into the
+  cached datastructure. Otherwise this can cause inconsistencies with readers
+  at different points in time.  (Robert Muir)
+
 Documentation
 
 * LUCENE-4718: Fixed documentation of oal.queryparser.classic.

lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java

@@ -173,30 +173,29 @@ public class DocTermOrds {
   }
 
   /** Inverts all terms */
-  public DocTermOrds(AtomicReader reader, String field) throws IOException {
-    this(reader, field, null, Integer.MAX_VALUE);
+  public DocTermOrds(AtomicReader reader, Bits liveDocs, String field) throws IOException {
+    this(reader, liveDocs, field, null, Integer.MAX_VALUE);
   }
 
   /** Inverts only terms starting w/ prefix */
-  public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix) throws IOException {
-    this(reader, field, termPrefix, Integer.MAX_VALUE);
+  public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix) throws IOException {
+    this(reader, liveDocs, field, termPrefix, Integer.MAX_VALUE);
   }
 
   /** Inverts only terms starting w/ prefix, and only terms
    *  whose docFreq (not taking deletions into account) is
    *  <=  maxTermDocFreq */
-  public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix, int maxTermDocFreq) throws IOException {
-    this(reader, field, termPrefix, maxTermDocFreq, DEFAULT_INDEX_INTERVAL_BITS);
-    uninvert(reader, termPrefix);
+  public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix, int maxTermDocFreq) throws IOException {
+    this(reader, liveDocs, field, termPrefix, maxTermDocFreq, DEFAULT_INDEX_INTERVAL_BITS);
   }
 
   /** Inverts only terms starting w/ prefix, and only terms
    *  whose docFreq (not taking deletions into account) is
    *  <=  maxTermDocFreq, with a custom indexing interval
    *  (default is every 128nd term). */
-  public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix, int maxTermDocFreq, int indexIntervalBits) throws IOException {
+  public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix, int maxTermDocFreq, int indexIntervalBits) throws IOException {
     this(field, maxTermDocFreq, indexIntervalBits);
-    uninvert(reader, termPrefix);
+    uninvert(reader, liveDocs, termPrefix);
   }
 
   /** Subclass inits w/ this, but be sure you then call
@@ -257,14 +256,14 @@ public class DocTermOrds {
   protected void visitTerm(TermsEnum te, int termNum) throws IOException {
   }
 
-  /** Invoked during {@link #uninvert(AtomicReader,BytesRef)}
+  /** Invoked during {@link #uninvert(AtomicReader,Bits,BytesRef)}
    *  to record the document frequency for each uninverted
    *  term. */
   protected void setActualDocFreq(int termNum, int df) throws IOException {
   }
 
   /** Call this only once (if you subclass!) */
-  protected void uninvert(final AtomicReader reader, final BytesRef termPrefix) throws IOException {
+  protected void uninvert(final AtomicReader reader, Bits liveDocs, final BytesRef termPrefix) throws IOException {
     final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
     if (info != null && info.hasDocValues()) {
       throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
@@ -304,8 +303,6 @@ public class DocTermOrds {
 
     boolean testedOrd = false;
 
-    final Bits liveDocs = reader.getLiveDocs();
-
     // we need a minimum of 9 bytes, but round up to 12 since the space would
     // be wasted with most allocators anyway.
     byte[] tempArr = new byte[12];

lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java

@@ -1399,7 +1399,7 @@ class FieldCacheImpl implements FieldCache {
     @Override
     protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
         throws IOException {
-      return new DocTermOrds(reader, key.field);
+      return new DocTermOrds(reader, null, key.field);
     }
   }
 

lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java

@@ -63,7 +63,7 @@ public class TestDocTermOrds extends LuceneTestCase {
     w.close();
 
     final AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
-    final DocTermOrds dto = new DocTermOrds(ar, "field");
+    final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field");
     SortedSetDocValues iter = dto.iterator(ar.terms("field").iterator(null));
     
     iter.setDocument(0);
@@ -295,7 +295,7 @@ public class TestDocTermOrds extends LuceneTestCase {
 
   private void verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef) throws Exception {
 
-    final DocTermOrds dto = new DocTermOrds(r,
+    final DocTermOrds dto = new DocTermOrds(r, r.getLiveDocs(),
                                             "field",
                                             prefixRef,
                                             Integer.MAX_VALUE,
@@ -372,4 +372,34 @@ public class TestDocTermOrds extends LuceneTestCase {
       assertEquals(answers.length, upto);
     }
   }
+  
+  public void testBackToTheFuture() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    
+    Document doc = new Document();
+    doc.add(newStringField("foo", "bar", Field.Store.NO));
+    iw.addDocument(doc);
+    
+    doc = new Document();
+    doc.add(newStringField("foo", "baz", Field.Store.NO));
+    iw.addDocument(doc);
+    
+    DirectoryReader r1 = DirectoryReader.open(iw, true);
+    
+    iw.deleteDocuments(new Term("foo", "baz"));
+    DirectoryReader r2 = DirectoryReader.open(iw, true);
+    
+    FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r2), "foo");
+    
+    SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r1), "foo");
+    assertEquals(2, v.getValueCount());
+    v.setDocument(1);
+    assertEquals(1, v.nextOrd());
+    
+    iw.close();
+    r1.close();
+    r2.close();
+    dir.close();
+  }
 }

lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java

@@ -472,7 +472,7 @@ public class TestFieldCache extends LuceneTestCase {
     } catch (IllegalStateException expected) {}
     
     try {
-      new DocTermOrds(ar, "binary");
+      new DocTermOrds(ar, null, "binary");
       fail();
     } catch (IllegalStateException expected) {}
     
@@ -486,7 +486,7 @@ public class TestFieldCache extends LuceneTestCase {
     } catch (IllegalStateException expected) {}
     
     try {
-      new DocTermOrds(ar, "sorted");
+      new DocTermOrds(ar, null, "sorted");
       fail();
     } catch (IllegalStateException expected) {}
     
@@ -529,7 +529,7 @@ public class TestFieldCache extends LuceneTestCase {
     } catch (IllegalStateException expected) {}
     
     try {
-      new DocTermOrds(ar, "numeric");
+      new DocTermOrds(ar, null, "numeric");
       fail();
     } catch (IllegalStateException expected) {}
     
@@ -554,7 +554,7 @@ public class TestFieldCache extends LuceneTestCase {
       } catch (IllegalStateException expected) {}
       
       try {
-        new DocTermOrds(ar, "sortedset");
+        new DocTermOrds(ar, null, "sortedset");
         fail();
       } catch (IllegalStateException expected) {}
     

solr/core/src/java/org/apache/solr/request/UnInvertedField.java

@@ -23,6 +23,7 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.DocTermOrds;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.Term;
@@ -174,7 +175,8 @@ public class UnInvertedField extends DocTermOrds {
     final String prefix = TrieField.getMainValuePrefix(searcher.getSchema().getFieldType(field));
     this.searcher = searcher;
     try {
-      uninvert(searcher.getAtomicReader(), prefix == null ? null : new BytesRef(prefix));
+      AtomicReader r = searcher.getAtomicReader();
+      uninvert(r, r.getLiveDocs(), prefix == null ? null : new BytesRef(prefix));
     } catch (IllegalStateException ise) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ise.getMessage());
     }