LUCENE-5666: fix some tests

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5666@1594311 13f79535-47bb-0310-9956-ffa450edef68

lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java

@@ -0,0 +1,248 @@
+package org.apache.lucene.uninverting;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.uninverting.UninvertingReader.Type;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+public class TestUninvertingReader extends LuceneTestCase {
+  
+  public void testSortedSetInteger() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    
+    Document doc = new Document();
+    doc.add(new IntField("foo", 5, Field.Store.NO));
+    iw.addDocument(doc);
+    
+    doc = new Document();
+    doc.add(new IntField("foo", 5, Field.Store.NO));
+    doc.add(new IntField("foo", -3, Field.Store.NO));
+    iw.addDocument(doc);
+    
+    iw.forceMerge(1);
+    iw.shutdown();
+    
+    DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), 
+                         Collections.singletonMap("foo", Type.SORTED_SET_INTEGER));
+    AtomicReader ar = ir.leaves().get(0).reader();
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    SortedSetDocValues v = ar.getSortedSetDocValues("foo");
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    assertEquals(2, v.getValueCount());
+    
+    v.setDocument(0);
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    v.setDocument(1);
+    assertEquals(0, v.nextOrd());
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    BytesRef value = new BytesRef();
+    v.lookupOrd(0, value);
+    assertEquals(-3, NumericUtils.prefixCodedToInt(value));
+    
+    v.lookupOrd(1, value);
+    assertEquals(5, NumericUtils.prefixCodedToInt(value));
+    
+    ir.close();
+    dir.close();
+  }
+  
+  public void testSortedSetFloat() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    
+    Document doc = new Document();
+    doc.add(new IntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
+    iw.addDocument(doc);
+    
+    doc = new Document();
+    doc.add(new IntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
+    doc.add(new IntField("foo", Float.floatToRawIntBits(-3f), Field.Store.NO));
+    iw.addDocument(doc);
+    
+    iw.forceMerge(1);
+    iw.shutdown();
+    
+    DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), 
+                         Collections.singletonMap("foo", Type.SORTED_SET_FLOAT));
+    AtomicReader ar = ir.leaves().get(0).reader();
+    
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    SortedSetDocValues v = ar.getSortedSetDocValues("foo");
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    assertEquals(2, v.getValueCount());
+    
+    v.setDocument(0);
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    v.setDocument(1);
+    assertEquals(0, v.nextOrd());
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    BytesRef value = new BytesRef();
+    v.lookupOrd(0, value);
+    assertEquals(Float.floatToRawIntBits(-3f), NumericUtils.prefixCodedToInt(value));
+    
+    v.lookupOrd(1, value);
+    assertEquals(Float.floatToRawIntBits(5f), NumericUtils.prefixCodedToInt(value));
+    
+    ir.close();
+    dir.close();
+  }
+  
+  public void testSortedSetLong() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    
+    Document doc = new Document();
+    doc.add(new LongField("foo", 5, Field.Store.NO));
+    iw.addDocument(doc);
+    
+    doc = new Document();
+    doc.add(new LongField("foo", 5, Field.Store.NO));
+    doc.add(new LongField("foo", -3, Field.Store.NO));
+    iw.addDocument(doc);
+    
+    iw.forceMerge(1);
+    iw.shutdown();
+    
+    DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), 
+        Collections.singletonMap("foo", Type.SORTED_SET_LONG));
+    AtomicReader ar = ir.leaves().get(0).reader();
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    SortedSetDocValues v = ar.getSortedSetDocValues("foo");
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    assertEquals(2, v.getValueCount());
+    
+    v.setDocument(0);
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    v.setDocument(1);
+    assertEquals(0, v.nextOrd());
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    BytesRef value = new BytesRef();
+    v.lookupOrd(0, value);
+    assertEquals(-3, NumericUtils.prefixCodedToLong(value));
+    
+    v.lookupOrd(1, value);
+    assertEquals(5, NumericUtils.prefixCodedToLong(value));
+    
+    ir.close();
+    dir.close();
+  }
+  
+  public void testSortedSetDouble() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    
+    Document doc = new Document();
+    doc.add(new LongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
+    iw.addDocument(doc);
+    
+    doc = new Document();
+    doc.add(new LongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
+    doc.add(new LongField("foo", Double.doubleToRawLongBits(-3d), Field.Store.NO));
+    iw.addDocument(doc);
+    
+    iw.forceMerge(1);
+    iw.shutdown();
+    
+    DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), 
+        Collections.singletonMap("foo", Type.SORTED_SET_DOUBLE));
+    AtomicReader ar = ir.leaves().get(0).reader();
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    SortedSetDocValues v = ar.getSortedSetDocValues("foo");
+    assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
+    assertEquals(2, v.getValueCount());
+    
+    v.setDocument(0);
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    v.setDocument(1);
+    assertEquals(0, v.nextOrd());
+    assertEquals(1, v.nextOrd());
+    assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
+    
+    BytesRef value = new BytesRef();
+    v.lookupOrd(0, value);
+    assertEquals(Double.doubleToRawLongBits(-3d), NumericUtils.prefixCodedToLong(value));
+    
+    v.lookupOrd(1, value);
+    assertEquals(Double.doubleToRawLongBits(5d), NumericUtils.prefixCodedToLong(value));
+    
+    ir.close();
+    dir.close();
+  }
+  
+  private void assertNoSilentInsanity(AtomicReader reader, String field, DocValuesType type) throws IOException {
+    Set<DocValuesType> insaneTypes = EnumSet.allOf(DocValuesType.class);
+    insaneTypes.remove(type);
+    
+    for (DocValuesType t : insaneTypes) {
+      tryToBeInsane(reader, field, type, t);
+    }
+  }
+  
+  private void tryToBeInsane(AtomicReader reader, String field, DocValuesType actualType, DocValuesType insaneType) throws IOException {
+    try {
+      switch(insaneType) {
+        case NUMERIC:
+          reader.getNumericDocValues(field);
+          break;
+        case SORTED:
+          reader.getSortedDocValues(field);
+          break;
+        case BINARY:
+          reader.getBinaryDocValues(field);
+          break;
+        case SORTED_SET:
+          reader.getSortedSetDocValues(field);
+        default:
+          throw new AssertionError();
+      }
+      fail("didn't get expected exception: actual=" + actualType + ",insane=" + insaneType);
+    } catch (IllegalStateException expected) {}
+  }
+}

solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java

@@ -120,6 +120,7 @@ public class PreAnalyzedField extends FieldType {
   
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
+    field.checkSortability();
     return new SortedSetSortField(field.getName(), top);
   }
   

solr/core/src/java/org/apache/solr/schema/TextField.java

@@ -96,6 +96,7 @@ public class TextField extends FieldType {
   @Override
   public SortField getSortField(SchemaField field, boolean reverse) {
     /* :TODO: maybe warn if isTokenized(), but doesn't use LimitTokenCountFilter in it's chain? */
+    field.checkSortability();
     return new SortedSetSortField(field.getName(), reverse);
   }
   

solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -369,7 +369,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
     
     long cpg = reader.getIndexCommit().getGeneration();
     try {
-      if (closeReader) reader.decRef();
+      if (closeReader) rawReader.decRef();
     } catch (Exception e) {
       SolrException.log(log, "Problem dec ref'ing reader", e);
     }

solr/core/src/test/org/apache/solr/request/TestFaceting.java

@@ -22,6 +22,9 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Random;
 
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.uninverting.DocTermOrds;
@@ -81,12 +84,11 @@ public class TestFaceting extends SolrTestCaseJ4 {
     createIndex(size);
     req = lrf.makeRequest("q","*:*");
 
-    UnInvertedField uif = new UnInvertedField(proto.field(), req.getSearcher());
+    SortedSetDocValues dv = DocValues.getSortedSet(req.getSearcher().getAtomicReader(), proto.field());
 
-    assertEquals(size, uif.getNumTerms());
+    assertEquals(size, dv.getValueCount());
 
-    TermsEnum te = uif.getOrdTermsEnum(req.getSearcher().getAtomicReader());
+    TermsEnum te = dv.termsEnum();
-    assertEquals(size == 0, te == null);
 
     Random r = new Random(size);
     // test seeking by term string
@@ -763,16 +765,16 @@ public class TestFaceting extends SolrTestCaseJ4 {
     RefCounted<SolrIndexSearcher> currentSearcherRef = h.getCore().getSearcher();
     try {
       SolrIndexSearcher currentSearcher = currentSearcherRef.get();
-      UnInvertedField ui0 = UnInvertedField.getUnInvertedField("f0_ws", currentSearcher);
+      SortedSetDocValues ui0 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f0_ws");
-      UnInvertedField ui1 = UnInvertedField.getUnInvertedField("f1_ws", currentSearcher);
+      SortedSetDocValues ui1 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f1_ws");
-      UnInvertedField ui2 = UnInvertedField.getUnInvertedField("f2_ws", currentSearcher);
+      SortedSetDocValues ui2 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f2_ws");
-      UnInvertedField ui3 = UnInvertedField.getUnInvertedField("f3_ws", currentSearcher);
+      SortedSetDocValues ui3 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f3_ws");
-      UnInvertedField ui4 = UnInvertedField.getUnInvertedField("f4_ws", currentSearcher);
+      SortedSetDocValues ui4 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f4_ws");
-      UnInvertedField ui5 = UnInvertedField.getUnInvertedField("f5_ws", currentSearcher);
+      SortedSetDocValues ui5 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f5_ws");
-      UnInvertedField ui6 = UnInvertedField.getUnInvertedField("f6_ws", currentSearcher);
+      SortedSetDocValues ui6 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f6_ws");
-      UnInvertedField ui7 = UnInvertedField.getUnInvertedField("f7_ws", currentSearcher);
+      SortedSetDocValues ui7 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f7_ws");
-      UnInvertedField ui8 = UnInvertedField.getUnInvertedField("f8_ws", currentSearcher);
+      SortedSetDocValues ui8 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f8_ws");
-      UnInvertedField ui9 = UnInvertedField.getUnInvertedField("f9_ws", currentSearcher);
+      SortedSetDocValues ui9 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f9_ws");
 
       assertQ("check threading, more threads than fields",
           req("q", "id:*", "indent", "true", "fl", "id", "rows", "1"
@@ -924,28 +926,39 @@ public class TestFaceting extends SolrTestCaseJ4 {
       // Now, are all the UnInvertedFields still the same? Meaning they weren't re-fetched even when a bunch were
       // requested at the same time?
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui0, UnInvertedField.getUnInvertedField("f0_ws", currentSearcher));
+          ui0, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f0_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui1, UnInvertedField.getUnInvertedField("f1_ws", currentSearcher));
+          ui1, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f1_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui2, UnInvertedField.getUnInvertedField("f2_ws", currentSearcher));
+          ui2, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f2_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui3, UnInvertedField.getUnInvertedField("f3_ws", currentSearcher));
+          ui3, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f3_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui4, UnInvertedField.getUnInvertedField("f4_ws", currentSearcher));
+          ui4, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f4_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui5, UnInvertedField.getUnInvertedField("f5_ws", currentSearcher));
+          ui5, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f5_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui6, UnInvertedField.getUnInvertedField("f6_ws", currentSearcher));
+          ui6, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f6_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui7, UnInvertedField.getUnInvertedField("f7_ws", currentSearcher));
+          ui7, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f7_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui8, UnInvertedField.getUnInvertedField("f8_ws", currentSearcher));
+          ui8, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f8_ws"));
       assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
-          ui9, UnInvertedField.getUnInvertedField("f9_ws", currentSearcher));
+          ui9, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f9_ws"));
     } finally {
       currentSearcherRef.decref();
     }
   }
+  
+  // assert same instance: either same object, or both wrapping same single-valued object
+  private void assertEquals(String msg, SortedSetDocValues dv1, SortedSetDocValues dv2) {
+    SortedDocValues singleton1 = DocValues.unwrapSingleton(dv1);
+    SortedDocValues singleton2 = DocValues.unwrapSingleton(dv2);
+    if (singleton1 == null || singleton2 == null) {
+      assertSame(dv1, dv2);
+    } else {
+      assertSame(singleton1, singleton2);
+    }
+  }
 }
 

solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java

@@ -16,6 +16,7 @@
  */
 package org.apache.solr.search;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LogDocMergePolicy;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReaderContext;
@@ -71,7 +72,7 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
     assertU(commit());
 
     SolrQueryRequest sr1 = req("q","foo");
-    IndexReaderContext rCtx1 = sr1.getSearcher().getTopReaderContext();
+    IndexReader r1 = sr1.getSearcher().getRawReader();
 
     String sval1 = getStringVal(sr1, "v_s1",0);
     assertEquals("string1", sval1);
@@ -81,28 +82,28 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
     assertU(commit());
 
     SolrQueryRequest sr2 = req("q","foo");
-    IndexReaderContext rCtx2 = sr2.getSearcher().getTopReaderContext();
+    IndexReader r2 = sr2.getSearcher().getRawReader();
 
     // make sure the readers share the first segment
     // Didn't work w/ older versions of lucene2.9 going from segment -> multi
-    assertEquals(rCtx1.leaves().get(0).reader(), rCtx2.leaves().get(0).reader());
+    assertEquals(r1.leaves().get(0).reader(), r2.leaves().get(0).reader());
 
     assertU(adoc("id","5", "v_f","3.14159"));
     assertU(adoc("id","6", "v_f","8983", "v_s1","string6"));
     assertU(commit());
 
     SolrQueryRequest sr3 = req("q","foo");
-    IndexReaderContext rCtx3 = sr3.getSearcher().getTopReaderContext();
+    IndexReader r3 = sr3.getSearcher().getRawReader();
     // make sure the readers share segments
     // assertEquals(r1.getLeafReaders()[0], r3.getLeafReaders()[0]);
-    assertEquals(rCtx2.leaves().get(0).reader(), rCtx3.leaves().get(0).reader());
+    assertEquals(r2.leaves().get(0).reader(), r3.leaves().get(0).reader());
-    assertEquals(rCtx2.leaves().get(1).reader(), rCtx3.leaves().get(1).reader());
+    assertEquals(r2.leaves().get(1).reader(), r3.leaves().get(1).reader());
 
     sr1.close();
     sr2.close();            
 
     // should currently be 1, but this could change depending on future index management
-    int baseRefCount = rCtx3.reader().getRefCount();
+    int baseRefCount = r3.getRefCount();
     assertEquals(1, baseRefCount);
 
     Object sr3SearcherRegAt = sr3.getSearcher().getStatistics().get("registeredAt");
@@ -112,7 +113,7 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
                sr3.getSearcher(), sr4.getSearcher());
     assertEquals("nothing changed, searcher should not have been re-registered",
                  sr3SearcherRegAt, sr4.getSearcher().getStatistics().get("registeredAt"));
-    IndexReaderContext rCtx4 = sr4.getSearcher().getTopReaderContext();
+    IndexReader r4 = sr4.getSearcher().getRawReader();
 
     // force an index change so the registered searcher won't be the one we are testing (and
     // then we should be able to test the refCount going all the way to 0
@@ -120,12 +121,12 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
     assertU(commit()); 
 
     // test that reader didn't change
-    assertSame(rCtx3.reader(), rCtx4.reader());
+    assertSame(r3, r4);
-    assertEquals(baseRefCount, rCtx4.reader().getRefCount());
+    assertEquals(baseRefCount, r4.getRefCount());
     sr3.close();
-    assertEquals(baseRefCount, rCtx4.reader().getRefCount());
+    assertEquals(baseRefCount, r4.getRefCount());
     sr4.close();
-    assertEquals(baseRefCount-1, rCtx4.reader().getRefCount());
+    assertEquals(baseRefCount-1, r4.getRefCount());
 
 
     SolrQueryRequest sr5 = req("q","foo");