From cd2bf59fdf7aca0344a05d06589067b367c237a1 Mon Sep 17 00:00:00 2001
From: Michael McCandless <mikemccand@apache.org>
Date: Sat, 21 Jul 2012 13:31:16 +0000
Subject: [PATCH] LUCENE-3892: turn on all options testing in
 TestPostingsFormat; fix it to skip offsets if the PF can't index them

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1364085 13f79535-47bb-0310-9956-ffa450edef68
---
 .../lucene/index/TestPostingsFormat.java      | 69 ++++++++++++-------
 .../analysis/BaseTokenStreamTestCase.java     | 10 ---
 .../apache/lucene/util/LuceneTestCase.java    |  9 +++
 3 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
index 746e1e979e7..ff7c6c00862 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
@@ -330,15 +330,25 @@ public class TestPostingsFormat extends LuceneTestCase {
       System.out.println("\nTEST: now build index");
     }
 
+    int maxIndexOptionNoOffsets = Arrays.asList(IndexOptions.values()).indexOf(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+
     // nocommit use allowPayloads
 
     FieldInfo[] newFieldInfoArray = new FieldInfo[fields.size()];
     for(int fieldUpto=0;fieldUpto<fields.size();fieldUpto++) {
       FieldInfo oldFieldInfo = fieldInfos.fieldInfo(fieldUpto);
 
+      String pf = _TestUtil.getPostingsFormat(oldFieldInfo.name);
+      int fieldMaxIndexOption;
+      if (doesntSupportOffsets.contains(pf)) {
+        fieldMaxIndexOption = Math.min(maxIndexOptionNoOffsets, maxIndexOption);
+      } else {
+        fieldMaxIndexOption = maxIndexOption;
+      }
+    
       // Randomly picked the IndexOptions to index this
       // field with:
-      IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+maxIndexOption)];
+      IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+fieldMaxIndexOption)];
       boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
 
       newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.name,
@@ -366,12 +376,13 @@ public class TestPostingsFormat extends LuceneTestCase {
       Map<BytesRef,List<Posting>> terms = fieldEnt.getValue();
 
       FieldInfo fieldInfo = newFieldInfos.fieldInfo(field);
-      if (VERBOSE) {
-        System.out.println("field=" + field);
-      }
 
       IndexOptions indexOptions = fieldInfo.getIndexOptions();
 
+      if (VERBOSE) {
+        System.out.println("field=" + field + " indexOtions=" + indexOptions);
+      }
+
       boolean doFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
       boolean doPos = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
       boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
@@ -413,6 +424,7 @@ public class TestPostingsFormat extends LuceneTestCase {
           } else {
             totalTF++;
           }
+          postingsConsumer.finishDoc();
           docCount++;
         }
         termsConsumer.finishTerm(term, new TermStats(postings.size(), totalTF));
@@ -539,9 +551,9 @@ public class TestPostingsFormat extends LuceneTestCase {
       docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum;
     }
 
-    assertNotNull(docsEnum);
+    assertNotNull("null DocsEnum", docsEnum);
     int initialDocID = docsEnum.docID();
-    assertTrue(initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
+    assertTrue("inital docID should be -1 or NO_MORE_DOCS", initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
 
     if (VERBOSE) {
       if (prevDocsEnum == null) {
@@ -600,10 +612,10 @@ public class TestPostingsFormat extends LuceneTestCase {
     while (nextPosting <= stopAt) {
       if (nextPosting == stopAt) {
         if (stopAt == expected.size()) {
-          assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
+          assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
 
           // Common bug is to forget to set this.doc=NO_MORE_DOCS in the enum!:
-          assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.docID());
+          assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.docID());
         }
         break;
       }
@@ -631,7 +643,7 @@ public class TestPostingsFormat extends LuceneTestCase {
           if (VERBOSE) {
             System.out.println("  now advance to end (target=" + target + ")");
           }
-          assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
+          assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
           break;
         } else {
           posting = expected.get(nextPosting++);
@@ -643,7 +655,7 @@ public class TestPostingsFormat extends LuceneTestCase {
             }
           }
           int docID = docsEnum.advance(targetDocID != -1 ? targetDocID : posting.docID);
-          assertEquals(posting.docID, docID);
+          assertEquals("docID is wrong", posting.docID, docID);
         }
       } else {
         posting = expected.get(nextPosting++);
@@ -651,7 +663,7 @@ public class TestPostingsFormat extends LuceneTestCase {
           System.out.println("  now nextDoc to " + posting.docID + " (" + nextPosting + " of " + stopAt + ")");
         }
         int docID = docsEnum.nextDoc();
-        assertEquals(posting.docID, docID);
+        assertEquals("docID is wrong", posting.docID, docID);
       }
 
       if (doCheckFreqs && random().nextDouble() <= freqAskChance) {
@@ -659,7 +671,7 @@ public class TestPostingsFormat extends LuceneTestCase {
           System.out.println("    now freq()=" + posting.positions.size());
         }
         int freq = docsEnum.freq();
-        assertEquals(posting.positions.size(), freq);
+        assertEquals("freq is wrong", posting.positions.size(), freq);
       }
 
       if (doCheckPositions) {
@@ -676,7 +688,7 @@ public class TestPostingsFormat extends LuceneTestCase {
           if (VERBOSE) {
             System.out.println("    now nextPosition to " + position.position);
           }
-          assertEquals(position.position, docsAndPositionsEnum.nextPosition());
+          assertEquals("position is wrong", position.position, docsAndPositionsEnum.nextPosition());
 
           // nocommit sometimes don't pull the payload even
           // though we pulled the position
@@ -687,17 +699,18 @@ public class TestPostingsFormat extends LuceneTestCase {
                 System.out.println("      now check payload length=" + (position.payload == null ? 0 : position.payload.length));
               }
               if (position.payload == null || position.payload.length == 0) {
-                assertFalse(docsAndPositionsEnum.hasPayload());
+                assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
               } else {
-                assertTrue(docsAndPositionsEnum.hasPayload());
+                assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
 
                 BytesRef payload = docsAndPositionsEnum.getPayload();
-                assertFalse(docsAndPositionsEnum.hasPayload());
+                assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
 
-                assertNotNull(payload);
-                assertEquals(position.payload.length, payload.length);
+                assertNotNull("payload should not be null", payload);
+                assertEquals("payload length is wrong", position.payload.length, payload.length);
                 for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
-                  assertEquals(position.payload[byteUpto],
+                  assertEquals("payload bytes are wrong",
+                               position.payload[byteUpto],
                                payload.bytes[payload.offset+byteUpto]);
                 }
               }
@@ -713,16 +726,19 @@ public class TestPostingsFormat extends LuceneTestCase {
               if (VERBOSE) {
                 System.out.println("      now check offsets: startOff=" + position.startOffset + " endOffset=" + position.endOffset);
               }
-              assertEquals(position.startOffset, docsAndPositionsEnum.startOffset());
-              assertEquals(position.endOffset, docsAndPositionsEnum.endOffset());
+              assertEquals("startOffset is wrong", position.startOffset, docsAndPositionsEnum.startOffset());
+              assertEquals("endOffset is wrong", position.endOffset, docsAndPositionsEnum.endOffset());
             } else {
               if (VERBOSE) {
                 System.out.println("      skip check offsets");
               }
             }
           } else {
-            assertEquals(-1, docsAndPositionsEnum.startOffset());
-            assertEquals(-1, docsAndPositionsEnum.endOffset());
+            if (VERBOSE) {
+              System.out.println("      now check offsets are -1");
+            }
+            assertEquals("startOffset isn't -1", -1, docsAndPositionsEnum.startOffset());
+            assertEquals("endOffset isn't -1", -1, docsAndPositionsEnum.endOffset());
           }
         }
       }
@@ -846,6 +862,7 @@ public class TestPostingsFormat extends LuceneTestCase {
 
     boolean indexPayloads = random().nextBoolean();
     // nocommit test thread safety of buildIndex too
+
     FieldsProducer fieldsProducer = buildIndex(dir, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads);
 
     //testTerms(fieldsProducer, EnumSet.noneOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -855,7 +872,11 @@ public class TestPostingsFormat extends LuceneTestCase {
     //testTerms(fieldsProducer, EnumSet.of(Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
     //testTerms(fieldsProducer, EnumSet.of(Option.THREADS, Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
     //testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-    testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+    //testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+
+    // NOTE: you can also test "weaker" index options than
+    // you indexed with:
+    testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
 
     fieldsProducer.close();
     dir.close();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
index 473046fb571..9d531e58b3a 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
@@ -473,16 +473,6 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
     }
   }
 
-  static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList( 
-    "Lucene3x",
-    "MockFixedIntBlock",
-    "MockVariableIntBlock",
-    "MockSep",
-    "MockRandom",
-    "For",
-    "PFor"
-  ));
-
   private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
 
     final LineFileDocs docs = new LineFileDocs(random);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index e9dec35c4d0..e195450d63c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -277,6 +277,15 @@ public abstract class LuceneTestCase extends Assert {
     CORE_DIRECTORIES.add("RAMDirectory");
   };
   
+  protected static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList( 
+    "Lucene3x",
+    "MockFixedIntBlock",
+    "MockVariableIntBlock",
+    "MockSep",
+    "MockRandom",
+    "For",
+    "PFor"
+  ));
   
   // -----------------------------------------------------------------
   // Fields initialized in class or instance rules.