LUCENE-140:

- Add 2 more checks on initializing SegmentReader that raise
    IllegalStateException if corruption is detected.  This would have
    caught the second cause in LUCENE-140 (incorrectly re-using old
    .del files) earlier.

  - Fixed bugs in two unit tests that tripped up on these new checks.

  - Fixed (tightened) one more boundary case (when lastDoc was 0) in
    the pre-existing "docs out of order" check in SegmentMerger.java.

  - Simplified the unit test I added to TestIndexReader to test this
    issue.


git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@494933 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/lucene/index/SegmentMerger.java

@@ -348,7 +348,7 @@ final class SegmentMerger {
           doc = docMap[doc];                      // map around deletions
         doc += base;                              // convert to merged space
 
-        if (lastDoc != 0 && doc <= lastDoc)
+        if (doc < 0 || (df > 0 && doc <= lastDoc))
           throw new IllegalStateException("docs out of order (" + doc +
               " <= " + lastDoc + " )");
 

src/java/org/apache/lucene/index/SegmentReader.java

@@ -159,11 +159,21 @@ class SegmentReader extends IndexReader {
       fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
       fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
 
+      // Verify two sources of "maxDoc" agree:
+      if (fieldsReader.size() != si.docCount) {
+        throw new IllegalStateException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.size() + " but segmentInfo shows " + si.docCount);
+      }
+
       tis = new TermInfosReader(cfsDir, segment, fieldInfos);
       
       // NOTE: the bitvector is stored using the regular directory, not cfs
       if (hasDeletions(si)) {
         deletedDocs = new BitVector(directory(), si.getDelFileName());
+
+        // Verify # deletes does not exceed maxDoc for this segment:
+        if (deletedDocs.count() > maxDoc()) {
+          throw new IllegalStateException("number of deletes (" + deletedDocs.count() + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name);
+        }
       }
 
       // make sure that all index files have been read or are kept open

src/test/org/apache/lucene/index/TestDoc.java

@@ -109,19 +109,19 @@ public class TestDoc extends TestCase {
       directory.close();
 
       indexDoc("one", "test.txt");
-      printSegment(out, "one");
+      printSegment(out, "one", 1);
 
       indexDoc("two", "test2.txt");
-      printSegment(out, "two");
+      printSegment(out, "two", 1);
 
       merge("one", 1, "two", 1, "merge", false);
-      printSegment(out, "merge");
+      printSegment(out, "merge", 2);
 
       merge("one", 1, "two", 1, "merge2", false);
-      printSegment(out, "merge2");
+      printSegment(out, "merge2", 2);
 
       merge("merge", 2, "merge2", 2, "merge3", false);
-      printSegment(out, "merge3");
+      printSegment(out, "merge3", 4);
 
       out.close();
       sw.close();
@@ -135,19 +135,19 @@ public class TestDoc extends TestCase {
       directory.close();
 
       indexDoc("one", "test.txt");
-      printSegment(out, "one");
+      printSegment(out, "one", 1);
 
       indexDoc("two", "test2.txt");
-      printSegment(out, "two");
+      printSegment(out, "two", 1);
 
       merge("one", 1, "two", 1, "merge", true);
-      printSegment(out, "merge");
+      printSegment(out, "merge", 2);
 
       merge("one", 1, "two", 1, "merge2", true);
-      printSegment(out, "merge2");
+      printSegment(out, "merge2", 2);
 
       merge("merge", 2, "merge2", 2, "merge3", true);
-      printSegment(out, "merge3");
+      printSegment(out, "merge3", 4);
 
       out.close();
       sw.close();
@@ -199,11 +199,11 @@ public class TestDoc extends TestCase {
    }
 
 
-   private void printSegment(PrintWriter out, String segment)
+   private void printSegment(PrintWriter out, String segment, int docCount)
    throws Exception {
       Directory directory = FSDirectory.getDirectory(indexDir, false);
       SegmentReader reader =
-        SegmentReader.get(new SegmentInfo(segment, 1, directory));
+        SegmentReader.get(new SegmentInfo(segment, docCount, directory));
 
       for (int i = 0; i < reader.numDocs(); i++)
         out.println(reader.document(i));

src/test/org/apache/lucene/index/TestIndexReader.java

@@ -777,12 +777,11 @@ public class TestIndexReader extends TestCase
         addDoc(writer, "aaa");
       }
 
-      try {
-        writer.optimize();
-      } catch (IllegalStateException e) {
-        e.printStackTrace();
-        fail("hit unexpected illegal state exception during optimize");
-      }
+      // Without the fix for LUCENE-140 this call will
+      // [incorrectly] hit a "docs out of order"
+      // IllegalStateException because above out-of-bounds
+      // deleteDocument corrupted the index:
+      writer.optimize();
 
       if (!gotException) {
         fail("delete of out-of-bounds doc number failed to hit exception");

src/test/org/apache/lucene/index/TestSegmentTermDocs.java

@@ -68,7 +68,7 @@ public class TestSegmentTermDocs extends TestCase {
   public void testBadSeek() throws IOException {
     {
       //After adding the document, we should be able to read it back in
-      SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 3, dir));
+      SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 1, dir));
       assertTrue(reader != null);
       SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
       assertTrue(segTermDocs != null);
@@ -78,7 +78,7 @@ public class TestSegmentTermDocs extends TestCase {
     }
     {
       //After adding the document, we should be able to read it back in
-      SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 3, dir));
+      SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 1, dir));
       assertTrue(reader != null);
       SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
       assertTrue(segTermDocs != null);