LUCENE-3661: simplify del count tracking during merge

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1237114 13f79535-47bb-0310-9956-ffa450edef68

lucene/src/java/org/apache/lucene/index/IndexWriter.java

@@ -447,6 +447,24 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
       return rc > myRefCounts;
     }
 
+    // Call only from assert!
+    public synchronized boolean verifyDocCounts() {
+      int count;
+      if (liveDocs != null) {
+        count = 0;
+        for(int docID=0;docID<info.docCount;docID++) {
+          if (liveDocs.get(docID)) {
+            count++;
+          }
+        }
+      } else {
+        count = info.docCount;
+      }
+
+      assert info.docCount - info.getDelCount() - pendingDeleteCount == count: "info.docCount=" + info.docCount + " info.getDelCount()=" + info.getDelCount() + " pendingDeleteCount=" + pendingDeleteCount + " count=" + count;;
+      return true;
+    }
+
     // Returns true if any reader remains
     public synchronized boolean removeReader(SegmentReader sr, boolean drop) throws IOException {
       if (sr == reader) {
@@ -517,6 +535,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     public synchronized boolean delete(int docID) {
       assert liveDocs != null;
       assert docID >= 0 && docID < liveDocs.length();
+      assert !shared;
       final boolean didDelete = liveDocs.get(docID);
       if (didDelete) {
         liveDocs.clear(docID);
@@ -2981,7 +3000,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
    *  saves the resulting deletes file (incrementing the
    *  delete generation for merge.info).  If no deletes were
    *  flushed, no new deletes file is saved. */
-  synchronized private ReadersAndLiveDocs commitMergedDeletes(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
+  synchronized private ReadersAndLiveDocs commitMergedDeletes(MergePolicy.OneMerge merge) throws IOException {
 
     assert testPoint("startCommitMergeDeletes");
 
@@ -3048,7 +3067,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
             }
           }
         } else {
-          docUpto += mergeState.segmentDocCounts.get(info);
+          docUpto += info.docCount - info.getDelCount() - rld.pendingDeleteCount;
         }
       } else if (currentLiveDocs != null) {
         // This segment had no deletes before but now it
@@ -3089,7 +3108,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     return mergedDeletes;
   }
 
-  synchronized private boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
+  synchronized private boolean commitMerge(MergePolicy.OneMerge merge) throws IOException {
 
     assert testPoint("startCommitMerge");
 
@@ -3116,7 +3135,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
       return false;
     }
 
-    final ReadersAndLiveDocs mergedDeletes = commitMergedDeletes(merge, mergeState);
+    final ReadersAndLiveDocs mergedDeletes = commitMergedDeletes(merge);
 
     assert mergedDeletes == null || mergedDeletes.pendingDeleteCount != 0;
 
@@ -3577,6 +3596,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
           // we pull a copy:
           liveDocs = rld.getReadOnlyLiveDocs();
 
+          assert rld.verifyDocCounts();
+
           if (infoStream.isEnabled("IW")) {
             if (rld.pendingDeleteCount != 0) {
               infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount() + " pendingDelCount=" + rld.pendingDeleteCount);
@@ -3589,7 +3610,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
         }
         merge.readerLiveDocs.add(liveDocs);
         merge.readers.add(reader);
-        merger.add(reader, liveDocs);
+        final int delCount = rld.pendingDeleteCount + info.getDelCount();
+        assert delCount <= info.docCount;
+        if (delCount < info.docCount) {
+          merger.add(reader, liveDocs);
+        }
         segUpto++;
       }
 
@@ -3691,7 +3716,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
 
       // Force READ context because we merge deletes onto
       // this reader:
-      if (!commitMerge(merge, mergeState)) {
+      if (!commitMerge(merge)) {
         // commitMerge will return false if this merge was aborted
         return 0;
       }

lucene/src/java/org/apache/lucene/index/MergeState.java

@@ -18,7 +18,6 @@ package org.apache.lucene.index;
  */
 
 import java.util.List;
-import java.util.Map;
 
 import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
 import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor;
@@ -45,7 +44,6 @@ public class MergeState {
   public List<IndexReaderAndLiveDocs> readers;        // Readers & liveDocs being merged
   public int[][] docMaps;                             // Maps docIDs around deletions
   public int[] docBase;                               // New docID base per reader
-  public Map<SegmentInfo,Integer> segmentDocCounts;   // Non-deleted docCount per reader
   public int mergedDocCount;                          // Total # merged docs
   public CheckAbort checkAbort;
   public InfoStream infoStream;

lucene/src/java/org/apache/lucene/index/SegmentMerger.java

@@ -279,18 +279,6 @@ final class SegmentMerger {
     }
   }
 
-  private int[] shrink(int[] in, int size) {
-    final int[] newArray = new int[size];
-    System.arraycopy(in, 0, newArray, 0, size);
-    return newArray;
-  }
-
-  private int[][] shrink(int[][] in, int size) {
-    final int[][] newArray = new int[size][];
-    System.arraycopy(in, 0, newArray, 0, size);
-    return newArray;
-  }
-
   // NOTE: removes any "all deleted" readers from mergeState.readers
   private int setDocMaps() throws IOException {
     final int numReaders = mergeState.readers.size();
@@ -298,7 +286,6 @@ final class SegmentMerger {
     // Remap docIDs
     mergeState.docMaps = new int[numReaders][];
     mergeState.docBase = new int[numReaders];
-    mergeState.segmentDocCounts = new HashMap<SegmentInfo,Integer>();
     mergeState.dirPayloadProcessor = new PayloadProcessorProvider.DirPayloadProcessor[numReaders];
     mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders];
 
@@ -332,16 +319,6 @@ final class SegmentMerger {
         docMap = null;
       }
 
-      if (reader.reader instanceof SegmentReader) {
-        mergeState.segmentDocCounts.put(((SegmentReader) reader.reader).getSegmentInfo(), docCount);
-      }
-
-      if (docCount == 0) {
-        // Skip this reader (all docs are deleted):
-        mergeState.readers.remove(i);
-        continue;
-      }
-
       mergeState.docMaps[i] = docMap;
       docBase += docCount;
 
@@ -352,13 +329,6 @@ final class SegmentMerger {
       i++;
     }
 
-    final int numReadersLeft = mergeState.readers.size();
-
-    if (numReadersLeft < mergeState.docMaps.length) {
-      mergeState.docMaps = shrink(mergeState.docMaps, numReadersLeft);
-      mergeState.docBase = shrink(mergeState.docBase, numReadersLeft);
-    }
-    
     return docBase;
   }