LUCENE-3661: dump my current state

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1233479 13f79535-47bb-0310-9956-ffa450edef68

lucene/src/java/org/apache/lucene/codecs/Codec.java

@@ -75,6 +75,9 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
   /** Encodes/decodes document normalization values */
   public abstract NormsFormat normsFormat();
   
+  /** Encodes/decodes live docs */
+  public abstract LiveDocsFormat liveDocsFormat();
+  
   /** looks up a codec by name */
   public static Codec forName(String name) {
     return loader.lookup(name);

lucene/src/java/org/apache/lucene/codecs/LiveDocsFormat.java

@@ -0,0 +1,34 @@
+package org.apache.lucene.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.MutableBits;
+
+public abstract class LiveDocsFormat {
+  public abstract MutableBits newLiveDocs(int size) throws IOException;
+  public abstract Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException;
+  public abstract void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException;
+  public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException;
+}

lucene/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java

@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.appending;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfosFormat;
@@ -28,6 +29,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40Codec;
 import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
@@ -50,6 +52,7 @@ public class AppendingCodec extends Codec {
   private final TermVectorsFormat vectors = new Lucene40TermVectorsFormat();
   private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
   private final NormsFormat norms = new Lucene40NormsFormat();
+  private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat();
   
   @Override
   public PostingsFormat postingsFormat() {
@@ -85,4 +88,9 @@ public class AppendingCodec extends Codec {
   public NormsFormat normsFormat() {
     return norms;
   }
+  
+  @Override
+  public LiveDocsFormat liveDocsFormat() {
+    return liveDocs;
+  }
 }

lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java

@@ -23,6 +23,7 @@ import java.util.Set;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.PerDocConsumer;
 import org.apache.lucene.codecs.PerDocProducer;
@@ -30,6 +31,7 @@ import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfosFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
 import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
@@ -57,6 +59,9 @@ public class Lucene3xCodec extends Codec {
   
   private final NormsFormat normsFormat = new Lucene3xNormsFormat();
   
+  // TODO: this should really be a different impl
+  private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+  
   // 3.x doesn't support docvalues
   private final DocValuesFormat docValuesFormat = new DocValuesFormat() {
     @Override
@@ -107,4 +112,9 @@ public class Lucene3xCodec extends Codec {
   public NormsFormat normsFormat() {
     return normsFormat;
   }
+  
+  @Override
+  public LiveDocsFormat liveDocsFormat() {
+    return liveDocsFormat;
+  }
 }

lucene/src/java/org/apache/lucene/codecs/lucene40/BitVector.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.util;
+package org.apache.lucene.codecs.lucene40;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,6 +25,8 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.MutableBits;
 
 /** Optimized implementation of a vector of bits.  This is more-or-less like
  *  java.util.BitSet, but also includes the following:
@@ -37,7 +39,7 @@ import org.apache.lucene.store.IndexOutput;
  *
  *  @lucene.internal
  */
-public final class BitVector implements Cloneable, Bits {
+public final class BitVector implements Cloneable, MutableBits {
 
   private byte[] bits;
   private int size;

lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java

@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene40;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfosFormat;
@@ -42,6 +43,8 @@ public class Lucene40Codec extends Codec {
   private final DocValuesFormat docValuesFormat = new Lucene40DocValuesFormat();
   private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat();
   private final NormsFormat normsFormat = new Lucene40NormsFormat();
+  private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+  
   private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
     @Override
     public PostingsFormat getPostingsFormatForField(String field) {
@@ -87,6 +90,11 @@ public class Lucene40Codec extends Codec {
   public NormsFormat normsFormat() {
     return normsFormat;
   }
+  
+  @Override
+  public LiveDocsFormat liveDocsFormat() {
+    return liveDocsFormat;
+  }
 
   /** Returns the postings format that should be used for writing 
    *  new segments of <code>field</code>.

lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java

@@ -0,0 +1,42 @@
+package org.apache.lucene.codecs.lucene40;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.MutableBits;
+
+public class Lucene40LiveDocsFormat extends LiveDocsFormat {
+
+  @Override
+  public MutableBits newLiveDocs(int size) throws IOException {
+    BitVector bitVector = new BitVector(size);
+    bitVector.invertAll();
+    return bitVector;
+  }
+
+  @Override
+  public Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException {
+    // nocommit: compute filename here
+    return new BitVector(dir, info.getDelFileName(), context);
+  }
+
+  @Override
+  public void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException {
+    // nocommit: compute filename here
+    // nocommit: this api is ugly...
+    ((BitVector)bits).write(dir, info.getDelFileName(), context);
+  }
+
+  @Override
+  public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+    // nocommit: compute filename here
+    if (info.hasDeletions()) {
+      files.add(info.getDelFileName());
+    }
+  }
+}

lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java

@@ -20,12 +20,14 @@ package org.apache.lucene.codecs.simpletext;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfosFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
 
 /**
  * plain text index format.
@@ -43,6 +45,8 @@ public final class SimpleTextCodec extends Codec {
   private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
   // TODO: need a plain-text impl (using the above)
   private final NormsFormat normsFormat = new SimpleTextNormsFormat();
+  // TODO: need a plain-text impl
+  private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat();
   
   public SimpleTextCodec() {
     super("SimpleText");
@@ -82,4 +86,9 @@ public final class SimpleTextCodec extends Codec {
   public NormsFormat normsFormat() {
     return normsFormat;
   }
+  
+  @Override
+  public LiveDocsFormat liveDocsFormat() {
+    return liveDocs;
+  }
 }

lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java

@@ -25,12 +25,12 @@ import java.text.NumberFormat;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
 import org.apache.lucene.search.similarities.SimilarityProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FlushInfo;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Counter;
 import org.apache.lucene.util.ByteBlockPool.Allocator;
 import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;

lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java

@@ -27,8 +27,8 @@ import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.PostingsConsumer;
 import org.apache.lucene.codecs.TermStats;
 import org.apache.lucene.codecs.TermsConsumer;
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.RamUsageEstimator;

lucene/src/java/org/apache/lucene/index/IndexWriter.java

@@ -34,6 +34,7 @@ import java.util.regex.Pattern;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
 import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -48,7 +49,6 @@ import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.InfoStream;

lucene/src/java/org/apache/lucene/index/MergePolicy.java

@@ -22,9 +22,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.SetOnce.AlreadySetException;
 import org.apache.lucene.util.SetOnce;
 

lucene/src/java/org/apache/lucene/index/SegmentReader.java

@@ -23,9 +23,9 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.codecs.PerDocProducer;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.search.FieldCache; // javadocs
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Bits;
 
 /**
@@ -36,7 +36,7 @@ public final class SegmentReader extends IndexReader {
   private final SegmentInfo si;
   private final ReaderContext readerContext = new AtomicReaderContext(this);
   
-  private final BitVector liveDocs;
+  private final Bits liveDocs;
 
   // Normally set to si.docCount - si.delDocCount, unless we
   // were created as an NRT reader from IW, in which case IW
@@ -56,7 +56,7 @@ public final class SegmentReader extends IndexReader {
     try {
       if (si.hasDeletions()) {
         // NOTE: the bitvector is stored using the regular directory, not cfs
-        liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(IOContext.READ, true));
+        liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
       } else {
         assert si.getDelCount() == 0;
         liveDocs = null;
@@ -124,7 +124,9 @@ public final class SegmentReader extends IndexReader {
     return liveDocs;
   }
 
+  // nocommit
   private boolean checkLiveCounts(boolean isNRT) throws IOException {
+    BitVector liveDocs = (BitVector) this.liveDocs;
     if (liveDocs != null) {
       if (liveDocs.size() != si.docCount) {
         throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name);

lucene/src/java/org/apache/lucene/index/SegmentWriteState.java

@@ -18,9 +18,9 @@ package org.apache.lucene.index;
  */
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.BitVector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.InfoStream;
 
 /**

lucene/src/java/org/apache/lucene/util/MutableBits.java

@@ -0,0 +1,22 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public interface MutableBits extends Bits {
+  
+}

lucene/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.util;
+package org.apache.lucene.codecs.lucene40;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,6 +22,8 @@ import java.io.IOException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 /**
  * <code>TestBitVector</code> tests the <code>BitVector</code>, obviously.

lucene/src/test/org/apache/lucene/index/TestAddIndexes.java

@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfosFormat;
@@ -35,6 +36,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40Codec;
 import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfosFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
@@ -1156,6 +1158,11 @@ public class TestAddIndexes extends LuceneTestCase {
     public NormsFormat normsFormat() {
       return new Lucene40NormsFormat();
     }
+    
+    @Override
+    public LiveDocsFormat liveDocsFormat() {
+      return new Lucene40LiveDocsFormat();
+    }
   }
   
   /*

lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java

@@ -42,7 +42,6 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;