Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3661: dump my current state 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1233479 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-01-19 17:16:25 +00:00
parent 5280476fcb
commit de495a3c0d
18 changed files with 160 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/src/java/org/apache/lucene/codecs/Codec.java
View File

@ -75,6 +75,9 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
/** Encodes/decodes document normalization values */
public abstract NormsFormat normsFormat();
/** Encodes/decodes live docs */
public abstract LiveDocsFormat liveDocsFormat();
/** looks up a codec by name */
public static Codec forName(String name) {
return loader.lookup(name);

34
lucene/src/java/org/apache/lucene/codecs/LiveDocsFormat.java Normal file
View File

@ -0,0 +1,34 @@
package org.apache.lucene.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MutableBits;
public abstract class LiveDocsFormat {
public abstract MutableBits newLiveDocs(int size) throws IOException;
public abstract Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException;
public abstract void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException;
public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException;
}

8
lucene/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs.appending;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
@ -28,6 +29,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40Codec;
import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
@ -50,6 +52,7 @@ public class AppendingCodec extends Codec {
private final TermVectorsFormat vectors = new Lucene40TermVectorsFormat();
private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
private final NormsFormat norms = new Lucene40NormsFormat();
private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat();
@Override
public PostingsFormat postingsFormat() {
@ -85,4 +88,9 @@ public class AppendingCodec extends Codec {
public NormsFormat normsFormat() {
return norms;
}
@Override
public LiveDocsFormat liveDocsFormat() {
return liveDocs;
}
}

10
lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java
View File

@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.PerDocProducer;
@ -30,6 +31,7 @@ import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
@ -57,6 +59,9 @@ public class Lucene3xCodec extends Codec {
private final NormsFormat normsFormat = new Lucene3xNormsFormat();
// TODO: this should really be a different impl
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
// 3.x doesn't support docvalues
private final DocValuesFormat docValuesFormat = new DocValuesFormat() {
@Override
@ -107,4 +112,9 @@ public class Lucene3xCodec extends Codec {
public NormsFormat normsFormat() {
return normsFormat;
}
@Override
public LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
}

6
lucene/src/java/org/apache/lucene/util/BitVector.java → lucene/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.util;
package org.apache.lucene.codecs.lucene40;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -25,6 +25,8 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.MutableBits;
/** Optimized implementation of a vector of bits. This is more-or-less like
* java.util.BitSet, but also includes the following:
@ -37,7 +39,7 @@ import org.apache.lucene.store.IndexOutput;
*
* @lucene.internal
*/
public final class BitVector implements Cloneable, Bits {
public final class BitVector implements Cloneable, MutableBits {
private byte[] bits;
private int size;

8
lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene40;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
@ -42,6 +43,8 @@ public class Lucene40Codec extends Codec {
private final DocValuesFormat docValuesFormat = new Lucene40DocValuesFormat();
private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat();
private final NormsFormat normsFormat = new Lucene40NormsFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
@ -87,6 +90,11 @@ public class Lucene40Codec extends Codec {
public NormsFormat normsFormat() {
return normsFormat;
}
@Override
public LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.

42
lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java Normal file
View File

@ -0,0 +1,42 @@
package org.apache.lucene.codecs.lucene40;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MutableBits;
public class Lucene40LiveDocsFormat extends LiveDocsFormat {
@Override
public MutableBits newLiveDocs(int size) throws IOException {
BitVector bitVector = new BitVector(size);
bitVector.invertAll();
return bitVector;
}
@Override
public Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException {
// nocommit: compute filename here
return new BitVector(dir, info.getDelFileName(), context);
}
@Override
public void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException {
// nocommit: compute filename here
// nocommit: this api is ugly...
((BitVector)bits).write(dir, info.getDelFileName(), context);
}
@Override
public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
// nocommit: compute filename here
if (info.hasDeletions()) {
files.add(info.getDelFileName());
}
}
}

9
lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java
View File

@ -20,12 +20,14 @@ package org.apache.lucene.codecs.simpletext;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
/**
* plain text index format.
@ -43,6 +45,8 @@ public final class SimpleTextCodec extends Codec {
private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
// TODO: need a plain-text impl (using the above)
private final NormsFormat normsFormat = new SimpleTextNormsFormat();
// TODO: need a plain-text impl
private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat();
public SimpleTextCodec() {
super("SimpleText");
@ -82,4 +86,9 @@ public final class SimpleTextCodec extends Codec {
public NormsFormat normsFormat() {
return normsFormat;
}
@Override
public LiveDocsFormat liveDocsFormat() {
return liveDocs;
}
}

2
lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -25,12 +25,12 @@ import java.text.NumberFormat;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.ByteBlockPool.Allocator;
import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;

2
lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
View File

@ -27,8 +27,8 @@ import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;

2
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -34,6 +34,7 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@ -48,7 +49,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;

2
lucene/src/java/org/apache/lucene/index/MergePolicy.java
View File

@ -22,9 +22,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.SetOnce.AlreadySetException;
import org.apache.lucene.util.SetOnce;

8
lucene/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -23,9 +23,9 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
/**
@ -36,7 +36,7 @@ public final class SegmentReader extends IndexReader {
private final SegmentInfo si;
private final ReaderContext readerContext = new AtomicReaderContext(this);
private final BitVector liveDocs;
private final Bits liveDocs;
// Normally set to si.docCount - si.delDocCount, unless we
// were created as an NRT reader from IW, in which case IW
@ -56,7 +56,7 @@ public final class SegmentReader extends IndexReader {
try {
if (si.hasDeletions()) {
// NOTE: the bitvector is stored using the regular directory, not cfs
liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(IOContext.READ, true));
liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
} else {
assert si.getDelCount() == 0;
liveDocs = null;
@ -124,7 +124,9 @@ public final class SegmentReader extends IndexReader {
return liveDocs;
}
// nocommit
private boolean checkLiveCounts(boolean isNRT) throws IOException {
BitVector liveDocs = (BitVector) this.liveDocs;
if (liveDocs != null) {
if (liveDocs.size() != si.docCount) {
throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name);

2
lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
View File

@ -18,9 +18,9 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene40.BitVector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.InfoStream;
/**

22
lucene/src/java/org/apache/lucene/util/MutableBits.java Normal file
View File

@ -0,0 +1,22 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public interface MutableBits extends Bits {
}

4
lucene/src/test/org/apache/lucene/util/TestBitVector.java → lucene/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.util;
package org.apache.lucene.codecs.lucene40;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -22,6 +22,8 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
/**
* <code>TestBitVector</code> tests the <code>BitVector</code>, obviously.

7
lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
View File

@ -27,6 +27,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
@ -35,6 +36,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40Codec;
import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
@ -1156,6 +1158,11 @@ public class TestAddIndexes extends LuceneTestCase {
public NormsFormat normsFormat() {
return new Lucene40NormsFormat();
}
@Override
public LiveDocsFormat liveDocsFormat() {
return new Lucene40LiveDocsFormat();
}
}
/*

1
lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
View File

@ -42,7 +42,6 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;