Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3070: enable docvalues by default for every codec 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/docvalues@1099717 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2011-05-05 08:35:40 +00:00
parent 471cb20526
commit 9e77d3cdc6
23 changed files with 237 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
View File

@ -20,13 +20,19 @@ package org.apache.lucene.index.codecs.appending;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@ -133,10 +139,22 @@ public class AppendingCodec extends Codec {
StandardPostingsReader.files(dir, segmentInfo, codecIdAsString, files);
BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

2
lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;

2
lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
View File

@ -32,7 +32,7 @@ import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.store.Directory;

8
lucene/src/java/org/apache/lucene/index/codecs/Codec.java
View File

@ -52,13 +52,9 @@ public abstract class Codec {
* use; else, those files may be deleted. */
public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return null;
}
public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return null;
}
public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
/**
* Gathers files associated with this segment

84
lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodec.java → lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.index.codecs.docvalues;
package org.apache.lucene.index.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -16,87 +16,50 @@ package org.apache.lucene.index.codecs.docvalues;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Comparator;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.Writer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
/**
* A codec that adds DocValues support to a given codec transparently.
* @lucene.experimental
*/
public class DocValuesCodec extends Codec {
private final Codec other;
public class DefaultDocValuesConsumer extends PerDocConsumer {
private final String segmentName;
private final int codecId;
private final Directory directory;
private final AtomicLong bytesUsed;
private final Comparator<BytesRef> comparator;
public DocValuesCodec(Codec other, Comparator<BytesRef> comparator) {
this.name = other.name;
this.other = other;
public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator) {
this.segmentName = state.segmentName;
this.codecId = state.codecId;
this.bytesUsed = state.bytesUsed;
this.directory = state.directory;
this.comparator = comparator;
}
public DocValuesCodec(Codec other) {
this(other, null);
}
@Override
public PerDocConsumer docsConsumer(final PerDocWriteState state)
throws IOException {
return new PerDocConsumer() {
public void close() throws IOException {
}
@Override
public DocValuesConsumer addValuesField(FieldInfo field)
throws IOException {
final DocValuesConsumer consumer = Writer.create(field.getDocValues(),
docValuesId(state.segmentName, state.codecId, field.number),
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
return Writer.create(field.getDocValues(),
docValuesId(segmentName, codecId, field.number),
// TODO can we have a compound file per segment and codec for
// docvalues?
state.directory, comparator, state.bytesUsed);
return consumer;
}
};
directory, comparator, bytesUsed);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DocValuesProducerBase(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state)
throws IOException {
return other.fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state)
throws IOException {
return other.fieldsProducer(state);
}
static String docValuesId(String segmentsName, int codecID, int fieldId) {
return segmentsName + "_" + codecID + "-" + fieldId;
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId,
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
Set<String> files) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
boolean indexed = false;
@ -131,14 +94,13 @@ public class DocValuesCodec extends Codec {
}
}
if (indexed) {
other.files(dir, segmentInfo, codecId, files);
}
}
@Override
public void getExtensions(Set<String> extensions) {
other.getExtensions(extensions);
static String docValuesId(String segmentsName, int codecID, int fieldId) {
return segmentsName + "_" + codecID + "-" + fieldId;
}
public static void getDocValuesExtensions(Set<String> extensions) {
extensions.add(Writer.DATA_EXTENSION);
extensions.add(Writer.INDEX_EXTENSION);
}

10
lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java → lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.index.codecs.docvalues;
package org.apache.lucene.index.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -37,12 +37,12 @@ import org.apache.lucene.store.Directory;
*
* @lucene.experimental
*/
public class DocValuesProducerBase extends PerDocValues {
public class DefaultDocValuesProducer extends PerDocValues {
protected final TreeMap<String, DocValues> docValues = new TreeMap<String, DocValues>();
/**
* Creates a new {@link DocValuesProducerBase} instance and loads all
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
* {@link DocValues} instances for this segment and codec.
*
* @param si
@ -56,7 +56,7 @@ public class DocValuesProducerBase extends PerDocValues {
* @throws IOException
* if an {@link IOException} occurs
*/
protected DocValuesProducerBase(SegmentInfo si, Directory dir,
public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
FieldInfos fieldInfo, int codecId) throws IOException {
load(fieldInfo, si.name, si.docCount, dir, codecId);
}
@ -77,7 +77,7 @@ public class DocValuesProducerBase extends PerDocValues {
if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
final String field = fieldInfo.name;
// TODO can we have a compound file per segment and codec for docvalues?
final String id = DocValuesCodec.docValuesId(segment, codecId, fieldInfo.number);
final String id = DefaultDocValuesConsumer.docValuesId(segment, codecId, fieldInfo.number);
docValues.put(field, loadDocValues(docCount, dir, id, fieldInfo
.getDocValues()));
}

2
lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java → lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.index.codecs.docvalues;
package org.apache.lucene.index.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more

1
lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
View File

@ -19,7 +19,6 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.values.DocValues;
/**

34
lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodecProvider.java
View File

@ -1,34 +0,0 @@
package org.apache.lucene.index.codecs.docvalues;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
/**
* Simple Codec provider that wrapps all registered codecs into a {@link DocValuesCodec}
*/
public class DocValuesCodecProvider extends CodecProvider {
@Override
public synchronized void register(Codec codec) {
if (codec instanceof DocValuesCodec) {
super.register(codec);
} else {
super.register(new DocValuesCodec(codec));
}
}
}

13
lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
View File

@ -22,11 +22,14 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
/** Codec that reads the pre-flex-indexing postings
* format. It does not provide a writer because newly
@ -78,4 +81,14 @@ public class PreFlexCodec extends Codec {
extensions.add(TERMS_EXTENSION);
extensions.add(TERMS_INDEX_EXTENSION);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return null;
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return null;
}
}

19
lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.pulsing;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
@ -28,8 +29,13 @@ import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
import org.apache.lucene.index.codecs.BlockTermsReader;
@ -38,6 +44,7 @@ import org.apache.lucene.index.codecs.TermsIndexReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
/** This codec "inlines" the postings for terms that have
* low docFreq. It wraps another codec, which is used for
@ -155,10 +162,22 @@ public class PulsingCodec extends Codec {
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

20
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
View File

@ -20,14 +20,21 @@ package org.apache.lucene.index.codecs.simpletext;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
/** For debugging, curiosity, transparency only!! Do not
* use this codec in production.
@ -63,10 +70,23 @@ public class SimpleTextCodec extends Codec {
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, ""+id));
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(POSTINGS_EXTENSION);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
// TODO: would be great if these used a plain text impl
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

20
lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
View File

@ -20,12 +20,17 @@ package org.apache.lucene.index.codecs.standard;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
@ -34,7 +39,10 @@ import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.values.Writer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
/** Default codec.
* @lucene.experimental */
@ -135,11 +143,13 @@ public class StandardCodec extends Codec {
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getStandardExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
public static void getStandardExtensions(Set<String> extensions) {
@ -148,4 +158,14 @@ public class StandardCodec extends Codec {
BlockTermsReader.getExtensions(extensions);
VariableGapTermsIndexReader.getIndexExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

2
lucene/src/java/org/apache/lucene/index/values/PerDocFieldValues.java
View File

@ -20,7 +20,7 @@ import java.util.Comparator;
import org.apache.lucene.document.DocValuesField;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.util.BytesRef;
/**

2
lucene/src/java/org/apache/lucene/index/values/Writer.java
View File

@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;

20
lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
@ -33,8 +34,13 @@ import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.BlockTermsReader;
@ -186,11 +192,12 @@ public class MockFixedIntBlockCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
final String codecIdAsString = "" + codecId;
SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
@ -198,5 +205,16 @@ public class MockFixedIntBlockCodec extends Codec {
SepPostingsWriterImpl.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

20
lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
@ -33,8 +34,13 @@ import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.BlockTermsReader;
@ -211,11 +217,12 @@ public class MockVariableIntBlockCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
final String codecIdAsString = "" + codecId;
SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
@ -223,5 +230,16 @@ public class MockVariableIntBlockCodec extends Codec {
SepPostingsWriterImpl.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

20
lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
View File

@ -26,16 +26,22 @@ import java.util.Set;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.TermStats;
@ -338,7 +344,7 @@ public class MockRandomCodec extends Codec {
BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
// hackish!
Iterator<String> it = files.iterator();
while(it.hasNext()) {
@ -356,7 +362,19 @@ public class MockRandomCodec extends Codec {
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
VariableGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
extensions.add(SEED_EXT);
//System.out.println("MockRandom.getExtensions return " + extensions);
}
// can we make this more evil?
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

20
lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
View File

@ -20,14 +20,20 @@ package org.apache.lucene.index.codecs.mocksep;
import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.BlockTermsReader;
@ -130,16 +136,18 @@ public class MockSepCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
final String codecIdAsString = "" + codecId;
SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getSepExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
public static void getSepExtensions(Set<String> extensions) {
@ -147,4 +155,14 @@ public class MockSepCodec extends Codec {
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}

10
lucene/src/test/org/apache/lucene/TestExternalCodecs.java
View File

@ -489,6 +489,16 @@ public class TestExternalCodecs extends LuceneTestCase {
}
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return null;
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return null;
}
@Override
public void getExtensions(Set<String> extensions) {
}

17
lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
View File

@ -33,10 +33,15 @@ import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CoreCodecProvider;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
@ -196,11 +201,13 @@ public class TestDocTermOrds extends LuceneTestCase {
StandardPostingsReader.files(dir, segmentInfo, ""+id, files);
BlockTermsReader.files(dir, segmentInfo, ""+id, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, ""+id, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getStandardExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
}
public static void getStandardExtensions(Set<String> extensions) {
@ -209,6 +216,16 @@ public class TestDocTermOrds extends LuceneTestCase {
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
}
}
public void testRandom() throws Exception {

9
lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
View File

@ -41,7 +41,6 @@ import org.apache.lucene.index.MultiPerDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.docvalues.DocValuesCodecProvider;
import org.apache.lucene.index.values.DocValues.MissingValue;
import org.apache.lucene.index.values.DocValues.Source;
import org.apache.lucene.queryParser.ParseException;
@ -76,13 +75,10 @@ public class TestDocValuesIndexing extends LuceneTestCase {
* - run RAT
*/
private CodecProvider provider;
@Before
public void setUp() throws Exception {
super.setUp();
provider = new DocValuesCodecProvider();
provider.copyFrom(CodecProvider.getDefault());
assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex"));
}
/*
@ -105,7 +101,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
writer.close(true);
IndexReader reader = IndexReader.open(dir, null, true, 1, provider);
IndexReader reader = IndexReader.open(dir, null, true, 1);
assertTrue(reader.isOptimized());
IndexSearcher searcher = new IndexSearcher(reader);
@ -244,7 +240,6 @@ public class TestDocValuesIndexing extends LuceneTestCase {
LogMergePolicy policy = new LogDocMergePolicy();
cfg.setMergePolicy(policy);
policy.setUseCompoundFile(useCompoundFile);
cfg.setCodecProvider(provider);
return cfg;
}

7
lucene/src/test/org/apache/lucene/search/TestSort.java
View File

@ -37,7 +37,6 @@ import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.docvalues.DocValuesCodecProvider;
import org.apache.lucene.index.values.Type;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
@ -228,10 +227,8 @@ public class TestSort extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
//nocommit - enable doc values by default for all tests
DocValuesCodecProvider provider = new DocValuesCodecProvider();
provider.copyFrom(CodecProvider.getDefault());
CodecProvider.setDefault(provider);
//nocommit - we need to be able to run this test with preflex, but without docvalues!!!!
assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex"));
full = getFullIndex();
searchX = getXIndex();
searchY = getYIndex();