mirror of https://github.com/apache/lucene.git
LUCENE-3480: refactoring of docvalues params in Codec.java
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1177851 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir
parent
ec2b654231
commit
981a297488
|
|
@ -137,22 +137,22 @@ public class AppendingCodec extends Codec {
|
|||
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
|
||||
BlockTermsReader.files(dir, segmentInfo, codecId, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
StandardCodec.getStandardExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
|
|||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.PerDocWriteState;
|
||||
|
|
@ -26,7 +25,6 @@ import org.apache.lucene.index.SegmentInfo;
|
|||
import org.apache.lucene.index.SegmentWriteState;
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
/** @lucene.experimental */
|
||||
public abstract class Codec {
|
||||
|
|
@ -34,22 +32,9 @@ public abstract class Codec {
|
|||
/** Unique name that's used to retrieve this codec when
|
||||
* reading the index */
|
||||
public final String name;
|
||||
protected final boolean dvUseCompoundFile;
|
||||
protected final Comparator<BytesRef> docValuesSortComparator;
|
||||
|
||||
protected Codec(String name) {
|
||||
this(name, true);
|
||||
}
|
||||
|
||||
protected Codec(String name, boolean docValuesUseCompoundFile) {
|
||||
this(name, docValuesUseCompoundFile, BytesRef.getUTF8SortedAsUnicodeComparator());
|
||||
}
|
||||
|
||||
protected Codec(String name, boolean docValuesUseCompoundFile,
|
||||
Comparator<BytesRef> docValuesSortComparator) {
|
||||
this.name = name;
|
||||
this.dvUseCompoundFile = docValuesUseCompoundFile;
|
||||
this.docValuesSortComparator = docValuesSortComparator;
|
||||
}
|
||||
|
||||
/** Writes a new segment */
|
||||
|
|
@ -87,38 +72,6 @@ public abstract class Codec {
|
|||
|
||||
/** Records all file extensions this codec uses */
|
||||
public abstract void getExtensions(Set<String> extensions);
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff compound file should be used for
|
||||
* IndexDocValues, otherwise <code>false</code>. The default is
|
||||
* <code>true</code>.
|
||||
* <p>
|
||||
* NOTE: To change the default value you need to subclass a {@link Codec} with
|
||||
* a distinct name since this value is final and should not be changed to
|
||||
* prevent the risk of a index corruption. This setting is private to a
|
||||
* {@link Codec}. If you intend to change this value on an existing
|
||||
* {@link Codec} re-indexing is required.
|
||||
*
|
||||
* @return <code>true</code> iff compound file should be used for
|
||||
* IndexDocValues, otherwise <code>false</code>.
|
||||
*/
|
||||
public boolean getDocValuesUseCFS() {
|
||||
return dvUseCompoundFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
|
||||
* The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
|
||||
* <p>
|
||||
* NOTE: To change the default value you need to subclass a {@link Codec} with
|
||||
* a distinct name since this value is final and should not be changed to
|
||||
* prevent the risk of a index corruption. This setting is private to a
|
||||
* {@link Codec}. If you intend to change this value on an existing
|
||||
* {@link Codec} re-indexing is required.
|
||||
*/
|
||||
public Comparator<BytesRef> getDocValuesSortComparator() {
|
||||
return docValuesSortComparator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
|
|||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
|
|
@ -26,113 +25,51 @@ import org.apache.lucene.index.FieldInfos;
|
|||
import org.apache.lucene.index.IndexFileNames;
|
||||
import org.apache.lucene.index.PerDocWriteState;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.index.values.Writer;
|
||||
import org.apache.lucene.index.codecs.DocValuesWriterBase;
|
||||
import org.apache.lucene.store.CompoundFileDirectory;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.Counter;
|
||||
|
||||
/**
|
||||
*
|
||||
* Default PerDocConsumer implementation that uses compound file.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class DefaultDocValuesConsumer extends PerDocConsumer {
|
||||
private final String segmentName;
|
||||
private final int codecId;
|
||||
public class DefaultDocValuesConsumer extends DocValuesWriterBase {
|
||||
private final Directory directory;
|
||||
private final Counter bytesUsed;
|
||||
private final Comparator<BytesRef> comparator;
|
||||
private boolean useCompoundFile;
|
||||
private final IOContext context;
|
||||
|
||||
public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
|
||||
this.segmentName = state.segmentName;
|
||||
this.codecId = state.codecId;
|
||||
this.bytesUsed = state.bytesUsed;
|
||||
this.context = state.context;
|
||||
public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException {
|
||||
super(state);
|
||||
//TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
|
||||
this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory,
|
||||
IndexFileNames.segmentFileName(segmentName, codecId,
|
||||
IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory;
|
||||
this.comparator = comparator;
|
||||
this.useCompoundFile = useCompoundFile;
|
||||
this.directory = new CompoundFileDirectory(state.directory,
|
||||
IndexFileNames.segmentFileName(state.segmentName, state.codecId,
|
||||
IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true);
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
if (useCompoundFile) {
|
||||
this.directory.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Directory getDirectory() {
|
||||
return directory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
|
||||
return Writer.create(field.getDocValues(),
|
||||
docValuesId(segmentName, codecId, field.number),
|
||||
directory, comparator, bytesUsed, context);
|
||||
public void close() throws IOException {
|
||||
this.directory.close();
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("fallthrough")
|
||||
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
|
||||
Set<String> files, boolean useCompoundFile) throws IOException {
|
||||
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
|
||||
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
|
||||
for (FieldInfo fieldInfo : fieldInfos) {
|
||||
if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
|
||||
String filename = docValuesId(segmentInfo.name, codecId,
|
||||
fieldInfo.number);
|
||||
if (useCompoundFile) {
|
||||
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
|
||||
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
|
||||
return;
|
||||
} else {
|
||||
switch (fieldInfo.getDocValues()) {
|
||||
case BYTES_FIXED_DEREF:
|
||||
case BYTES_VAR_DEREF:
|
||||
case BYTES_VAR_SORTED:
|
||||
case BYTES_FIXED_SORTED:
|
||||
case BYTES_VAR_STRAIGHT:
|
||||
files.add(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.INDEX_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.INDEX_EXTENSION));
|
||||
// until here all types use an index
|
||||
case BYTES_FIXED_STRAIGHT:
|
||||
case FLOAT_32:
|
||||
case FLOAT_64:
|
||||
case VAR_INTS:
|
||||
case FIXED_INTS_16:
|
||||
case FIXED_INTS_32:
|
||||
case FIXED_INTS_64:
|
||||
case FIXED_INTS_8:
|
||||
files.add(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.DATA_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.DATA_EXTENSION));
|
||||
break;
|
||||
|
||||
default:
|
||||
assert false;
|
||||
}
|
||||
}
|
||||
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
|
||||
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static String docValuesId(String segmentsName, int codecID, int fieldId) {
|
||||
return segmentsName + "_" + codecID + "-" + fieldId;
|
||||
public static void getExtensions(Set<String> extensions) {
|
||||
extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
|
||||
extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
|
||||
}
|
||||
|
||||
public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
|
||||
if (useCompoundFile) {
|
||||
extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
|
||||
extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
|
||||
} else {
|
||||
extensions.add(Writer.DATA_EXTENSION);
|
||||
extensions.add(Writer.INDEX_EXTENSION);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,184 +16,50 @@ package org.apache.lucene.index.codecs;
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.IndexFileNames;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.index.values.Bytes;
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.index.codecs.DocValuesReaderBase;
|
||||
import org.apache.lucene.index.values.IndexDocValues;
|
||||
import org.apache.lucene.index.values.Floats;
|
||||
import org.apache.lucene.index.values.Ints;
|
||||
import org.apache.lucene.index.values.ValueType;
|
||||
import org.apache.lucene.store.CompoundFileDirectory;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
/**
|
||||
* Abstract base class for FieldsProducer implementations supporting
|
||||
* {@link IndexDocValues}.
|
||||
*
|
||||
* Default PerDocValues implementation that uses compound file.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class DefaultDocValuesProducer extends PerDocValues {
|
||||
|
||||
public class DefaultDocValuesProducer extends DocValuesReaderBase {
|
||||
protected final TreeMap<String, IndexDocValues> docValues;
|
||||
private final boolean useCompoundFile;
|
||||
private final Closeable cfs;
|
||||
private final Comparator<BytesRef> sortComparator;
|
||||
private final Directory cfs;
|
||||
|
||||
/**
|
||||
*
|
||||
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
|
||||
* {@link IndexDocValues} instances for this segment and codec.
|
||||
*
|
||||
* @param si
|
||||
* the segment info to load the {@link IndexDocValues} for.
|
||||
* @param dir
|
||||
* the directory to load the {@link IndexDocValues} from.
|
||||
* @param fieldInfo
|
||||
* the {@link FieldInfos}
|
||||
* @param codecId
|
||||
* the codec ID
|
||||
* @param useCompoundFile
|
||||
* if <code>true</code> this producer opens a compound file to read
|
||||
* IndexDocValues fields, otherwise each field defines its own set of
|
||||
* files.
|
||||
* @param sortComparator
|
||||
* defines the sort order for sorted IndexDocValues variants
|
||||
* @throws IOException
|
||||
* if an {@link IOException} occurs
|
||||
*/
|
||||
public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
|
||||
FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
|
||||
this.useCompoundFile = useCompoundFile;
|
||||
this.sortComparator = sortComparator;
|
||||
final Directory directory;
|
||||
if (useCompoundFile) {
|
||||
cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
|
||||
} else {
|
||||
cfs = null;
|
||||
directory = dir;
|
||||
}
|
||||
docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link IndexDocValues} instance for the given field name or
|
||||
* <code>null</code> if this field has no {@link IndexDocValues}.
|
||||
*/
|
||||
@Override
|
||||
public IndexDocValues docValues(String field) throws IOException {
|
||||
return docValues.get(field);
|
||||
}
|
||||
|
||||
// Only opens files... doesn't actually load any values
|
||||
protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
|
||||
String segment, int docCount, Directory dir, int codecId, IOContext context)
|
||||
throws IOException {
|
||||
TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
|
||||
boolean success = false;
|
||||
try {
|
||||
|
||||
for (FieldInfo fieldInfo : fieldInfos) {
|
||||
if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
|
||||
final String field = fieldInfo.name;
|
||||
// TODO can we have a compound file per segment and codec for
|
||||
// docvalues?
|
||||
final String id = DefaultDocValuesConsumer.docValuesId(segment,
|
||||
codecId, fieldInfo.number);
|
||||
values.put(field,
|
||||
loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context));
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
// if we fail we must close all opened resources if there are any
|
||||
closeInternal(values.values());
|
||||
}
|
||||
}
|
||||
return values;
|
||||
public DefaultDocValuesProducer(SegmentReadState state) throws IOException {
|
||||
cfs = new CompoundFileDirectory(state.dir,
|
||||
IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION),
|
||||
state.context, false);
|
||||
docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
|
||||
* Codecs that use different implementations for a certain {@link ValueType} can
|
||||
* simply override this method and return their custom implementations.
|
||||
*
|
||||
* @param docCount
|
||||
* number of documents in the segment
|
||||
* @param dir
|
||||
* the {@link Directory} to load the {@link IndexDocValues} from
|
||||
* @param id
|
||||
* the unique file ID within the segment
|
||||
* @param type
|
||||
* the type to load
|
||||
* @param sortComparator byte comparator used by sorted variants
|
||||
* @return a {@link IndexDocValues} instance for the given type
|
||||
* @throws IOException
|
||||
* if an {@link IOException} occurs
|
||||
* @throws IllegalArgumentException
|
||||
* if the given {@link ValueType} is not supported
|
||||
*/
|
||||
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
|
||||
ValueType type, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
|
||||
switch (type) {
|
||||
case FIXED_INTS_16:
|
||||
case FIXED_INTS_32:
|
||||
case FIXED_INTS_64:
|
||||
case FIXED_INTS_8:
|
||||
case VAR_INTS:
|
||||
return Ints.getValues(dir, id, docCount, type, context);
|
||||
case FLOAT_32:
|
||||
return Floats.getValues(dir, id, docCount, context);
|
||||
case FLOAT_64:
|
||||
return Floats.getValues(dir, id, docCount, context);
|
||||
case BYTES_FIXED_STRAIGHT:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context);
|
||||
case BYTES_FIXED_DEREF:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context);
|
||||
case BYTES_FIXED_SORTED:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context);
|
||||
case BYTES_VAR_STRAIGHT:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context);
|
||||
case BYTES_VAR_DEREF:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context);
|
||||
case BYTES_VAR_SORTED:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context);
|
||||
default:
|
||||
throw new IllegalStateException("unrecognized index values mode " + type);
|
||||
}
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
closeInternal(docValues.values());
|
||||
}
|
||||
|
||||
private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
|
||||
final Collection<? extends Closeable> toClose;
|
||||
if (useCompoundFile) {
|
||||
final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
|
||||
list.add(cfs);
|
||||
toClose = list;
|
||||
} else {
|
||||
toClose = closeables;
|
||||
}
|
||||
IOUtils.close(toClose);
|
||||
@Override
|
||||
protected Map<String,IndexDocValues> docValues() {
|
||||
return docValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> fields() {
|
||||
return docValues.keySet();
|
||||
protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
|
||||
final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
|
||||
list.add(cfs);
|
||||
IOUtils.close(list);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,143 @@
|
|||
package org.apache.lucene.index.codecs;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.values.Bytes;
|
||||
import org.apache.lucene.index.values.Floats;
|
||||
import org.apache.lucene.index.values.IndexDocValues;
|
||||
import org.apache.lucene.index.values.Ints;
|
||||
import org.apache.lucene.index.values.ValueType;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
/**
|
||||
* Abstract base class for PerDocValues implementations
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public abstract class DocValuesReaderBase extends PerDocValues {
|
||||
|
||||
protected abstract void closeInternal(Collection<? extends Closeable> closeables) throws IOException;
|
||||
protected abstract Map<String, IndexDocValues> docValues();
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeInternal(docValues().values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDocValues docValues(String field) throws IOException {
|
||||
return docValues().get(field);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> fields() {
|
||||
return docValues().keySet();
|
||||
}
|
||||
|
||||
public Comparator<BytesRef> getComparator() throws IOException {
|
||||
return BytesRef.getUTF8SortedAsUnicodeComparator();
|
||||
}
|
||||
|
||||
// Only opens files... doesn't actually load any values
|
||||
protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
|
||||
String segment, int docCount, Directory dir, int codecId, IOContext context)
|
||||
throws IOException {
|
||||
TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
|
||||
boolean success = false;
|
||||
try {
|
||||
|
||||
for (FieldInfo fieldInfo : fieldInfos) {
|
||||
if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
|
||||
final String field = fieldInfo.name;
|
||||
// TODO can we have a compound file per segment and codec for
|
||||
// docvalues?
|
||||
final String id = DefaultDocValuesConsumer.docValuesId(segment,
|
||||
codecId, fieldInfo.number);
|
||||
values.put(field,
|
||||
loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
// if we fail we must close all opened resources if there are any
|
||||
closeInternal(values.values());
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
|
||||
* Codecs that use different implementations for a certain {@link ValueType} can
|
||||
* simply override this method and return their custom implementations.
|
||||
*
|
||||
* @param docCount
|
||||
* number of documents in the segment
|
||||
* @param dir
|
||||
* the {@link Directory} to load the {@link IndexDocValues} from
|
||||
* @param id
|
||||
* the unique file ID within the segment
|
||||
* @param type
|
||||
* the type to load
|
||||
* @return a {@link IndexDocValues} instance for the given type
|
||||
* @throws IOException
|
||||
* if an {@link IOException} occurs
|
||||
* @throws IllegalArgumentException
|
||||
* if the given {@link ValueType} is not supported
|
||||
*/
|
||||
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
|
||||
ValueType type, IOContext context) throws IOException {
|
||||
switch (type) {
|
||||
case FIXED_INTS_16:
|
||||
case FIXED_INTS_32:
|
||||
case FIXED_INTS_64:
|
||||
case FIXED_INTS_8:
|
||||
case VAR_INTS:
|
||||
return Ints.getValues(dir, id, docCount, type, context);
|
||||
case FLOAT_32:
|
||||
return Floats.getValues(dir, id, docCount, context);
|
||||
case FLOAT_64:
|
||||
return Floats.getValues(dir, id, docCount, context);
|
||||
case BYTES_FIXED_STRAIGHT:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context);
|
||||
case BYTES_FIXED_DEREF:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context);
|
||||
case BYTES_FIXED_SORTED:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context);
|
||||
case BYTES_VAR_STRAIGHT:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context);
|
||||
case BYTES_VAR_DEREF:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context);
|
||||
case BYTES_VAR_SORTED:
|
||||
return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context);
|
||||
default:
|
||||
throw new IllegalStateException("unrecognized index values mode " + type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package org.apache.lucene.index.codecs;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.PerDocWriteState;
|
||||
import org.apache.lucene.index.values.Writer;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.Counter;
|
||||
|
||||
/**
|
||||
* Abstract base class for PerDocConsumer implementations
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public abstract class DocValuesWriterBase extends PerDocConsumer {
|
||||
private final String segmentName;
|
||||
private final int codecId;
|
||||
private final Counter bytesUsed;
|
||||
private final IOContext context;
|
||||
|
||||
protected DocValuesWriterBase(PerDocWriteState state) {
|
||||
this.segmentName = state.segmentName;
|
||||
this.codecId = state.codecId;
|
||||
this.bytesUsed = state.bytesUsed;
|
||||
this.context = state.context;
|
||||
}
|
||||
|
||||
protected abstract Directory getDirectory();
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
|
||||
return Writer.create(field.getDocValues(),
|
||||
docValuesId(segmentName, codecId, field.number),
|
||||
getDirectory(), getComparator(), bytesUsed, context);
|
||||
}
|
||||
|
||||
public static String docValuesId(String segmentsName, int codecID, int fieldId) {
|
||||
return segmentsName + "_" + codecID + "-" + fieldId;
|
||||
}
|
||||
|
||||
public Comparator<BytesRef> getComparator() throws IOException {
|
||||
return BytesRef.getUTF8SortedAsUnicodeComparator();
|
||||
}
|
||||
}
|
||||
|
|
@ -51,5 +51,4 @@ public abstract class PerDocValues implements Closeable {
|
|||
* Returns all fields this {@link PerDocValues} contains values for.
|
||||
*/
|
||||
public abstract Collection<String> fields();
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -787,22 +787,22 @@ public class MemoryCodec extends Codec {
|
|||
@Override
|
||||
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
|
||||
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
extensions.add(EXTENSION);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,22 +134,22 @@ public class PulsingCodec extends Codec {
|
|||
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
|
||||
StandardPostingsReader.files(dir, segmentInfo, codecID, files);
|
||||
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
StandardCodec.getStandardExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
package org.apache.lucene.index.codecs.sep;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.IndexFileNames;
|
||||
import org.apache.lucene.index.PerDocWriteState;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.index.codecs.DocValuesWriterBase;
|
||||
import org.apache.lucene.index.values.Writer;
|
||||
import org.apache.lucene.store.Directory;
|
||||
|
||||
/**
|
||||
* Implementation of PerDocConsumer that uses separate files.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class SepDocValuesConsumer extends DocValuesWriterBase {
|
||||
private final Directory directory;
|
||||
|
||||
public SepDocValuesConsumer(PerDocWriteState state) throws IOException {
|
||||
super(state);
|
||||
this.directory = state.directory;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Directory getDirectory() {
|
||||
return directory;
|
||||
}
|
||||
|
||||
@SuppressWarnings("fallthrough")
|
||||
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
|
||||
Set<String> files) throws IOException {
|
||||
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
|
||||
for (FieldInfo fieldInfo : fieldInfos) {
|
||||
if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
|
||||
String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number);
|
||||
switch (fieldInfo.getDocValues()) {
|
||||
case BYTES_FIXED_DEREF:
|
||||
case BYTES_VAR_DEREF:
|
||||
case BYTES_VAR_SORTED:
|
||||
case BYTES_FIXED_SORTED:
|
||||
case BYTES_VAR_STRAIGHT:
|
||||
files.add(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.INDEX_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.INDEX_EXTENSION));
|
||||
// until here all types use an index
|
||||
case BYTES_FIXED_STRAIGHT:
|
||||
case FLOAT_32:
|
||||
case FLOAT_64:
|
||||
case VAR_INTS:
|
||||
case FIXED_INTS_16:
|
||||
case FIXED_INTS_32:
|
||||
case FIXED_INTS_64:
|
||||
case FIXED_INTS_8:
|
||||
files.add(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.DATA_EXTENSION));
|
||||
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
|
||||
Writer.DATA_EXTENSION));
|
||||
break;
|
||||
|
||||
default:
|
||||
assert false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void getExtensions(Set<String> extensions) {
|
||||
extensions.add(Writer.DATA_EXTENSION);
|
||||
extensions.add(Writer.INDEX_EXTENSION);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package org.apache.lucene.index.codecs.sep;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.index.codecs.DocValuesReaderBase;
|
||||
import org.apache.lucene.index.values.IndexDocValues;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
/**
|
||||
* Implementation of PerDocValues that uses separate files.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class SepDocValuesProducer extends DocValuesReaderBase {
|
||||
private final TreeMap<String, IndexDocValues> docValues;
|
||||
|
||||
/**
|
||||
* Creates a new {@link SepDocValuesProducer} instance and loads all
|
||||
* {@link IndexDocValues} instances for this segment and codec.
|
||||
*/
|
||||
public SepDocValuesProducer(SegmentReadState state) throws IOException {
|
||||
docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String,IndexDocValues> docValues() {
|
||||
return docValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
|
||||
IOUtils.close(closeables);
|
||||
}
|
||||
}
|
||||
|
|
@ -69,23 +69,23 @@ public class SimpleTextCodec extends Codec {
|
|||
@Override
|
||||
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
|
||||
files.add(getPostingsFileName(segmentInfo.name, id));
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
extensions.add(POSTINGS_EXTENSION);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
// TODO: would be great if these used a plain text impl
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,19 +110,19 @@ public class StandardCodec extends Codec {
|
|||
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
|
||||
StandardPostingsReader.files(dir, segmentInfo, codecID, files);
|
||||
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
getStandardExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
}
|
||||
|
||||
public static void getStandardExtensions(Set<String> extensions) {
|
||||
extensions.add(FREQ_EXTENSION);
|
||||
extensions.add(PROX_EXTENSION);
|
||||
BlockTreeTermsReader.getExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -132,11 +132,11 @@ public class StandardCodec extends Codec {
|
|||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.FieldsProducer;
|
|||
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
|
||||
import org.apache.lucene.index.codecs.sep.IntIndexInput;
|
||||
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
|
||||
import org.apache.lucene.index.codecs.standard.StandardCodec;
|
||||
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
|
||||
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
|
||||
import org.apache.lucene.index.codecs.PerDocConsumer;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.PerDocValues;
|
||||
import org.apache.lucene.index.codecs.PostingsWriterBase;
|
||||
import org.apache.lucene.index.codecs.PostingsReaderBase;
|
||||
|
|
@ -207,7 +207,7 @@ public class MockFixedIntBlockCodec extends Codec {
|
|||
SepPostingsReader.files(segmentInfo, codecId, files);
|
||||
BlockTermsReader.files(dir, segmentInfo, codecId, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
|
||||
SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -215,16 +215,16 @@ public class MockFixedIntBlockCodec extends Codec {
|
|||
SepPostingsWriter.getExtensions(extensions);
|
||||
BlockTermsReader.getExtensions(extensions);
|
||||
FixedGapTermsIndexReader.getIndexExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
SepDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new SepDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new SepDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.FieldsProducer;
|
|||
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
|
||||
import org.apache.lucene.index.codecs.sep.IntIndexInput;
|
||||
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
|
||||
import org.apache.lucene.index.codecs.standard.StandardCodec;
|
||||
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
|
||||
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
|
||||
import org.apache.lucene.index.codecs.PerDocConsumer;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.PerDocValues;
|
||||
import org.apache.lucene.index.codecs.PostingsWriterBase;
|
||||
import org.apache.lucene.index.codecs.PostingsReaderBase;
|
||||
|
|
@ -230,7 +230,7 @@ public class MockVariableIntBlockCodec extends Codec {
|
|||
SepPostingsReader.files(segmentInfo, codecId, files);
|
||||
BlockTermsReader.files(dir, segmentInfo, codecId, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
|
||||
SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -238,16 +238,16 @@ public class MockVariableIntBlockCodec extends Codec {
|
|||
SepPostingsWriter.getExtensions(extensions);
|
||||
BlockTermsReader.getExtensions(extensions);
|
||||
FixedGapTermsIndexReader.getIndexExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
SepDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new SepDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new SepDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter;
|
|||
import org.apache.lucene.index.codecs.sep.IntIndexInput;
|
||||
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
|
||||
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
|
||||
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
|
||||
|
|
@ -75,17 +77,13 @@ import org.apache.lucene.util._TestUtil;
|
|||
*/
|
||||
|
||||
public class MockRandomCodec extends Codec {
|
||||
|
||||
private final boolean useSepDocValues;
|
||||
private final Random seedRandom;
|
||||
private final String SEED_EXT = "sd";
|
||||
|
||||
public MockRandomCodec(Random random) {
|
||||
this(random, "MockRandom", true);
|
||||
|
||||
}
|
||||
|
||||
protected MockRandomCodec(Random random, String name, boolean docValuesUseCompoundFile) {
|
||||
super(name, docValuesUseCompoundFile);
|
||||
super("MockRandom");
|
||||
this.useSepDocValues = random.nextBoolean();
|
||||
this.seedRandom = new Random(random.nextLong());
|
||||
}
|
||||
|
||||
|
|
@ -428,7 +426,11 @@ public class MockRandomCodec extends Codec {
|
|||
BlockTreeTermsReader.files(dir, segmentInfo, codecId, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
|
||||
if (useSepDocValues) {
|
||||
SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
} else {
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
}
|
||||
// hackish!
|
||||
Iterator<String> it = files.iterator();
|
||||
while(it.hasNext()) {
|
||||
|
|
@ -447,7 +449,11 @@ public class MockRandomCodec extends Codec {
|
|||
BlockTreeTermsReader.getExtensions(extensions);
|
||||
FixedGapTermsIndexReader.getIndexExtensions(extensions);
|
||||
VariableGapTermsIndexReader.getIndexExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
if (useSepDocValues) {
|
||||
SepDocValuesConsumer.getExtensions(extensions);
|
||||
} else {
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
extensions.add(SEED_EXT);
|
||||
//System.out.println("MockRandom.getExtensions return " + extensions);
|
||||
}
|
||||
|
|
@ -455,11 +461,19 @@ public class MockRandomCodec extends Codec {
|
|||
// can we make this more evil?
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
if (useSepDocValues) {
|
||||
return new SepDocValuesConsumer(state);
|
||||
} else {
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
if (useSepDocValues) {
|
||||
return new SepDocValuesProducer(state);
|
||||
} else {
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
package org.apache.lucene.index.codecs.mockrandom;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* Randomly combines terms index impl w/ postings impls. and uses non-CFS format for docvalues
|
||||
*/
|
||||
public class MockRandomDocValuesCodec extends MockRandomCodec {
|
||||
|
||||
public MockRandomDocValuesCodec(Random random) {
|
||||
super(random, "MockDocValuesCodec", false);
|
||||
// uses noCFS for docvalues for test coverage
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -25,13 +25,11 @@ import org.apache.lucene.index.SegmentInfo;
|
|||
import org.apache.lucene.index.SegmentWriteState;
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.index.codecs.Codec;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.FieldsConsumer;
|
||||
import org.apache.lucene.index.codecs.FieldsProducer;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
|
||||
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
|
||||
import org.apache.lucene.index.codecs.PerDocConsumer;
|
||||
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.PerDocValues;
|
||||
import org.apache.lucene.index.codecs.PostingsReaderBase;
|
||||
import org.apache.lucene.index.codecs.PostingsWriterBase;
|
||||
|
|
@ -40,6 +38,8 @@ import org.apache.lucene.index.codecs.BlockTermsWriter;
|
|||
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
|
||||
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
|
||||
import org.apache.lucene.index.codecs.standard.StandardCodec;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
|
||||
import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
|
||||
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
|
||||
import org.apache.lucene.store.Directory;
|
||||
|
|
@ -139,28 +139,28 @@ public class MockSepCodec extends Codec {
|
|||
SepPostingsReader.files(segmentInfo, codecId, files);
|
||||
BlockTermsReader.files(dir, segmentInfo, codecId, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
|
||||
SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
getSepExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
}
|
||||
|
||||
public static void getSepExtensions(Set<String> extensions) {
|
||||
SepPostingsWriter.getExtensions(extensions);
|
||||
BlockTermsReader.getExtensions(extensions);
|
||||
FixedGapTermsIndexReader.getIndexExtensions(extensions);
|
||||
SepDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new SepDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new SepDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec;
|
|||
import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
|
||||
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
|
||||
import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec;
|
||||
import org.apache.lucene.index.codecs.mockrandom.MockRandomDocValuesCodec;
|
||||
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
|
||||
import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
|
||||
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
|
||||
|
|
@ -281,8 +280,6 @@ public abstract class LuceneTestCase extends Assert {
|
|||
// baseBlockSize cannot be over 127:
|
||||
swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp);
|
||||
swapCodec(new MockRandomCodec(random), cp);
|
||||
// give docvalues non-cfs testcoverage
|
||||
swapCodec(new MockRandomDocValuesCodec(random), cp);
|
||||
|
||||
return cp.lookup(codec);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -203,13 +203,13 @@ public class TestDocTermOrds extends LuceneTestCase {
|
|||
StandardPostingsReader.files(dir, segmentInfo, id, files);
|
||||
BlockTermsReader.files(dir, segmentInfo, id, files);
|
||||
FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getExtensions(Set<String> extensions) {
|
||||
getStandardExtensions(extensions);
|
||||
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
|
||||
DefaultDocValuesConsumer.getExtensions(extensions);
|
||||
}
|
||||
|
||||
public static void getStandardExtensions(Set<String> extensions) {
|
||||
|
|
@ -221,12 +221,12 @@ public class TestDocTermOrds extends LuceneTestCase {
|
|||
|
||||
@Override
|
||||
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
|
||||
return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
|
||||
return new DefaultDocValuesConsumer(state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
|
||||
return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
|
||||
return new DefaultDocValuesProducer(state);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue