Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-17 23:45:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2741: Added support for codecs using the same file extension within the same segment

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1035354 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Simon Willnauer 2010-11-15 17:01:26 +00:00
parent 467a25a8f5
commit 62f9291625
27 changed files with 169 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
View File

@ -1,6 +1,6 @@
package org.apache.lucene.index.codecs.appending;
/*
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@ -88,7 +88,7 @@ public class AppendingCodec extends Codec {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state)
throws IOException {
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize);
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@ -97,7 +97,8 @@ public class AppendingCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.codecId);
success = true;
} finally {
if (!success) {
@ -111,7 +112,8 @@ public class AppendingCodec extends Codec {
docsReader,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -126,11 +128,11 @@ public class AppendingCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files)
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files)
throws IOException {
StandardPostingsReader.files(dir, segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override

4
lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java
View File

@ -35,9 +35,9 @@ public class AppendingTermsDictReader extends PrefixCodedTermsReader {
public AppendingTermsDictReader(TermsIndexReaderBase indexReader,
Directory dir, FieldInfos fieldInfos, String segment,
PostingsReaderBase postingsReader, int readBufferSize,
Comparator<BytesRef> termComp, int termsCacheSize) throws IOException {
Comparator<BytesRef> termComp, int termsCacheSize, String codecId) throws IOException {
super(indexReader, dir, fieldInfos, segment, postingsReader, readBufferSize,
termComp, termsCacheSize);
termComp, termsCacheSize, codecId);
}
@Override

4
lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java
View File

@ -30,9 +30,9 @@ import org.apache.lucene.util.CodecUtil;
public class AppendingTermsIndexReader extends FixedGapTermsIndexReader {
public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos,
String segment, int indexDivisor, Comparator<BytesRef> termComp)
String segment, int indexDivisor, Comparator<BytesRef> termComp, String codecId)
throws IOException {
super(dir, fieldInfos, segment, indexDivisor, termComp);
super(dir, fieldInfos, segment, indexDivisor, termComp, codecId);
}
@Override

9
lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
View File

@ -61,9 +61,7 @@ final class PerFieldCodecWrapper extends Codec {
assert segmentCodecs == state.segmentCodecs;
final Codec[] codecs = segmentCodecs.codecs;
for (int i = 0; i < codecs.length; i++) {
state.currentCodecId = i; // actual codec should use that to create its
// files
consumers.add(codecs[i].fieldsConsumer(state));
consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, "" + i)));
}
}
@ -111,7 +109,7 @@ final class PerFieldCodecWrapper extends Codec {
Codec codec = segmentCodecs.codecs[fi.codecId];
if (!producers.containsKey(codec)) {
producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir,
si, fieldInfos, readBufferSize, indexDivisor)));
si, fieldInfos, readBufferSize, indexDivisor, ""+fi.codecId)));
}
codecs.put(fi.name, producers.get(codec));
}
@ -195,8 +193,9 @@ final class PerFieldCodecWrapper extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo info, Set<String> files)
public void files(Directory dir, SegmentInfo info, String codecId, Set<String> files)
throws IOException {
// ignore codecid sicne segmentCodec will assign it per codec
segmentCodecs.files(dir, info, files);
}

12
lucene/src/java/org/apache/lucene/index/SegmentCodecs.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Set;
@ -120,14 +119,11 @@ final class SegmentCodecs implements Cloneable {
void files(Directory dir, SegmentInfo info, Set<String> files)
throws IOException {
final Set<Codec> seen = new HashSet<Codec>();
final Codec[] codecArray = codecs;
for (Codec codec : codecArray) {
if (!seen.contains(codec)) {
seen.add(codec);
codec.files(dir, info, files);
}
}
for (int i = 0; i < codecArray.length; i++) {
codecArray[i].files(dir, info, ""+i, files);
}
}
@Override

10
lucene/src/java/org/apache/lucene/index/SegmentReadState.java
View File

@ -34,16 +34,24 @@ public class SegmentReadState {
// that must do so), then it should negate this value to
// get the app's terms divisor:
public final int termsIndexDivisor;
public final String codecId;
public SegmentReadState(Directory dir, SegmentInfo info,
FieldInfos fieldInfos, int readBufferSize, int termsIndexDivisor) {
this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, "");
}
public SegmentReadState(Directory dir,
SegmentInfo info,
FieldInfos fieldInfos,
int readBufferSize,
int termsIndexDivisor) {
int termsIndexDivisor,
String codecId) {
this.dir = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
this.readBufferSize = readBufferSize;
this.termsIndexDivisor = termsIndexDivisor;
this.codecId = codecId;
}
}

20
lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
View File

@ -37,7 +37,7 @@ public class SegmentWriteState {
public final Collection<String> flushedFiles;
final SegmentCodecs segmentCodecs;
public int currentCodecId;
public final String codecId;
/** Expert: The fraction of terms in the "dictionary" which should be stored
* in RAM. Smaller values use more memory, but make searching slightly
@ -73,5 +73,23 @@ public class SegmentWriteState {
this.termIndexInterval = termIndexInterval;
this.segmentCodecs = segmentCodecs;
flushedFiles = new HashSet<String>();
codecId = "";
}
/**
* Create a shallow {@link SegmentWriteState} copy final a codec ID
*/
SegmentWriteState(SegmentWriteState state, String codecId) {
infoStream = state.infoStream;
directory = state.directory;
segmentName = state.segmentName;
fieldInfos = state.fieldInfos;
docStoreSegmentName = state.docStoreSegmentName;
numDocs = state.numDocs;
numDocsInStore = state.numDocsInStore;
termIndexInterval = state.termIndexInterval;
segmentCodecs = state.segmentCodecs;
flushedFiles = state.flushedFiles;
this.codecId = codecId;
}
}

11
lucene/src/java/org/apache/lucene/index/codecs/Codec.java
View File

@ -51,8 +51,15 @@ public abstract class Codec {
* use; else, those files may be deleted. */
public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
/** Gathers files associated with this segment */
public abstract void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException;
/**
* Gathers files associated with this segment
*
* @param dir the {@link Directory} this segment was written to
* @param segmentInfo the {@link SegmentInfo} for this segment
* @param id the codec id within this segment
* @param files the of files to add the codec files to.
*/
public abstract void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException;
/** Records all file extensions this codec uses */
public abstract void getExtensions(Set<String> extensions);

8
lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java
View File

@ -90,12 +90,12 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase {
// start of the field info data
protected long dirOffset;
public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp)
public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp, String codecId)
throws IOException {
this.termComp = termComp;
IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, "", FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
boolean success = false;
@ -436,8 +436,8 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase {
return fields.get(fieldInfo);
}
public static void files(Directory dir, SegmentInfo info, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(info.name, "", FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
public static void files(Directory dir, SegmentInfo info, String id, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
}
public static void getIndexExtensions(Collection<String> extensions) {

2
lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java
View File

@ -49,7 +49,7 @@ public class FixedGapTermsIndexWriter extends TermsIndexWriterBase {
private IndexOutput termsOut;
public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException {
final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, "", TERMS_INDEX_EXTENSION);
final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION);
state.flushedFiles.add(indexFileName);
termIndexInterval = state.termIndexInterval;
out = state.directory.createOutput(indexFileName);

8
lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java
View File

@ -112,7 +112,7 @@ public class PrefixCodedTermsReader extends FieldsProducer {
}
public PrefixCodedTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize,
Comparator<BytesRef> termComp, int termsCacheSize)
Comparator<BytesRef> termComp, int termsCacheSize, String codecId)
throws IOException {
this.postingsReader = postingsReader;
@ -120,7 +120,7 @@ public class PrefixCodedTermsReader extends FieldsProducer {
this.termComp = termComp;
in = dir.openInput(IndexFileNames.segmentFileName(segment, "", PrefixCodedTermsWriter.TERMS_EXTENSION),
in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, PrefixCodedTermsWriter.TERMS_EXTENSION),
readBufferSize);
boolean success = false;
@ -203,8 +203,8 @@ public class PrefixCodedTermsReader extends FieldsProducer {
}
}
public static void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", PrefixCodedTermsWriter.TERMS_EXTENSION));
public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, PrefixCodedTermsWriter.TERMS_EXTENSION));
}
public static void getExtensions(Collection<String> extensions) {

2
lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java
View File

@ -69,7 +69,7 @@ public class PrefixCodedTermsWriter extends FieldsConsumer {
PostingsWriterBase postingsWriter,
Comparator<BytesRef> termComp) throws IOException
{
final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, "", TERMS_EXTENSION);
final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION);
this.termsIndexWriter = termsIndexWriter;
this.termComp = termComp;
out = state.directory.createOutput(termsFileName);

3
lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
View File

@ -66,7 +66,8 @@ public class PreFlexCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
public void files(Directory dir, SegmentInfo info, String id, Set<String> files) throws IOException {
// preflex fields have no codec ID - we ignore it here
PreFlexFields.files(dir, info, files);
}

16
lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
View File

@ -108,7 +108,7 @@ public class PulsingCodec extends Codec {
// We wrap StandardPostingsReader, but any StandardPostingsReader
// will work:
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize);
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader);
// Terms dict index reader
@ -120,7 +120,8 @@ public class PulsingCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.codecId);
success = true;
} finally {
if (!success) {
@ -136,7 +137,8 @@ public class PulsingCodec extends Codec {
pulsingReader,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -151,10 +153,10 @@ public class PulsingCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, id, files);
PrefixCodedTermsReader.files(dir, segmentInfo, id, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
}
@Override

24
lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
View File

@ -54,20 +54,20 @@ public class SepPostingsReaderImpl extends PostingsReaderBase {
int skipInterval;
int maxSkipLevels;
public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory) throws IOException {
public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, String codecId) throws IOException {
boolean success = false;
try {
final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.DOC_EXTENSION);
final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION);
docIn = intFactory.openInput(dir, docFileName);
skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize);
skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize);
if (segmentInfo.getHasProx()) {
freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.FREQ_EXTENSION));
posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.POS_EXTENSION), readBufferSize);
payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize);
freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION));
posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION), readBufferSize);
payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize);
} else {
posIn = null;
payloadIn = null;
@ -81,14 +81,14 @@ public class SepPostingsReaderImpl extends PostingsReaderBase {
}
}
public static void files(SegmentInfo segmentInfo, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.DOC_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.SKIP_EXTENSION));
public static void files(SegmentInfo segmentInfo, String codecId, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION));
if (segmentInfo.getHasProx()) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.FREQ_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.POS_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.PAYLOAD_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION));
}
}

10
lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java
View File

@ -84,24 +84,24 @@ public final class SepPostingsWriterImpl extends PostingsWriterBase {
public SepPostingsWriterImpl(SegmentWriteState state, IntStreamFactory factory) throws IOException {
super();
final String docFileName = IndexFileNames.segmentFileName(state.segmentName, "", DOC_EXTENSION);
final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION);
state.flushedFiles.add(docFileName);
docOut = factory.createOutput(state.directory, docFileName);
docIndex = docOut.index();
if (state.fieldInfos.hasProx()) {
final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, "", FREQ_EXTENSION);
final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION);
state.flushedFiles.add(frqFileName);
freqOut = factory.createOutput(state.directory, frqFileName);
freqIndex = freqOut.index();
final String posFileName = IndexFileNames.segmentFileName(state.segmentName, "", POS_EXTENSION);
final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION);
posOut = factory.createOutput(state.directory, posFileName);
state.flushedFiles.add(posFileName);
posIndex = posOut.index();
// TODO: -- only if at least one field stores payloads?
final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, "", PAYLOAD_EXTENSION);
final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION);
state.flushedFiles.add(payloadFileName);
payloadOut = state.directory.createOutput(payloadFileName);
@ -113,7 +113,7 @@ public final class SepPostingsWriterImpl extends PostingsWriterBase {
payloadOut = null;
}
final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, "", SKIP_EXTENSION);
final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION);
state.flushedFiles.add(skipFileName);
skipOut = state.directory.createOutput(skipFileName);

6
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
View File

@ -56,13 +56,13 @@ public class SimpleTextCodec extends Codec {
/** Extension of freq postings file */
static final String POSTINGS_EXTENSION = "pst";
static String getPostingsFileName(String segment) {
static String getPostingsFileName(String segment, String id) {
return IndexFileNames.segmentFileName(segment, "", POSTINGS_EXTENSION);
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name));
public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, id));
}
@Override

3
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
View File

@ -56,7 +56,8 @@ class SimpleTextFieldsReader extends FieldsProducer {
final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name));
in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, ""+state.codecId));
fieldInfos = state.fieldInfos;
}

2
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java
View File

@ -44,7 +44,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer {
final static BytesRef PAYLOAD = new BytesRef(" payload ");
public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException {
final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName);
final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId);
out = state.directory.createOutput(fileName);
state.flushedFiles.add(fileName);
}

16
lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
View File

@ -84,7 +84,7 @@ public class StandardCodec extends Codec {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize);
PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@ -93,7 +93,8 @@ public class StandardCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.codecId);
success = true;
} finally {
if (!success) {
@ -110,7 +111,8 @@ public class StandardCodec extends Codec {
postings,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
TERMS_CACHE_SIZE);
TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -131,10 +133,10 @@ public class StandardCodec extends Codec {
static final String PROX_EXTENSION = "prx";
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, id, files);
PrefixCodedTermsReader.files(dir, segmentInfo, id, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
}
@Override

12
lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
View File

@ -45,13 +45,13 @@ public class StandardPostingsReader extends PostingsReaderBase {
int skipInterval;
int maxSkipLevels;
public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException {
freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.FREQ_EXTENSION),
public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, String codecId) throws IOException {
freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION),
readBufferSize);
if (segmentInfo.getHasProx()) {
boolean success = false;
try {
proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.PROX_EXTENSION),
proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION),
readBufferSize);
success = true;
} finally {
@ -64,10 +64,10 @@ public class StandardPostingsReader extends PostingsReaderBase {
}
}
public static void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.FREQ_EXTENSION));
public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.FREQ_EXTENSION));
if (segmentInfo.getHasProx()) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.PROX_EXTENSION));
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.PROX_EXTENSION));
}
}

4
lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
View File

@ -60,14 +60,14 @@ public final class StandardPostingsWriter extends PostingsWriterBase {
public StandardPostingsWriter(SegmentWriteState state) throws IOException {
super();
String fileName = IndexFileNames.segmentFileName(state.segmentName, "", StandardCodec.FREQ_EXTENSION);
String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION);
state.flushedFiles.add(fileName);
freqOut = state.directory.createOutput(fileName);
if (state.fieldInfos.hasProx()) {
// At least one field does not omit TF, so create the
// prox file
fileName = IndexFileNames.segmentFileName(state.segmentName, "", StandardCodec.PROX_EXTENSION);
fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION);
state.flushedFiles.add(fileName);
proxOut = state.directory.createOutput(fileName);
} else {

18
lucene/src/test/org/apache/lucene/TestExternalCodecs.java
View File

@ -480,7 +480,7 @@ public class TestExternalCodecs extends LuceneTestCase {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
}
}
@ -549,7 +549,7 @@ public class TestExternalCodecs extends LuceneTestCase {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize);
PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader);
// Terms dict index reader
@ -561,7 +561,8 @@ public class TestExternalCodecs extends LuceneTestCase {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
reverseUnicodeComparator);
reverseUnicodeComparator,
state.codecId);
success = true;
} finally {
if (!success) {
@ -579,7 +580,8 @@ public class TestExternalCodecs extends LuceneTestCase {
pulsingReader,
state.readBufferSize,
reverseUnicodeComparator,
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -594,10 +596,10 @@ public class TestExternalCodecs extends LuceneTestCase {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override

20
lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
View File

@ -28,7 +28,10 @@ import org.apache.lucene.index.CheckIndex.Status.SegmentInfoStatus;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec;
import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.search.IndexSearcher;
@ -255,15 +258,16 @@ public class TestPerFieldCodecSupport extends LuceneTestCase {
final int docsPerRound = 97;
for (int i = 0; i < 5; i++) {
CodecProvider provider = new CodecProvider();
provider.register(new StandardCodec());
provider.register(new SimpleTextCodec());
// provider.register(new MockSepCodec()); // TODO enable once we have
// files per codec
// provider.register(new PulsingCodec());
Codec[] codecs = new Codec[] { new StandardCodec(),
new SimpleTextCodec(), new MockSepCodec(),
new PulsingCodec(1 + random.nextInt(10)),
new MockVariableIntBlockCodec(1 + random.nextInt(10)),
new MockFixedIntBlockCodec(1 + random.nextInt(10)) };
for (Codec codec : codecs) {
provider.register(codec);
}
for (int j = 0; j < 30 * RANDOM_MULTIPLIER; j++) {
provider.setFieldCodec("" + j, random.nextBoolean() ? "SimpleText"
: "Standard"); // TODO enable other codecs once possible
provider.setFieldCodec("" + j, codecs[random.nextInt(codecs.length)].name);
}
IndexWriterConfig config = newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer());

15
lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
View File

@ -139,7 +139,7 @@ public class MockFixedIntBlockCodec extends Codec {
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
state.segmentInfo,
state.readBufferSize,
new MockIntFactory());
new MockIntFactory(), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@ -148,7 +148,7 @@ public class MockFixedIntBlockCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId);
success = true;
} finally {
if (!success) {
@ -165,7 +165,8 @@ public class MockFixedIntBlockCodec extends Codec {
postingsReader,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -180,10 +181,10 @@ public class MockFixedIntBlockCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, codecId, files);
PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override

16
lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
View File

@ -162,7 +162,7 @@ public class MockVariableIntBlockCodec extends Codec {
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
state.segmentInfo,
state.readBufferSize,
new MockIntFactory());
new MockIntFactory(), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@ -171,7 +171,8 @@ public class MockVariableIntBlockCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.codecId);
success = true;
} finally {
if (!success) {
@ -188,7 +189,8 @@ public class MockVariableIntBlockCodec extends Codec {
postingsReader,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -203,10 +205,10 @@ public class MockVariableIntBlockCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, codecId, files);
PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override

17
lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
View File

@ -87,7 +87,8 @@ public class MockSepCodec extends Codec {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize, new MockSingleIntFactory());
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
state.readBufferSize, new MockSingleIntFactory(), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@ -96,7 +97,8 @@ public class MockSepCodec extends Codec {
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.codecId);
success = true;
} finally {
if (!success) {
@ -113,7 +115,8 @@ public class MockSepCodec extends Codec {
postingsReader,
state.readBufferSize,
BytesRef.getUTF8SortedAsUnicodeComparator(),
StandardCodec.TERMS_CACHE_SIZE);
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
return ret;
} finally {
@ -128,10 +131,10 @@ public class MockSepCodec extends Codec {
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, files);
PrefixCodedTermsReader.files(dir, segmentInfo, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, files);
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, codecId, files);
PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override