mirror of https://github.com/apache/lucene.git
LUCENE-5969: give remaining codecs segment headers and cleanups
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5969@1628697 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2890eceab4
commit
6ab3036b6a
|
@ -75,7 +75,6 @@ public class FSTOrdTermsReader extends FieldsProducer {
|
|||
static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
|
||||
final TreeMap<String, TermsReader> fields = new TreeMap<>();
|
||||
final PostingsReaderBase postingsReader;
|
||||
int version;
|
||||
//static final boolean TEST = false;
|
||||
|
||||
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
|
||||
|
@ -89,12 +88,21 @@ public class FSTOrdTermsReader extends FieldsProducer {
|
|||
try {
|
||||
indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
|
||||
blockIn = state.directory.openInput(termsBlockFileName, state.context);
|
||||
version = readHeader(indexIn);
|
||||
readHeader(blockIn);
|
||||
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
|
||||
CodecUtil.checksumEntireFile(blockIn);
|
||||
int version = CodecUtil.checkSegmentHeader(indexIn, FSTOrdTermsWriter.TERMS_INDEX_CODEC_NAME,
|
||||
FSTOrdTermsWriter.VERSION_START,
|
||||
FSTOrdTermsWriter.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
int version2 = CodecUtil.checkSegmentHeader(blockIn, FSTOrdTermsWriter.TERMS_CODEC_NAME,
|
||||
FSTOrdTermsWriter.VERSION_START,
|
||||
FSTOrdTermsWriter.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
if (version != version2) {
|
||||
throw new CorruptIndexException("Format versions mismatch: index=" + version + ", terms=" + version2, blockIn);
|
||||
}
|
||||
|
||||
CodecUtil.checksumEntireFile(blockIn);
|
||||
|
||||
this.postingsReader.init(blockIn);
|
||||
seekDir(blockIn);
|
||||
|
||||
|
@ -114,11 +122,7 @@ public class FSTOrdTermsReader extends FieldsProducer {
|
|||
TermsReader previous = fields.put(fieldInfo.name, current);
|
||||
checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
|
||||
}
|
||||
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
|
||||
CodecUtil.checkFooter(indexIn);
|
||||
} else {
|
||||
CodecUtil.checkEOF(indexIn);
|
||||
}
|
||||
success = true;
|
||||
} finally {
|
||||
if (success) {
|
||||
|
@ -129,17 +133,8 @@ public class FSTOrdTermsReader extends FieldsProducer {
|
|||
}
|
||||
}
|
||||
|
||||
private int readHeader(IndexInput in) throws IOException {
|
||||
return CodecUtil.checkHeader(in, FSTOrdTermsWriter.TERMS_CODEC_NAME,
|
||||
FSTOrdTermsWriter.TERMS_VERSION_START,
|
||||
FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
|
||||
}
|
||||
private void seekDir(IndexInput in) throws IOException {
|
||||
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
|
||||
in.seek(in.length() - CodecUtil.footerLength() - 8);
|
||||
} else {
|
||||
in.seek(in.length() - 8);
|
||||
}
|
||||
in.seek(in.readLong());
|
||||
}
|
||||
private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException {
|
||||
|
|
|
@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutputStream;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.IntsRef;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.fst.Builder;
|
||||
import org.apache.lucene.util.fst.FST;
|
||||
|
@ -149,10 +148,11 @@ import org.apache.lucene.util.fst.Util;
|
|||
public class FSTOrdTermsWriter extends FieldsConsumer {
|
||||
static final String TERMS_INDEX_EXTENSION = "tix";
|
||||
static final String TERMS_BLOCK_EXTENSION = "tbk";
|
||||
static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
|
||||
public static final int TERMS_VERSION_START = 0;
|
||||
public static final int TERMS_VERSION_CHECKSUM = 1;
|
||||
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
|
||||
static final String TERMS_CODEC_NAME = "FSTOrdTerms";
|
||||
static final String TERMS_INDEX_CODEC_NAME = "FSTOrdIndex";
|
||||
|
||||
public static final int VERSION_START = 2;
|
||||
public static final int VERSION_CURRENT = VERSION_START;
|
||||
public static final int SKIP_INTERVAL = 8;
|
||||
|
||||
final PostingsWriterBase postingsWriter;
|
||||
|
@ -174,8 +174,10 @@ public class FSTOrdTermsWriter extends FieldsConsumer {
|
|||
try {
|
||||
this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
|
||||
this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
|
||||
writeHeader(indexOut);
|
||||
writeHeader(blockOut);
|
||||
CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
CodecUtil.writeSegmentHeader(blockOut, TERMS_CODEC_NAME, VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
this.postingsWriter.init(blockOut);
|
||||
success = true;
|
||||
} finally {
|
||||
|
@ -260,9 +262,6 @@ public class FSTOrdTermsWriter extends FieldsConsumer {
|
|||
}
|
||||
}
|
||||
|
||||
private void writeHeader(IndexOutput out) throws IOException {
|
||||
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
|
||||
}
|
||||
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
|
||||
out.writeLong(dirStart);
|
||||
}
|
||||
|
|
|
@ -72,7 +72,6 @@ public class FSTTermsReader extends FieldsProducer {
|
|||
final TreeMap<String, TermsReader> fields = new TreeMap<>();
|
||||
final PostingsReaderBase postingsReader;
|
||||
//static boolean TEST = false;
|
||||
final int version;
|
||||
|
||||
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
|
||||
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
|
||||
|
@ -82,10 +81,11 @@ public class FSTTermsReader extends FieldsProducer {
|
|||
|
||||
boolean success = false;
|
||||
try {
|
||||
version = readHeader(in);
|
||||
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
|
||||
CodecUtil.checkSegmentHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
|
||||
FSTTermsWriter.TERMS_VERSION_START,
|
||||
FSTTermsWriter.TERMS_VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
CodecUtil.checksumEntireFile(in);
|
||||
}
|
||||
this.postingsReader.init(in);
|
||||
seekDir(in);
|
||||
|
||||
|
@ -113,17 +113,8 @@ public class FSTTermsReader extends FieldsProducer {
|
|||
}
|
||||
}
|
||||
|
||||
private int readHeader(IndexInput in) throws IOException {
|
||||
return CodecUtil.checkHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
|
||||
FSTTermsWriter.TERMS_VERSION_START,
|
||||
FSTTermsWriter.TERMS_VERSION_CURRENT);
|
||||
}
|
||||
private void seekDir(IndexInput in) throws IOException {
|
||||
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
|
||||
in.seek(in.length() - CodecUtil.footerLength() - 8);
|
||||
} else {
|
||||
in.seek(in.length() - 8);
|
||||
}
|
||||
in.seek(in.readLong());
|
||||
}
|
||||
private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException {
|
||||
|
|
|
@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutputStream;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.IntsRef;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.fst.Builder;
|
||||
import org.apache.lucene.util.fst.FST;
|
||||
|
@ -123,10 +122,9 @@ import org.apache.lucene.util.fst.Util;
|
|||
|
||||
public class FSTTermsWriter extends FieldsConsumer {
|
||||
static final String TERMS_EXTENSION = "tmp";
|
||||
static final String TERMS_CODEC_NAME = "FST_TERMS_DICT";
|
||||
public static final int TERMS_VERSION_START = 0;
|
||||
public static final int TERMS_VERSION_CHECKSUM = 1;
|
||||
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
|
||||
static final String TERMS_CODEC_NAME = "FSTTerms";
|
||||
public static final int TERMS_VERSION_START = 2;
|
||||
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
|
||||
|
||||
final PostingsWriterBase postingsWriter;
|
||||
final FieldInfos fieldInfos;
|
||||
|
@ -144,7 +142,9 @@ public class FSTTermsWriter extends FieldsConsumer {
|
|||
|
||||
boolean success = false;
|
||||
try {
|
||||
writeHeader(out);
|
||||
CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
this.postingsWriter.init(out);
|
||||
success = true;
|
||||
} finally {
|
||||
|
@ -154,10 +154,6 @@ public class FSTTermsWriter extends FieldsConsumer {
|
|||
}
|
||||
}
|
||||
|
||||
private void writeHeader(IndexOutput out) throws IOException {
|
||||
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
|
||||
}
|
||||
|
||||
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
|
||||
out.writeLong(dirStart);
|
||||
}
|
||||
|
|
|
@ -33,7 +33,6 @@ import org.apache.lucene.store.IndexOutput;
|
|||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.IntsRef;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.MathUtil;
|
||||
import org.apache.lucene.util.fst.Builder;
|
||||
|
@ -75,10 +74,10 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
|
|||
try {
|
||||
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
|
||||
data = state.directory.createOutput(dataName, state.context);
|
||||
CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
|
||||
CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
|
||||
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
|
||||
meta = state.directory.createOutput(metaName, state.context);
|
||||
CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
|
||||
CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
|
|
|
@ -110,7 +110,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
|
|||
static final byte BLOCK_COMPRESSED = 2;
|
||||
static final byte GCD_COMPRESSED = 3;
|
||||
|
||||
static final int VERSION_START = 3;
|
||||
static final int VERSION_START = 4;
|
||||
static final int VERSION_CURRENT = VERSION_START;
|
||||
|
||||
// clone for merge: when merging we don't do any instances.put()s
|
||||
|
@ -146,9 +146,8 @@ class MemoryDocValuesProducer extends DocValuesProducer {
|
|||
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
|
||||
boolean success = false;
|
||||
try {
|
||||
version = CodecUtil.checkHeader(in, metaCodec,
|
||||
VERSION_START,
|
||||
VERSION_CURRENT);
|
||||
version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
numEntries = readFields(in, state.fieldInfos);
|
||||
CodecUtil.checkFooter(in);
|
||||
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
|
||||
|
@ -165,9 +164,8 @@ class MemoryDocValuesProducer extends DocValuesProducer {
|
|||
this.data = state.directory.openInput(dataName, state.context);
|
||||
success = false;
|
||||
try {
|
||||
final int version2 = CodecUtil.checkHeader(data, dataCodec,
|
||||
VERSION_START,
|
||||
VERSION_CURRENT);
|
||||
final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
if (version != version2) {
|
||||
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
|
||||
}
|
||||
|
|
|
@ -99,15 +99,10 @@ public class IDVersionPostingsFormat extends PostingsFormat {
|
|||
|
||||
@Override
|
||||
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
|
||||
PostingsReaderBase postingsReader = new IDVersionPostingsReader();
|
||||
PostingsReaderBase postingsReader = new IDVersionPostingsReader(state);
|
||||
boolean success = false;
|
||||
try {
|
||||
FieldsProducer ret = new VersionBlockTreeTermsReader(state.directory,
|
||||
state.fieldInfos,
|
||||
state.segmentInfo,
|
||||
postingsReader,
|
||||
state.context,
|
||||
state.segmentSuffix);
|
||||
FieldsProducer ret = new VersionBlockTreeTermsReader(postingsReader, state);
|
||||
success = true;
|
||||
return ret;
|
||||
} finally {
|
||||
|
|
|
@ -26,21 +26,27 @@ import org.apache.lucene.codecs.PostingsReaderBase;
|
|||
import org.apache.lucene.index.DocsAndPositionsEnum;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.store.DataInput;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.util.Accountable;
|
||||
import org.apache.lucene.util.BitUtil;
|
||||
import org.apache.lucene.util.Bits;
|
||||
|
||||
final class IDVersionPostingsReader extends PostingsReaderBase {
|
||||
final SegmentReadState state;
|
||||
|
||||
public IDVersionPostingsReader(SegmentReadState state) {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(IndexInput termsIn) throws IOException {
|
||||
// Make sure we are talking to the matching postings writer
|
||||
CodecUtil.checkHeader(termsIn,
|
||||
CodecUtil.checkSegmentHeader(termsIn,
|
||||
IDVersionPostingsWriter.TERMS_CODEC,
|
||||
IDVersionPostingsWriter.VERSION_START,
|
||||
IDVersionPostingsWriter.VERSION_CURRENT);
|
||||
IDVersionPostingsWriter.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldInfo;
|
|||
import org.apache.lucene.index.SegmentWriteState;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.util.BitUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
final class IDVersionPostingsWriter extends PushPostingsWriterBase {
|
||||
|
@ -34,7 +33,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
|
|||
final static String TERMS_CODEC = "IDVersionPostingsWriterTerms";
|
||||
|
||||
// Increment version to change it
|
||||
final static int VERSION_START = 0;
|
||||
final static int VERSION_START = 1;
|
||||
final static int VERSION_CURRENT = VERSION_START;
|
||||
|
||||
final static IDVersionTermState emptyState = new IDVersionTermState();
|
||||
|
@ -57,7 +56,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
|
|||
|
||||
@Override
|
||||
public void init(IndexOutput termsOut) throws IOException {
|
||||
CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT);
|
||||
CodecUtil.writeSegmentHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -283,7 +283,7 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum {
|
|||
targetUpto = 0;
|
||||
|
||||
IDVersionSegmentTermsEnumFrame lastFrame = stack[0];
|
||||
assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent.segment;
|
||||
assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent;
|
||||
|
||||
final int targetLimit = Math.min(target.length, validIndexPrefix);
|
||||
|
||||
|
@ -1063,6 +1063,6 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum {
|
|||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "IDVersionSegmentTermsEnum(seg=" + fr.parent.segment + ")";
|
||||
return "IDVersionSegmentTermsEnum(seg=" + fr.parent + ")";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,12 +29,9 @@ import org.apache.lucene.codecs.FieldsProducer;
|
|||
import org.apache.lucene.codecs.PostingsReaderBase;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.IndexFileNames;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.index.SegmentReadState;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.util.Accountable;
|
||||
import org.apache.lucene.util.Accountables;
|
||||
|
@ -61,38 +58,36 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
|
|||
|
||||
private final TreeMap<String,VersionFieldReader> fields = new TreeMap<>();
|
||||
|
||||
/** File offset where the directory starts in the terms file. */
|
||||
private long dirOffset;
|
||||
|
||||
/** File offset where the directory starts in the index file. */
|
||||
private long indexDirOffset;
|
||||
|
||||
final String segment;
|
||||
|
||||
private final int version;
|
||||
|
||||
/** Sole constructor. */
|
||||
public VersionBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
|
||||
PostingsReaderBase postingsReader, IOContext ioContext,
|
||||
String segmentSuffix)
|
||||
throws IOException {
|
||||
public VersionBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
|
||||
|
||||
this.postingsReader = postingsReader;
|
||||
|
||||
this.segment = info.name;
|
||||
in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_EXTENSION),
|
||||
ioContext);
|
||||
String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
|
||||
state.segmentSuffix,
|
||||
VersionBlockTreeTermsWriter.TERMS_EXTENSION);
|
||||
in = state.directory.openInput(termsFile, state.context);
|
||||
|
||||
boolean success = false;
|
||||
IndexInput indexIn = null;
|
||||
|
||||
try {
|
||||
version = readHeader(in);
|
||||
indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
|
||||
ioContext);
|
||||
int indexVersion = readIndexHeader(indexIn);
|
||||
if (indexVersion != version) {
|
||||
throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
|
||||
int termsVersion = CodecUtil.checkSegmentHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
|
||||
VersionBlockTreeTermsWriter.VERSION_START,
|
||||
VersionBlockTreeTermsWriter.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
|
||||
state.segmentSuffix,
|
||||
VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
|
||||
indexIn = state.directory.openInput(indexFile, state.context);
|
||||
int indexVersion = CodecUtil.checkSegmentHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
|
||||
VersionBlockTreeTermsWriter.VERSION_START,
|
||||
VersionBlockTreeTermsWriter.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
if (indexVersion != termsVersion) {
|
||||
throw new CorruptIndexException("mixmatched version files: " + in + "=" + termsVersion + "," + indexIn + "=" + indexVersion, indexIn);
|
||||
}
|
||||
|
||||
// verify
|
||||
|
@ -108,8 +103,8 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
|
|||
CodecUtil.retrieveChecksum(in);
|
||||
|
||||
// Read per-field details
|
||||
seekDir(in, dirOffset);
|
||||
seekDir(indexIn, indexDirOffset);
|
||||
seekDir(in);
|
||||
seekDir(indexIn);
|
||||
|
||||
final int numFields = in.readVInt();
|
||||
if (numFields < 0) {
|
||||
|
@ -126,7 +121,7 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
|
|||
code.length = numBytes;
|
||||
final long version = in.readVLong();
|
||||
final Pair<BytesRef,Long> rootCode = VersionBlockTreeTermsWriter.FST_OUTPUTS.newPair(code, version);
|
||||
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
|
||||
final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
|
||||
assert fieldInfo != null: "field=" + field;
|
||||
final long sumTotalTermFreq = numTerms;
|
||||
final long sumDocFreq = numTerms;
|
||||
|
@ -136,8 +131,8 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
|
|||
|
||||
BytesRef minTerm = readBytesRef(in);
|
||||
BytesRef maxTerm = readBytesRef(in);
|
||||
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
|
||||
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
|
||||
if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
|
||||
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
|
||||
}
|
||||
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
|
||||
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
|
||||
|
@ -172,27 +167,10 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
|
|||
return bytes;
|
||||
}
|
||||
|
||||
/** Reads terms file header. */
|
||||
private int readHeader(IndexInput input) throws IOException {
|
||||
int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
|
||||
VersionBlockTreeTermsWriter.VERSION_START,
|
||||
VersionBlockTreeTermsWriter.VERSION_CURRENT);
|
||||
return version;
|
||||
}
|
||||
|
||||
/** Reads index file header. */
|
||||
private int readIndexHeader(IndexInput input) throws IOException {
|
||||
int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
|
||||
VersionBlockTreeTermsWriter.VERSION_START,
|
||||
VersionBlockTreeTermsWriter.VERSION_CURRENT);
|
||||
return version;
|
||||
}
|
||||
|
||||
/** Seek {@code input} to the directory offset. */
|
||||
private void seekDir(IndexInput input, long dirOffset)
|
||||
throws IOException {
|
||||
private void seekDir(IndexInput input) throws IOException {
|
||||
input.seek(input.length() - CodecUtil.footerLength() - 8);
|
||||
dirOffset = input.readLong();
|
||||
long dirOffset = input.readLong();
|
||||
input.seek(dirOffset);
|
||||
}
|
||||
|
||||
|
|
|
@ -121,7 +121,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
|
|||
final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT";
|
||||
|
||||
/** Initial terms format. */
|
||||
public static final int VERSION_START = 0;
|
||||
public static final int VERSION_START = 1;
|
||||
|
||||
/** Current terms format. */
|
||||
public static final int VERSION_CURRENT = VERSION_START;
|
||||
|
@ -199,13 +199,13 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
|
|||
fieldInfos = state.fieldInfos;
|
||||
this.minItemsInBlock = minItemsInBlock;
|
||||
this.maxItemsInBlock = maxItemsInBlock;
|
||||
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT);
|
||||
CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
//DEBUG = state.segmentName.equals("_4a");
|
||||
|
||||
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
|
||||
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
|
||||
CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT);
|
||||
CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
|
||||
|
||||
this.postingsWriter = postingsWriter;
|
||||
// segment = state.segmentInfo.name;
|
||||
|
|
Loading…
Reference in New Issue