Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5969: give remaining codecs segment headers and cleanups 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5969@1628697 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-10-01 13:14:41 +00:00
parent 2890eceab4
commit 6ab3036b6a
12 changed files with 92 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
View File

@ -75,7 +75,6 @@ public class FSTOrdTermsReader extends FieldsProducer {
static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL; static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
final TreeMap<String, TermsReader> fields = new TreeMap<>(); final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader; final PostingsReaderBase postingsReader;
int version;
//static final boolean TEST = false; //static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
@ -89,11 +88,20 @@ public class FSTOrdTermsReader extends FieldsProducer {
try { try {
indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context); indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
blockIn = state.directory.openInput(termsBlockFileName, state.context); blockIn = state.directory.openInput(termsBlockFileName, state.context);
version = readHeader(indexIn); int version = CodecUtil.checkSegmentHeader(indexIn, FSTOrdTermsWriter.TERMS_INDEX_CODEC_NAME,
readHeader(blockIn); FSTOrdTermsWriter.VERSION_START,
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) { FSTOrdTermsWriter.VERSION_CURRENT,
CodecUtil.checksumEntireFile(blockIn); state.segmentInfo.getId(), state.segmentSuffix);
int version2 = CodecUtil.checkSegmentHeader(blockIn, FSTOrdTermsWriter.TERMS_CODEC_NAME,
FSTOrdTermsWriter.VERSION_START,
FSTOrdTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: index=" + version + ", terms=" + version2, blockIn);
} }
CodecUtil.checksumEntireFile(blockIn);
this.postingsReader.init(blockIn); this.postingsReader.init(blockIn);
seekDir(blockIn); seekDir(blockIn);
@ -114,11 +122,7 @@ public class FSTOrdTermsReader extends FieldsProducer {
TermsReader previous = fields.put(fieldInfo.name, current); TermsReader previous = fields.put(fieldInfo.name, current);
checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous); checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
} }
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) { CodecUtil.checkFooter(indexIn);
CodecUtil.checkFooter(indexIn);
} else {
CodecUtil.checkEOF(indexIn);
}
success = true; success = true;
} finally { } finally {
if (success) { if (success) {
@ -129,17 +133,8 @@ public class FSTOrdTermsReader extends FieldsProducer {
} }
} }
private int readHeader(IndexInput in) throws IOException {
return CodecUtil.checkHeader(in, FSTOrdTermsWriter.TERMS_CODEC_NAME,
FSTOrdTermsWriter.TERMS_VERSION_START,
FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
}
private void seekDir(IndexInput in) throws IOException { private void seekDir(IndexInput in) throws IOException {
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) { in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.length() - CodecUtil.footerLength() - 8);
} else {
in.seek(in.length() - 8);
}
in.seek(in.readLong()); in.seek(in.readLong());
} }
private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException { private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException {

19
lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
View File

@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder; import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.FST;
@ -149,10 +148,11 @@ import org.apache.lucene.util.fst.Util;
public class FSTOrdTermsWriter extends FieldsConsumer { public class FSTOrdTermsWriter extends FieldsConsumer {
static final String TERMS_INDEX_EXTENSION = "tix"; static final String TERMS_INDEX_EXTENSION = "tix";
static final String TERMS_BLOCK_EXTENSION = "tbk"; static final String TERMS_BLOCK_EXTENSION = "tbk";
static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT"; static final String TERMS_CODEC_NAME = "FSTOrdTerms";
public static final int TERMS_VERSION_START = 0; static final String TERMS_INDEX_CODEC_NAME = "FSTOrdIndex";
public static final int TERMS_VERSION_CHECKSUM = 1;
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM; public static final int VERSION_START = 2;
public static final int VERSION_CURRENT = VERSION_START;
public static final int SKIP_INTERVAL = 8; public static final int SKIP_INTERVAL = 8;
final PostingsWriterBase postingsWriter; final PostingsWriterBase postingsWriter;
@ -174,8 +174,10 @@ public class FSTOrdTermsWriter extends FieldsConsumer {
try { try {
this.indexOut = state.directory.createOutput(termsIndexFileName, state.context); this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
this.blockOut = state.directory.createOutput(termsBlockFileName, state.context); this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
writeHeader(indexOut); CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT,
writeHeader(blockOut); state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.writeSegmentHeader(blockOut, TERMS_CODEC_NAME, VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter.init(blockOut); this.postingsWriter.init(blockOut);
success = true; success = true;
} finally { } finally {
@ -260,9 +262,6 @@ public class FSTOrdTermsWriter extends FieldsConsumer {
} }
} }
private void writeHeader(IndexOutput out) throws IOException {
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
}
private void writeTrailer(IndexOutput out, long dirStart) throws IOException { private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart); out.writeLong(dirStart);
} }

21
lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
View File

@ -72,7 +72,6 @@ public class FSTTermsReader extends FieldsProducer {
final TreeMap<String, TermsReader> fields = new TreeMap<>(); final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader; final PostingsReaderBase postingsReader;
//static boolean TEST = false; //static boolean TEST = false;
final int version;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION); final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
@ -82,10 +81,11 @@ public class FSTTermsReader extends FieldsProducer {
boolean success = false; boolean success = false;
try { try {
version = readHeader(in); CodecUtil.checkSegmentHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { FSTTermsWriter.TERMS_VERSION_START,
CodecUtil.checksumEntireFile(in); FSTTermsWriter.TERMS_VERSION_CURRENT,
} state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.checksumEntireFile(in);
this.postingsReader.init(in); this.postingsReader.init(in);
seekDir(in); seekDir(in);
@ -113,17 +113,8 @@ public class FSTTermsReader extends FieldsProducer {
} }
} }
private int readHeader(IndexInput in) throws IOException {
return CodecUtil.checkHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
FSTTermsWriter.TERMS_VERSION_START,
FSTTermsWriter.TERMS_VERSION_CURRENT);
}
private void seekDir(IndexInput in) throws IOException { private void seekDir(IndexInput in) throws IOException {
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.length() - CodecUtil.footerLength() - 8);
} else {
in.seek(in.length() - 8);
}
in.seek(in.readLong()); in.seek(in.readLong());
} }
private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException { private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException {

16
lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
View File

@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder; import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.FST;
@ -123,10 +122,9 @@ import org.apache.lucene.util.fst.Util;
public class FSTTermsWriter extends FieldsConsumer { public class FSTTermsWriter extends FieldsConsumer {
static final String TERMS_EXTENSION = "tmp"; static final String TERMS_EXTENSION = "tmp";
static final String TERMS_CODEC_NAME = "FST_TERMS_DICT"; static final String TERMS_CODEC_NAME = "FSTTerms";
public static final int TERMS_VERSION_START = 0; public static final int TERMS_VERSION_START = 2;
public static final int TERMS_VERSION_CHECKSUM = 1; public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
final PostingsWriterBase postingsWriter; final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos; final FieldInfos fieldInfos;
@ -144,7 +142,9 @@ public class FSTTermsWriter extends FieldsConsumer {
boolean success = false; boolean success = false;
try { try {
writeHeader(out); CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter.init(out); this.postingsWriter.init(out);
success = true; success = true;
} finally { } finally {
@ -154,10 +154,6 @@ public class FSTTermsWriter extends FieldsConsumer {
} }
} }
private void writeHeader(IndexOutput out) throws IOException {
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
}
private void writeTrailer(IndexOutput out, long dirStart) throws IOException { private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart); out.writeLong(dirStart);
} }

5
lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder; import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.MathUtil; import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.Builder;
@ -75,10 +74,10 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
try { try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context); data = state.directory.createOutput(dataName, state.context);
CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT); CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context); meta = state.directory.createOutput(metaName, state.context);
CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT); CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true; success = true;
} finally { } finally {
if (!success) { if (!success) {

12
lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
View File

@ -110,7 +110,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
static final byte BLOCK_COMPRESSED = 2; static final byte BLOCK_COMPRESSED = 2;
static final byte GCD_COMPRESSED = 3; static final byte GCD_COMPRESSED = 3;
static final int VERSION_START = 3; static final int VERSION_START = 4;
static final int VERSION_CURRENT = VERSION_START; static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s // clone for merge: when merging we don't do any instances.put()s
@ -146,9 +146,8 @@ class MemoryDocValuesProducer extends DocValuesProducer {
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context); ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
boolean success = false; boolean success = false;
try { try {
version = CodecUtil.checkHeader(in, metaCodec, version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
VERSION_CURRENT);
numEntries = readFields(in, state.fieldInfos); numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in); CodecUtil.checkFooter(in);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
@ -165,9 +164,8 @@ class MemoryDocValuesProducer extends DocValuesProducer {
this.data = state.directory.openInput(dataName, state.context); this.data = state.directory.openInput(dataName, state.context);
success = false; success = false;
try { try {
final int version2 = CodecUtil.checkHeader(data, dataCodec, final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
VERSION_CURRENT);
if (version != version2) { if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data); throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
} }

9
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
View File

@ -99,15 +99,10 @@ public class IDVersionPostingsFormat extends PostingsFormat {
@Override @Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new IDVersionPostingsReader(); PostingsReaderBase postingsReader = new IDVersionPostingsReader(state);
boolean success = false; boolean success = false;
try { try {
FieldsProducer ret = new VersionBlockTreeTermsReader(state.directory, FieldsProducer ret = new VersionBlockTreeTermsReader(postingsReader, state);
state.fieldInfos,
state.segmentInfo,
postingsReader,
state.context,
state.segmentSuffix);
success = true; success = true;
return ret; return ret;
} finally { } finally {

16
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
View File

@ -26,21 +26,27 @@ import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
final class IDVersionPostingsReader extends PostingsReaderBase { final class IDVersionPostingsReader extends PostingsReaderBase {
final SegmentReadState state;
public IDVersionPostingsReader(SegmentReadState state) {
this.state = state;
}
@Override @Override
public void init(IndexInput termsIn) throws IOException { public void init(IndexInput termsIn) throws IOException {
// Make sure we are talking to the matching postings writer // Make sure we are talking to the matching postings writer
CodecUtil.checkHeader(termsIn, CodecUtil.checkSegmentHeader(termsIn,
IDVersionPostingsWriter.TERMS_CODEC, IDVersionPostingsWriter.TERMS_CODEC,
IDVersionPostingsWriter.VERSION_START, IDVersionPostingsWriter.VERSION_START,
IDVersionPostingsWriter.VERSION_CURRENT); IDVersionPostingsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
} }
@Override @Override

5
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
View File

@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
final class IDVersionPostingsWriter extends PushPostingsWriterBase { final class IDVersionPostingsWriter extends PushPostingsWriterBase {
@ -34,7 +33,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
final static String TERMS_CODEC = "IDVersionPostingsWriterTerms"; final static String TERMS_CODEC = "IDVersionPostingsWriterTerms";
// Increment version to change it // Increment version to change it
final static int VERSION_START = 0; final static int VERSION_START = 1;
final static int VERSION_CURRENT = VERSION_START; final static int VERSION_CURRENT = VERSION_START;
final static IDVersionTermState emptyState = new IDVersionTermState(); final static IDVersionTermState emptyState = new IDVersionTermState();
@ -57,7 +56,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
@Override @Override
public void init(IndexOutput termsOut) throws IOException { public void init(IndexOutput termsOut) throws IOException {
CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT); CodecUtil.writeSegmentHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
} }
@Override @Override

4
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
View File

@ -283,7 +283,7 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum {
targetUpto = 0; targetUpto = 0;
IDVersionSegmentTermsEnumFrame lastFrame = stack[0]; IDVersionSegmentTermsEnumFrame lastFrame = stack[0];
assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent.segment; assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent;
final int targetLimit = Math.min(target.length, validIndexPrefix); final int targetLimit = Math.min(target.length, validIndexPrefix);
@ -1063,6 +1063,6 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum {
@Override @Override
public String toString() { public String toString() {
return "IDVersionSegmentTermsEnum(seg=" + fr.parent.segment + ")"; return "IDVersionSegmentTermsEnum(seg=" + fr.parent + ")";
} }
} }

80
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
View File

@ -29,12 +29,9 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms; import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables; import org.apache.lucene.util.Accountables;
@ -61,38 +58,36 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
private final TreeMap<String,VersionFieldReader> fields = new TreeMap<>(); private final TreeMap<String,VersionFieldReader> fields = new TreeMap<>();
/** File offset where the directory starts in the terms file. */
private long dirOffset;
/** File offset where the directory starts in the index file. */
private long indexDirOffset;
final String segment;
private final int version;
/** Sole constructor. */ /** Sole constructor. */
public VersionBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info, public VersionBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader, IOContext ioContext,
String segmentSuffix)
throws IOException {
this.postingsReader = postingsReader; this.postingsReader = postingsReader;
this.segment = info.name; String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_EXTENSION), state.segmentSuffix,
ioContext); VersionBlockTreeTermsWriter.TERMS_EXTENSION);
in = state.directory.openInput(termsFile, state.context);
boolean success = false; boolean success = false;
IndexInput indexIn = null; IndexInput indexIn = null;
try { try {
version = readHeader(in); int termsVersion = CodecUtil.checkSegmentHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION), VersionBlockTreeTermsWriter.VERSION_START,
ioContext); VersionBlockTreeTermsWriter.VERSION_CURRENT,
int indexVersion = readIndexHeader(indexIn); state.segmentInfo.getId(), state.segmentSuffix);
if (indexVersion != version) {
throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn); String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
state.segmentSuffix,
VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
indexIn = state.directory.openInput(indexFile, state.context);
int indexVersion = CodecUtil.checkSegmentHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
VersionBlockTreeTermsWriter.VERSION_START,
VersionBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
if (indexVersion != termsVersion) {
throw new CorruptIndexException("mixmatched version files: " + in + "=" + termsVersion + "," + indexIn + "=" + indexVersion, indexIn);
} }
// verify // verify
@ -108,8 +103,8 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
CodecUtil.retrieveChecksum(in); CodecUtil.retrieveChecksum(in);
// Read per-field details // Read per-field details
seekDir(in, dirOffset); seekDir(in);
seekDir(indexIn, indexDirOffset); seekDir(indexIn);
final int numFields = in.readVInt(); final int numFields = in.readVInt();
if (numFields < 0) { if (numFields < 0) {
@ -126,7 +121,7 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
code.length = numBytes; code.length = numBytes;
final long version = in.readVLong(); final long version = in.readVLong();
final Pair<BytesRef,Long> rootCode = VersionBlockTreeTermsWriter.FST_OUTPUTS.newPair(code, version); final Pair<BytesRef,Long> rootCode = VersionBlockTreeTermsWriter.FST_OUTPUTS.newPair(code, version);
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field; assert fieldInfo != null: "field=" + field;
final long sumTotalTermFreq = numTerms; final long sumTotalTermFreq = numTerms;
final long sumDocFreq = numTerms; final long sumDocFreq = numTerms;
@ -136,8 +131,8 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
BytesRef minTerm = readBytesRef(in); BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in); BytesRef maxTerm = readBytesRef(in);
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in); throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
} }
if (sumDocFreq < docCount) { // #postings must be >= #docs with field if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in); throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@ -172,27 +167,10 @@ public final class VersionBlockTreeTermsReader extends FieldsProducer {
return bytes; return bytes;
} }
/** Reads terms file header. */
private int readHeader(IndexInput input) throws IOException {
int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
VersionBlockTreeTermsWriter.VERSION_START,
VersionBlockTreeTermsWriter.VERSION_CURRENT);
return version;
}
/** Reads index file header. */
private int readIndexHeader(IndexInput input) throws IOException {
int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
VersionBlockTreeTermsWriter.VERSION_START,
VersionBlockTreeTermsWriter.VERSION_CURRENT);
return version;
}
/** Seek {@code input} to the directory offset. */ /** Seek {@code input} to the directory offset. */
private void seekDir(IndexInput input, long dirOffset) private void seekDir(IndexInput input) throws IOException {
throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8); input.seek(input.length() - CodecUtil.footerLength() - 8);
dirOffset = input.readLong(); long dirOffset = input.readLong();
input.seek(dirOffset); input.seek(dirOffset);
} }

6
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
View File

@ -121,7 +121,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT"; final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT";
/** Initial terms format. */ /** Initial terms format. */
public static final int VERSION_START = 0; public static final int VERSION_START = 1;
/** Current terms format. */ /** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START; public static final int VERSION_CURRENT = VERSION_START;
@ -199,13 +199,13 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
fieldInfos = state.fieldInfos; fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock; this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock; this.maxItemsInBlock = maxItemsInBlock;
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT); CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
//DEBUG = state.segmentName.equals("_4a"); //DEBUG = state.segmentName.equals("_4a");
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION); final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context); indexOut = state.directory.createOutput(termsIndexFileName, state.context);
CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT); CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter; this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name; // segment = state.segmentInfo.name;