Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5969: more cleanups 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5969@1628692 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-10-01 12:50:02 +00:00
parent f0b7c715e7
commit 2890eceab4
8 changed files with 46 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java
View File

@ -94,12 +94,7 @@ public class Ords41PostingsFormat extends PostingsFormat {
state.segmentSuffix);
boolean success = false;
try {
FieldsProducer ret = new OrdsBlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
state.context,
state.segmentSuffix);
FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
success = true;
return ret;
} finally {

61
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
View File

@ -31,12 +31,9 @@ import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@ -62,42 +59,33 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
private final TreeMap<String,OrdsFieldReader> fields = new TreeMap<>();
/** File offset where the directory starts in the terms file. */
private long dirOffset;
/** File offset where the directory starts in the index file. */
private long indexDirOffset;
final String segment;
private final int version;
/** Sole constructor. */
public OrdsBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
PostingsReaderBase postingsReader, IOContext ioContext,
String segmentSuffix)
throws IOException {
public OrdsBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
this.segment = info.name;
in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_EXTENSION),
ioContext);
String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
state.segmentSuffix,
OrdsBlockTreeTermsWriter.TERMS_EXTENSION);
in = state.directory.openInput(termsFile, state.context);
boolean success = false;
IndexInput indexIn = null;
try {
version = CodecUtil.checkHeader(in,
OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME,
int version = CodecUtil.checkSegmentHeader(in, OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME,
OrdsBlockTreeTermsWriter.VERSION_START,
OrdsBlockTreeTermsWriter.VERSION_CURRENT);
indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
ioContext);
int indexVersion = CodecUtil.checkHeader(indexIn,
OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
OrdsBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
state.segmentSuffix,
OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
indexIn = state.directory.openInput(indexFile, state.context);
int indexVersion = CodecUtil.checkSegmentHeader(indexIn, OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
OrdsBlockTreeTermsWriter.VERSION_START,
OrdsBlockTreeTermsWriter.VERSION_CURRENT);
OrdsBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
if (indexVersion != version) {
throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
}
@ -116,8 +104,8 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
CodecUtil.retrieveChecksum(in);
// Read per-field details
seekDir(in, dirOffset);
seekDir(indexIn, indexDirOffset);
seekDir(in);
seekDir(indexIn);
final int numFields = in.readVInt();
if (numFields < 0) {
@ -134,7 +122,7 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
in.readBytes(code.bytes, 0, numBytes);
code.length = numBytes;
final Output rootCode = OrdsBlockTreeTermsWriter.FST_OUTPUTS.newOutput(code, 0, numTerms);
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field;
assert numTerms <= Integer.MAX_VALUE;
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
@ -145,8 +133,8 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in);
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@ -182,10 +170,9 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
}
/** Seek {@code input} to the directory offset. */
private void seekDir(IndexInput input, long dirOffset)
throws IOException {
private void seekDir(IndexInput input) throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8);
dirOffset = input.readLong();
long dirOffset = input.readLong();
input.seek(dirOffset);
}

11
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
View File

@ -42,7 +42,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.fst.Builder;
@ -115,17 +114,17 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
/** Extension of terms file */
static final String TERMS_EXTENSION = "tio";
final static String TERMS_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_DICT";
final static String TERMS_CODEC_NAME = "OrdsBlockTreeTerms";
/** Initial terms format. */
public static final int VERSION_START = 0;
public static final int VERSION_START = 1;
/** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tipo";
final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_INDEX";
final static String TERMS_INDEX_CODEC_NAME = "OrdsBlockTreeIndex";
private final IndexOutput out;
private final IndexOutput indexOut;
@ -204,11 +203,11 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT);
CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT);
CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;

2
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
View File

@ -1030,7 +1030,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum {
@Override
public String toString() {
return "OrdsSegmentTermsEnum(seg=" + fr.parent.segment + ")";
return "OrdsSegmentTermsEnum(seg=" + fr.parent + ")";
}
/** Holds a single input (IntsRef) + output pair. */

13
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
View File

@ -85,9 +85,8 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
public final class BloomFilteringPostingsFormat extends PostingsFormat {
public static final String BLOOM_CODEC_NAME = "BloomFilter";
public static final int VERSION_START = 1;
public static final int VERSION_CHECKSUM = 2;
public static final int VERSION_CURRENT = VERSION_CHECKSUM;
public static final int VERSION_START = 3;
public static final int VERSION_CURRENT = VERSION_START;
/** Extension of Bloom Filters file */
static final String BLOOM_EXTENSION = "blm";
@ -167,7 +166,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
boolean success = false;
try {
bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT);
CodecUtil.checkSegmentHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// // Load the hash function used in the BloomFilter
// hashFunction = HashFunction.forName(bloomIn.readString());
// Load the delegate postings format
@ -183,11 +182,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum);
bloomsByFieldName.put(fieldInfo.name, bloom);
}
if (version >= VERSION_CHECKSUM) {
CodecUtil.checkFooter(bloomIn);
} else {
CodecUtil.checkEOF(bloomIn);
}
IOUtils.close(bloomIn);
success = true;
} finally {
@ -507,7 +502,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
IndexOutput bloomOutput = null;
try {
bloomOutput = state.directory.createOutput(bloomFileName, state.context);
CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT);
CodecUtil.writeSegmentHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName());

4
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
View File

@ -52,10 +52,10 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {

12
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
View File

@ -86,7 +86,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
static final byte SORTED_NUMERIC = 5;
static final byte SORTED_NUMERIC_SINGLETON = 6;
static final int VERSION_START = 2;
static final int VERSION_START = 3;
static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s
@ -122,9 +122,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
boolean success = false;
try {
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
VERSION_CURRENT);
version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
@ -141,9 +140,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
this.data = state.directory.openInput(dataName, state.context);
success = false;
try {
final int version2 = CodecUtil.checkHeader(data, dataCodec,
VERSION_START,
VERSION_CURRENT);
final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
}

7
lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
View File

@ -383,12 +383,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
boolean success = false;
try {
fields = new OrdsBlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
state.context,
state.segmentSuffix);
fields = new OrdsBlockTreeTermsReader(postingsReader, state);
success = true;
} finally {
if (!success) {