Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5969: more cleanups 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5969@1628692 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-10-01 12:50:02 +00:00
parent f0b7c715e7
commit 2890eceab4
8 changed files with 46 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java
View File

@ -94,12 +94,7 @@ public class Ords41PostingsFormat extends PostingsFormat {
state.segmentSuffix); state.segmentSuffix);
boolean success = false; boolean success = false;
try { try {
FieldsProducer ret = new OrdsBlockTreeTermsReader(state.directory, FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
state.fieldInfos,
state.segmentInfo,
postingsReader,
state.context,
state.segmentSuffix);
success = true; success = true;
return ret; return ret;
} finally { } finally {

65
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
View File

@ -31,12 +31,9 @@ import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms; import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables; import org.apache.lucene.util.Accountables;
@ -62,42 +59,33 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
private final TreeMap<String,OrdsFieldReader> fields = new TreeMap<>(); private final TreeMap<String,OrdsFieldReader> fields = new TreeMap<>();
/** File offset where the directory starts in the terms file. */
private long dirOffset;
/** File offset where the directory starts in the index file. */
private long indexDirOffset;
final String segment;
private final int version;
/** Sole constructor. */ /** Sole constructor. */
public OrdsBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info, public OrdsBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader, IOContext ioContext,
String segmentSuffix)
throws IOException {
this.postingsReader = postingsReader; this.postingsReader = postingsReader;
this.segment = info.name; String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_EXTENSION), state.segmentSuffix,
ioContext); OrdsBlockTreeTermsWriter.TERMS_EXTENSION);
in = state.directory.openInput(termsFile, state.context);
boolean success = false; boolean success = false;
IndexInput indexIn = null; IndexInput indexIn = null;
try { try {
version = CodecUtil.checkHeader(in, int version = CodecUtil.checkSegmentHeader(in, OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME,
OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME, OrdsBlockTreeTermsWriter.VERSION_START,
OrdsBlockTreeTermsWriter.VERSION_START, OrdsBlockTreeTermsWriter.VERSION_CURRENT,
OrdsBlockTreeTermsWriter.VERSION_CURRENT); state.segmentInfo.getId(), state.segmentSuffix);
indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
ioContext); String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
int indexVersion = CodecUtil.checkHeader(indexIn, state.segmentSuffix,
OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME, OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
OrdsBlockTreeTermsWriter.VERSION_START, indexIn = state.directory.openInput(indexFile, state.context);
OrdsBlockTreeTermsWriter.VERSION_CURRENT); int indexVersion = CodecUtil.checkSegmentHeader(indexIn, OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
OrdsBlockTreeTermsWriter.VERSION_START,
OrdsBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
if (indexVersion != version) { if (indexVersion != version) {
throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn); throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
} }
@ -116,8 +104,8 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
CodecUtil.retrieveChecksum(in); CodecUtil.retrieveChecksum(in);
// Read per-field details // Read per-field details
seekDir(in, dirOffset); seekDir(in);
seekDir(indexIn, indexDirOffset); seekDir(indexIn);
final int numFields = in.readVInt(); final int numFields = in.readVInt();
if (numFields < 0) { if (numFields < 0) {
@ -134,7 +122,7 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
in.readBytes(code.bytes, 0, numBytes); in.readBytes(code.bytes, 0, numBytes);
code.length = numBytes; code.length = numBytes;
final Output rootCode = OrdsBlockTreeTermsWriter.FST_OUTPUTS.newOutput(code, 0, numTerms); final Output rootCode = OrdsBlockTreeTermsWriter.FST_OUTPUTS.newOutput(code, 0, numTerms);
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field; assert fieldInfo != null: "field=" + field;
assert numTerms <= Integer.MAX_VALUE; assert numTerms <= Integer.MAX_VALUE;
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
@ -145,8 +133,8 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
BytesRef minTerm = readBytesRef(in); BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in); BytesRef maxTerm = readBytesRef(in);
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in); throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
} }
if (sumDocFreq < docCount) { // #postings must be >= #docs with field if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in); throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@ -182,10 +170,9 @@ public final class OrdsBlockTreeTermsReader extends FieldsProducer {
} }
/** Seek {@code input} to the directory offset. */ /** Seek {@code input} to the directory offset. */
private void seekDir(IndexInput input, long dirOffset) private void seekDir(IndexInput input) throws IOException {
throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8); input.seek(input.length() - CodecUtil.footerLength() - 8);
dirOffset = input.readLong(); long dirOffset = input.readLong();
input.seek(dirOffset); input.seek(dirOffset);
} }

11
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
View File

@ -42,7 +42,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder; import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.Builder;
@ -115,17 +114,17 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
/** Extension of terms file */ /** Extension of terms file */
static final String TERMS_EXTENSION = "tio"; static final String TERMS_EXTENSION = "tio";
final static String TERMS_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_DICT"; final static String TERMS_CODEC_NAME = "OrdsBlockTreeTerms";
/** Initial terms format. */ /** Initial terms format. */
public static final int VERSION_START = 0; public static final int VERSION_START = 1;
/** Current terms format. */ /** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START; public static final int VERSION_CURRENT = VERSION_START;
/** Extension of terms index file */ /** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tipo"; static final String TERMS_INDEX_EXTENSION = "tipo";
final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_INDEX"; final static String TERMS_INDEX_CODEC_NAME = "OrdsBlockTreeIndex";
private final IndexOutput out; private final IndexOutput out;
private final IndexOutput indexOut; private final IndexOutput indexOut;
@ -204,11 +203,11 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
fieldInfos = state.fieldInfos; fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock; this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock; this.maxItemsInBlock = maxItemsInBlock;
CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT); CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION); final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context); indexOut = state.directory.createOutput(termsIndexFileName, state.context);
CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT); CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter; this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name; // segment = state.segmentInfo.name;

2
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
View File

@ -1030,7 +1030,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum {
@Override @Override
public String toString() { public String toString() {
return "OrdsSegmentTermsEnum(seg=" + fr.parent.segment + ")"; return "OrdsSegmentTermsEnum(seg=" + fr.parent + ")";
} }
/** Holds a single input (IntsRef) + output pair. */ /** Holds a single input (IntsRef) + output pair. */

15
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
View File

@ -85,9 +85,8 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
public final class BloomFilteringPostingsFormat extends PostingsFormat { public final class BloomFilteringPostingsFormat extends PostingsFormat {
public static final String BLOOM_CODEC_NAME = "BloomFilter"; public static final String BLOOM_CODEC_NAME = "BloomFilter";
public static final int VERSION_START = 1; public static final int VERSION_START = 3;
public static final int VERSION_CHECKSUM = 2; public static final int VERSION_CURRENT = VERSION_START;
public static final int VERSION_CURRENT = VERSION_CHECKSUM;
/** Extension of Bloom Filters file */ /** Extension of Bloom Filters file */
static final String BLOOM_EXTENSION = "blm"; static final String BLOOM_EXTENSION = "blm";
@ -167,7 +166,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
boolean success = false; boolean success = false;
try { try {
bloomIn = state.directory.openChecksumInput(bloomFileName, state.context); bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT); CodecUtil.checkSegmentHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// // Load the hash function used in the BloomFilter // // Load the hash function used in the BloomFilter
// hashFunction = HashFunction.forName(bloomIn.readString()); // hashFunction = HashFunction.forName(bloomIn.readString());
// Load the delegate postings format // Load the delegate postings format
@ -183,11 +182,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum); FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum);
bloomsByFieldName.put(fieldInfo.name, bloom); bloomsByFieldName.put(fieldInfo.name, bloom);
} }
if (version >= VERSION_CHECKSUM) { CodecUtil.checkFooter(bloomIn);
CodecUtil.checkFooter(bloomIn);
} else {
CodecUtil.checkEOF(bloomIn);
}
IOUtils.close(bloomIn); IOUtils.close(bloomIn);
success = true; success = true;
} finally { } finally {
@ -507,7 +502,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
IndexOutput bloomOutput = null; IndexOutput bloomOutput = null;
try { try {
bloomOutput = state.directory.createOutput(bloomFileName, state.context); bloomOutput = state.directory.createOutput(bloomFileName, state.context);
CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT); CodecUtil.writeSegmentHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// remember the name of the postings format we will delegate to // remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName()); bloomOutput.writeString(delegatePostingsFormat.getName());

4
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
View File

@ -52,10 +52,10 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
try { try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context); data = state.directory.createOutput(dataName, state.context);
CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT); CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context); meta = state.directory.createOutput(metaName, state.context);
CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT); CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true; success = true;
} finally { } finally {
if (!success) { if (!success) {

12
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
View File

@ -86,7 +86,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
static final byte SORTED_NUMERIC = 5; static final byte SORTED_NUMERIC = 5;
static final byte SORTED_NUMERIC_SINGLETON = 6; static final byte SORTED_NUMERIC_SINGLETON = 6;
static final int VERSION_START = 2; static final int VERSION_START = 3;
static final int VERSION_CURRENT = VERSION_START; static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s // clone for merge: when merging we don't do any instances.put()s
@ -122,9 +122,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
boolean success = false; boolean success = false;
try { try {
version = CodecUtil.checkHeader(in, metaCodec, version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
VERSION_CURRENT);
numEntries = readFields(in, state.fieldInfos); numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in); CodecUtil.checkFooter(in);
@ -141,9 +140,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
this.data = state.directory.openInput(dataName, state.context); this.data = state.directory.openInput(dataName, state.context);
success = false; success = false;
try { try {
final int version2 = CodecUtil.checkHeader(data, dataCodec, final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
VERSION_CURRENT);
if (version != version2) { if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data); throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
} }

7
lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
View File

@ -383,12 +383,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
boolean success = false; boolean success = false;
try { try {
fields = new OrdsBlockTreeTermsReader(state.directory, fields = new OrdsBlockTreeTermsReader(postingsReader, state);
state.fieldInfos,
state.segmentInfo,
postingsReader,
state.context,
state.segmentSuffix);
success = true; success = true;
} finally { } finally {
if (!success) { if (!success) {