add doPackFST option (default to false) to MemoryCodec; randomize it during tests

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239205 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-02-01 17:01:55 +00:00
parent c48c0d7462
commit f31cb419e7
3 changed files with 28 additions and 7 deletions

View File

@ -82,8 +82,20 @@ import org.apache.lucene.util.fst.Util;
// loads itself in ram?
public class MemoryPostingsFormat extends PostingsFormat {
private final boolean doPackFST;
public MemoryPostingsFormat() {
this(false);
}
public MemoryPostingsFormat(boolean doPackFST) {
super("Memory");
this.doPackFST = doPackFST;
}
@Override
public String toString() {
return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
}
private static final boolean VERBOSE = false;
@ -93,12 +105,14 @@ public class MemoryPostingsFormat extends PostingsFormat {
private final FieldInfo field;
private final Builder<BytesRef> builder;
private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
private final boolean doPackFST;
private int termCount;
public TermsWriter(IndexOutput out, FieldInfo field) {
public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) {
this.out = out;
this.field = field;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, outputs);
this.doPackFST = doPackFST;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST);
}
private class PostingsWriter extends PostingsConsumer {
@ -230,7 +244,11 @@ public class MemoryPostingsFormat extends PostingsFormat {
}
out.writeVLong(sumDocFreq);
out.writeVInt(docCount);
builder.finish().save(out);
FST<BytesRef> fst = builder.finish();
if (doPackFST) {
fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
}
fst.save(out);
if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
}
}
@ -256,7 +274,7 @@ public class MemoryPostingsFormat extends PostingsFormat {
throw new UnsupportedOperationException("this codec cannot index offsets");
}
if (VERBOSE) System.out.println("\naddField field=" + field.name);
return new TermsWriter(out, field);
return new TermsWriter(out, field, doPackFST);
}
@Override
@ -776,6 +794,9 @@ public class MemoryPostingsFormat extends PostingsFormat {
break;
}
final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
if (VERBOSE) {
System.out.println("load field=" + termsReader.field.name);
}
fields.put(termsReader.field.name, termsReader);
}
} finally {

View File

@ -94,7 +94,7 @@ public class RandomCodec extends Lucene40Codec {
formats.add(new Lucene40WithOrds());
if (!useNoMemoryExpensiveCodec) {
formats.add(new SimpleTextPostingsFormat());
formats.add(new MemoryPostingsFormat());
formats.add(new MemoryPostingsFormat(random.nextBoolean()));
}
Collections.shuffle(formats, random);
}

View File

@ -38,7 +38,7 @@ public class TestRollingUpdates extends LuceneTestCase {
//provider.register(new MemoryCodec());
if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) {
Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat()));
Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random.nextBoolean())));
}
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));