Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add doPackFST option (default to false) to MemoryCodec; randomize it during tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239205 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-02-01 17:01:55 +00:00
parent c48c0d7462
commit f31cb419e7
3 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
View File

@ -82,8 +82,20 @@ import org.apache.lucene.util.fst.Util;
// loads itself in ram? // loads itself in ram?
public class MemoryPostingsFormat extends PostingsFormat { public class MemoryPostingsFormat extends PostingsFormat {
private final boolean doPackFST;
public MemoryPostingsFormat() { public MemoryPostingsFormat() {
this(false);
}
public MemoryPostingsFormat(boolean doPackFST) {
super("Memory"); super("Memory");
this.doPackFST = doPackFST;
}
@Override
public String toString() {
return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
} }
private static final boolean VERBOSE = false; private static final boolean VERBOSE = false;
@ -93,12 +105,14 @@ public class MemoryPostingsFormat extends PostingsFormat {
private final FieldInfo field; private final FieldInfo field;
private final Builder<BytesRef> builder; private final Builder<BytesRef> builder;
private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
private final boolean doPackFST;
private int termCount; private int termCount;
public TermsWriter(IndexOutput out, FieldInfo field) { public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) {
this.out = out; this.out = out;
this.field = field; this.field = field;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, outputs); this.doPackFST = doPackFST;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST);
} }
private class PostingsWriter extends PostingsConsumer { private class PostingsWriter extends PostingsConsumer {
@ -230,7 +244,11 @@ public class MemoryPostingsFormat extends PostingsFormat {
} }
out.writeVLong(sumDocFreq); out.writeVLong(sumDocFreq);
out.writeVInt(docCount); out.writeVInt(docCount);
builder.finish().save(out); FST<BytesRef> fst = builder.finish();
if (doPackFST) {
fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
}
fst.save(out);
if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
} }
} }
@ -256,7 +274,7 @@ public class MemoryPostingsFormat extends PostingsFormat {
throw new UnsupportedOperationException("this codec cannot index offsets"); throw new UnsupportedOperationException("this codec cannot index offsets");
} }
if (VERBOSE) System.out.println("\naddField field=" + field.name); if (VERBOSE) System.out.println("\naddField field=" + field.name);
return new TermsWriter(out, field); return new TermsWriter(out, field, doPackFST);
} }
@Override @Override
@ -776,6 +794,9 @@ public class MemoryPostingsFormat extends PostingsFormat {
break; break;
} }
final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
if (VERBOSE) {
System.out.println("load field=" + termsReader.field.name);
}
fields.put(termsReader.field.name, termsReader); fields.put(termsReader.field.name, termsReader);
} }
} finally { } finally {

2
lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
View File

@ -94,7 +94,7 @@ public class RandomCodec extends Lucene40Codec {
formats.add(new Lucene40WithOrds()); formats.add(new Lucene40WithOrds());
if (!useNoMemoryExpensiveCodec) { if (!useNoMemoryExpensiveCodec) {
formats.add(new SimpleTextPostingsFormat()); formats.add(new SimpleTextPostingsFormat());
formats.add(new MemoryPostingsFormat()); formats.add(new MemoryPostingsFormat(random.nextBoolean()));
} }
Collections.shuffle(formats, random); Collections.shuffle(formats, random);
} }

2
lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
View File

@ -38,7 +38,7 @@ public class TestRollingUpdates extends LuceneTestCase {
//provider.register(new MemoryCodec()); //provider.register(new MemoryCodec());
if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) {
Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random.nextBoolean())));
} }
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));