Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4120: FST.pack: Use packed integer arrays for improved memory efficiency. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1349826 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2012-06-13 13:12:44 +00:00
parent 9182f9f907
commit 4104901e64
14 changed files with 217 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
View File

@ -37,6 +37,7 @@ import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.packed.PackedInts;
import com.ibm.icu.text.Normalizer2;
@ -161,7 +162,7 @@ public class TokenInfoDictionaryBuilder {
offset = next;
}
final FST<Long> fst = fstBuilder.finish().pack(2, 100000);
final FST<Long> fst = fstBuilder.finish().pack(2, 100000, PackedInts.DEFAULT);
System.out.print(" " + fst.getNodeCount() + " nodes, " + fst.getArcCount() + " arcs, " + fst.sizeInBytes() + " bytes... ");
dictionary.setFST(fst);

17
lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
View File

@ -54,6 +54,7 @@ import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.packed.PackedInts;
// TODO: would be nice to somehow allow this to act like
// InstantiatedIndex, by never writing to disk; ie you write
@ -81,14 +82,16 @@ import org.apache.lucene.util.fst.Util;
public class MemoryPostingsFormat extends PostingsFormat {
private final boolean doPackFST;
private final float acceptableOverheadRatio;
public MemoryPostingsFormat() {
this(false);
this(false, PackedInts.DEFAULT);
}
public MemoryPostingsFormat(boolean doPackFST) {
public MemoryPostingsFormat(boolean doPackFST, float acceptableOverheadRatio) {
super("Memory");
this.doPackFST = doPackFST;
this.acceptableOverheadRatio = acceptableOverheadRatio;
}
@Override
@ -102,13 +105,15 @@ public class MemoryPostingsFormat extends PostingsFormat {
private final Builder<BytesRef> builder;
private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
private final boolean doPackFST;
private final float acceptableOverheadRatio;
private int termCount;
public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) {
public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST, float acceptableOverheadRatio) {
this.out = out;
this.field = field;
this.doPackFST = doPackFST;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST);
this.acceptableOverheadRatio = acceptableOverheadRatio;
builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio);
}
private class PostingsWriter extends PostingsConsumer {
@ -265,7 +270,7 @@ public class MemoryPostingsFormat extends PostingsFormat {
out.writeVInt(docCount);
FST<BytesRef> fst = builder.finish();
if (doPackFST) {
fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4), acceptableOverheadRatio);
}
fst.save(out);
//System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
@ -290,7 +295,7 @@ public class MemoryPostingsFormat extends PostingsFormat {
@Override
public TermsConsumer addField(FieldInfo field) {
//System.out.println("\naddField field=" + field.name);
return new TermsWriter(out, field, doPackFST);
return new TermsWriter(out, field, doPackFST, acceptableOverheadRatio);
}
@Override

6
lucene/core/src/java/org/apache/lucene/index/DocValues.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.document.PackedLongDocValuesField; // javadocs
import org.apache.lucene.document.ShortDocValuesField; // javadocs
import org.apache.lucene.document.SortedBytesDocValuesField; // javadocs
import org.apache.lucene.document.StraightBytesDocValuesField; // javadocs
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.packed.PackedInts;
@ -411,6 +412,11 @@ public abstract class DocValues implements Closeable {
Arrays.fill(arr, off, off+len, 0);
return len;
}
@Override
public long ramBytesUsed() {
return 0;
}
};
return new SortedSource(type, BytesRef.getUTF8SortedAsUnicodeComparator()) {

23
lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.FST.INPUT_TYPE; // javadoc
import org.apache.lucene.util.packed.PackedInts;
/**
* Builds a minimal FST (maps an IntsRef term to an arbitrary
@ -83,7 +84,18 @@ public class Builder<T> {
* pruning options turned off.
*/
public Builder(FST.INPUT_TYPE inputType, Outputs<T> outputs) {
this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false);
this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false, PackedInts.COMPACT);
}
/**
* Instantiates an FST/FSA builder with {@link PackedInts#DEFAULT}
* <code>acceptableOverheadRatio</code>.
*/
public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix,
boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs<T> outputs,
FreezeTail<T> freezeTail, boolean willPackFST) {
this(inputType, minSuffixCount1, minSuffixCount2, doShareSuffix, doShareNonSingletonNodes,
shareMaxTailLength, outputs, freezeTail, willPackFST, PackedInts.DEFAULT);
}
/**
@ -126,17 +138,20 @@ public class Builder<T> {
* @param willPackFST Pass true if you will pack the FST before saving. This
* causes the FST to create additional data structures internally to facilitate packing, but
* it means the resulting FST cannot be saved: it must
* first be packed using {@link FST#pack(int, int)}}.
* first be packed using {@link FST#pack(int, int, float)}
*
* @param acceptableOverheadRatio How to trade speed for space when building the FST. This option
* is only relevant when willPackFST is true. @see PackedInts#getMutable(int, int, float)
*/
public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix,
boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs<T> outputs,
FreezeTail<T> freezeTail, boolean willPackFST) {
FreezeTail<T> freezeTail, boolean willPackFST, float acceptableOverheadRatio) {
this.minSuffixCount1 = minSuffixCount1;
this.minSuffixCount2 = minSuffixCount2;
this.freezeTail = freezeTail;
this.doShareNonSingletonNodes = doShareNonSingletonNodes;
this.shareMaxTailLength = shareMaxTailLength;
fst = new FST<T>(inputType, outputs, willPackFST);
fst = new FST<T>(inputType, outputs, willPackFST, acceptableOverheadRatio);
if (doShareSuffix) {
dedupHash = new NodeHash<T>(fst);
} else {

116
lucene/core/src/java/org/apache/lucene/util/fst/FST.java
View File

@ -37,8 +37,9 @@ import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.Builder.UnCompiledNode;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
// TODO: break this into WritableFST and ReadOnlyFST.. then
// we can have subclasses of ReadOnlyFST to handle the
@ -155,7 +156,7 @@ public final class FST<T> {
public int arcWithOutputCount;
private final boolean packed;
private final int[] nodeRefToAddress;
private PackedInts.Reader nodeRefToAddress;
// If arc has this label then that arc is final/accepted
public static final int END_LABEL = -1;
@ -252,25 +253,23 @@ public final class FST<T> {
private final BytesWriter writer;
// TODO: we can save RAM here by using growable packed
// ints...:
private int[] nodeAddress;
private GrowableWriter nodeAddress;
// TODO: we could be smarter here, and prune periodically
// as we go; high in-count nodes will "usually" become
// clear early on:
private int[] inCounts;
private GrowableWriter inCounts;
// make a new empty FST, for building; Builder invokes
// this ctor
FST(INPUT_TYPE inputType, Outputs<T> outputs, boolean willPackFST) {
FST(INPUT_TYPE inputType, Outputs<T> outputs, boolean willPackFST, float acceptableOverheadRatio) {
this.inputType = inputType;
this.outputs = outputs;
bytes = new byte[128];
NO_OUTPUT = outputs.getNoOutput();
if (willPackFST) {
nodeAddress = new int[8];
inCounts = new int[8];
nodeAddress = new GrowableWriter(PackedInts.bitsRequired(bytes.length - 1), 8, acceptableOverheadRatio);
inCounts = new GrowableWriter(1, 8, acceptableOverheadRatio);
} else {
nodeAddress = null;
inCounts = null;
@ -320,11 +319,7 @@ public final class FST<T> {
throw new IllegalStateException("invalid input type " + t);
}
if (packed) {
final int nodeRefCount = in.readVInt();
nodeRefToAddress = new int[nodeRefCount];
for(int idx=0;idx<nodeRefCount;idx++) {
nodeRefToAddress[idx] = in.readVInt();
}
nodeRefToAddress = PackedInts.getReader(in);
} else {
nodeRefToAddress = null;
}
@ -348,10 +343,10 @@ public final class FST<T> {
public int sizeInBytes() {
int size = bytes.length;
if (packed) {
size += nodeRefToAddress.length * RamUsageEstimator.NUM_BYTES_INT;
size += nodeRefToAddress.ramBytesUsed();
} else if (nodeAddress != null) {
size += nodeAddress.length * RamUsageEstimator.NUM_BYTES_INT;
size += inCounts.length * RamUsageEstimator.NUM_BYTES_INT;
size += nodeAddress.ramBytesUsed();
size += inCounts.ramBytesUsed();
}
return size;
}
@ -374,7 +369,7 @@ public final class FST<T> {
private int getNodeAddress(int node) {
if (nodeAddress != null) {
// Deref
return nodeAddress[node];
return (int) nodeAddress.get(node);
} else {
// Straight
return node;
@ -444,6 +439,9 @@ public final class FST<T> {
if (nodeAddress != null) {
throw new IllegalStateException("cannot save an FST pre-packed FST; it must first be packed");
}
if (packed && !(nodeRefToAddress instanceof PackedInts.Mutable)) {
throw new IllegalStateException("cannot save a FST which has been loaded from disk ");
}
CodecUtil.writeHeader(out, FILE_FORMAT_NAME, VERSION_CURRENT);
if (packed) {
out.writeByte((byte) 1);
@ -469,11 +467,7 @@ public final class FST<T> {
}
out.writeByte(t);
if (packed) {
assert nodeRefToAddress != null;
out.writeVInt(nodeRefToAddress.length);
for(int idx=0;idx<nodeRefToAddress.length;idx++) {
out.writeVInt(nodeRefToAddress[idx]);
}
((PackedInts.Mutable) nodeRefToAddress).save(out);
}
out.writeVInt(startNode);
out.writeVInt(nodeCount);
@ -624,7 +618,7 @@ public final class FST<T> {
if (!targetHasArcs) {
flags += BIT_STOP_NODE;
} else if (inCounts != null) {
inCounts[target.node]++;
inCounts.set(target.node, inCounts.get(target.node) + 1);
}
if (arc.output != NO_OUTPUT) {
@ -715,11 +709,11 @@ public final class FST<T> {
final int node;
if (nodeAddress != null) {
// Nodes are addressed by 1+ord:
if (nodeCount == nodeAddress.length) {
nodeAddress = ArrayUtil.grow(nodeAddress);
inCounts = ArrayUtil.grow(inCounts);
if (nodeCount == nodeAddress.size()) {
nodeAddress = nodeAddress.resize(ArrayUtil.oversize(nodeAddress.size() + 1, nodeAddress.getBitsPerValue()));
inCounts = inCounts.resize(ArrayUtil.oversize(inCounts.size() + 1, inCounts.getBitsPerValue()));
}
nodeAddress[nodeCount] = endAddress;
nodeAddress.set(nodeCount, endAddress);
// System.out.println(" write nodeAddress[" + nodeCount + "] = " + endAddress);
node = nodeCount;
} else {
@ -1005,9 +999,9 @@ public final class FST<T> {
// Address is delta-coded from current address:
arc.target = pos + code;
//System.out.println(" delta pos=" + pos + " delta=" + code + " target=" + arc.target);
} else if (code < nodeRefToAddress.length) {
} else if (code < nodeRefToAddress.size()) {
// Deref
arc.target = nodeRefToAddress[code];
arc.target = (int) nodeRefToAddress.get(code);
//System.out.println(" deref code=" + code + " target=" + arc.target);
} else {
// Absolute
@ -1420,7 +1414,7 @@ public final class FST<T> {
*/
// Creates a packed FST
private FST(INPUT_TYPE inputType, int[] nodeRefToAddress, Outputs<T> outputs) {
private FST(INPUT_TYPE inputType, PackedInts.Reader nodeRefToAddress, Outputs<T> outputs) {
packed = true;
this.inputType = inputType;
bytes = new byte[128];
@ -1432,8 +1426,10 @@ public final class FST<T> {
/** Expert: creates an FST by packing this one. This
* process requires substantial additional RAM (currently
* ~8 bytes per node), but then should produce a smaller FST. */
public FST<T> pack(int minInCountDeref, int maxDerefNodes) throws IOException {
* up to ~8 bytes per node depending on
* <code>acceptableOverheadRatio</code>), but then should
* produce a smaller FST. */
public FST<T> pack(int minInCountDeref, int maxDerefNodes, float acceptableOverheadRatio) throws IOException {
// TODO: other things to try
// - renumber the nodes to get more next / better locality?
@ -1454,22 +1450,22 @@ public final class FST<T> {
final BytesReader r = getBytesReader(0);
final int topN = Math.min(maxDerefNodes, inCounts.length);
final int topN = Math.min(maxDerefNodes, inCounts.size());
// Find top nodes with highest number of incoming arcs:
NodeQueue q = new NodeQueue(topN);
// TODO: we could use more RAM efficient selection algo here...
NodeAndInCount bottom = null;
for(int node=0;node<inCounts.length;node++) {
if (inCounts[node] >= minInCountDeref) {
for(int node=0; node<inCounts.size(); node++) {
if (inCounts.get(node) >= minInCountDeref) {
if (bottom == null) {
q.add(new NodeAndInCount(node, inCounts[node]));
q.add(new NodeAndInCount(node, (int) inCounts.get(node)));
if (q.size() == topN) {
bottom = q.top();
}
} else if (inCounts[node] > bottom.count) {
q.insertWithOverflow(new NodeAndInCount(node, inCounts[node]));
} else if (inCounts.get(node) > bottom.count) {
q.insertWithOverflow(new NodeAndInCount(node, (int) inCounts.get(node)));
}
}
}
@ -1484,20 +1480,17 @@ public final class FST<T> {
//System.out.println("map node=" + n.node + " inCount=" + n.count + " to newID=" + downTo);
}
// TODO: we can use packed ints:
// +1 because node ords start at 1 (0 is reserved as
// stop node):
final int[] nodeRefToAddressIn = new int[topNodeMap.size()];
final FST<T> fst = new FST<T>(inputType, nodeRefToAddressIn, outputs);
final FST<T> fst = new FST<T>(inputType, null, outputs);
final BytesWriter writer = fst.writer;
final int[] newNodeAddress = new int[1+nodeCount];
// +1 because node ords start at 1 (0 is reserved as stop node):
final GrowableWriter newNodeAddress = new GrowableWriter(
PackedInts.bitsRequired(bytes.length), 1 + nodeCount, acceptableOverheadRatio);
// Fill initial coarse guess:
for(int node=1;node<=nodeCount;node++) {
newNodeAddress[node] = 1 + bytes.length - nodeAddress[node];
newNodeAddress.set(node, 1 + bytes.length - nodeAddress.get(node));
}
int absCount;
@ -1537,11 +1530,11 @@ public final class FST<T> {
fst.nodeCount++;
final int address = writer.posWrite;
//System.out.println(" node: " + node + " address=" + address);
if (address != newNodeAddress[node]) {
addressError = address - newNodeAddress[node];
if (address != newNodeAddress.get(node)) {
addressError = address - (int) newNodeAddress.get(node);
//System.out.println(" change: " + (address - newNodeAddress[node]));
changed = true;
newNodeAddress[node] = address;
newNodeAddress.set(node, address);
changedCount++;
}
@ -1621,10 +1614,10 @@ public final class FST<T> {
if (ptr != null) {
absPtr = ptr;
} else {
absPtr = topNodeMap.size() + newNodeAddress[arc.target] + addressError;
absPtr = topNodeMap.size() + (int) newNodeAddress.get(arc.target) + addressError;
}
int delta = newNodeAddress[arc.target] + addressError - writer.posWrite - 2;
int delta = (int) newNodeAddress.get(arc.target) + addressError - writer.posWrite - 2;
if (delta < 0) {
//System.out.println("neg: " + delta);
anyNegDelta = true;
@ -1654,7 +1647,7 @@ public final class FST<T> {
if (doWriteTarget) {
int delta = newNodeAddress[arc.target] + addressError - writer.posWrite;
int delta = (int) newNodeAddress.get(arc.target) + addressError - writer.posWrite;
if (delta < 0) {
anyNegDelta = true;
//System.out.println("neg: " + delta);
@ -1745,11 +1738,20 @@ public final class FST<T> {
//System.out.println(" " + changedCount + " of " + fst.nodeCount + " changed; retry");
}
for(Map.Entry<Integer,Integer> ent : topNodeMap.entrySet()) {
nodeRefToAddressIn[ent.getValue()] = newNodeAddress[ent.getKey()];
long maxAddress = 0;
for (int key : topNodeMap.keySet()) {
maxAddress = Math.max(maxAddress, newNodeAddress.get(key));
}
fst.startNode = newNodeAddress[startNode];
PackedInts.Mutable nodeRefToAddressIn = PackedInts.getMutable(topNodeMap.size(),
PackedInts.bitsRequired(maxAddress), acceptableOverheadRatio);
for(Map.Entry<Integer,Integer> ent : topNodeMap.entrySet()) {
nodeRefToAddressIn.set(ent.getValue(), newNodeAddress.get(ent.getKey()));
}
fst.nodeRefToAddress = nodeRefToAddressIn;
fst.startNode = (int) newNodeAddress.get(startNode);
//System.out.println("new startNode=" + fst.startNode + " old startNode=" + startNode);
if (emptyOutput != null) {

5
lucene/core/src/java/org/apache/lucene/util/packed/DirectPacked64SingleBlockReader.java
View File

@ -51,4 +51,9 @@ final class DirectPacked64SingleBlockReader extends PackedInts.ReaderImpl {
throw new IllegalStateException("failed", e);
}
}
@Override
public long ramBytesUsed() {
return 0;
}
}

5
lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java
View File

@ -73,4 +73,9 @@ final class DirectPackedReader extends PackedInts.ReaderImpl {
throw new IllegalStateException("failed", ioe);
}
}
@Override
public long ramBytesUsed() {
return 0;
}
}

14
lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
View File

@ -17,6 +17,10 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
/**
* Implements {@link PackedInts.Mutable}, but grows the
* bit count of the underlying packed ints on-demand.
@ -111,4 +115,14 @@ public class GrowableWriter implements PackedInts.Mutable {
current.fill(fromIndex, toIndex, val);
}
@Override
public long ramBytesUsed() {
return current.ramBytesUsed();
}
@Override
public void save(DataOutput out) throws IOException {
current.save(out);
}
}

5
lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
View File

@ -284,6 +284,11 @@ abstract class Packed64SingleBlock extends PackedInts.MutableImpl {
return RamUsageEstimator.sizeOf(blocks);
}
@Override
protected int getFormat() {
return PackedInts.PACKED_SINGLE_BLOCK;
}
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue

55
lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
View File

@ -100,6 +100,11 @@ public class PackedInts {
*/
int size();
/**
* Return the in-memory size in bytes.
*/
long ramBytesUsed();
/**
* Expert: if the bit-width of this reader matches one of
* java's native types, returns the underlying array
@ -118,6 +123,7 @@ public class PackedInts {
* @see #getArray
*/
boolean hasArray();
}
/**
@ -171,6 +177,7 @@ public class PackedInts {
* @lucene.internal
*/
public static interface Mutable extends Reader {
/**
* Set the value at the given index in the array.
* @param index where the value should be positioned.
@ -197,6 +204,13 @@ public class PackedInts {
*/
void clear();
/**
* Save this mutable into <code>out</code>. Instantiating a reader from
* the generated data will return a reader with the same number of bits
* per value.
*/
void save(DataOutput out) throws IOException;
}
/**
@ -239,6 +253,7 @@ public class PackedInts {
}
return gets;
}
}
public static abstract class MutableImpl extends ReaderImpl implements Mutable {
@ -267,6 +282,18 @@ public class PackedInts {
}
}
protected int getFormat() {
return PACKED;
}
@Override
public void save(DataOutput out) throws IOException {
Writer writer = getWriterByFormat(out, valueCount, bitsPerValue, getFormat());
for (int i = 0; i < valueCount; ++i) {
writer.add(get(i));
}
writer.finish();
}
}
/** A write-once Writer.
@ -470,28 +497,40 @@ public class PackedInts {
int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue;
if (bitsPerValue <= 8 && maxBitsPerValue >= 8) {
return new PackedWriter(out, valueCount, 8);
return getWriterByFormat(out, valueCount, 8, PACKED);
} else if (bitsPerValue <= 16 && maxBitsPerValue >= 16) {
return new PackedWriter(out, valueCount, 16);
return getWriterByFormat(out, valueCount, 16, PACKED);
} else if (bitsPerValue <= 32 && maxBitsPerValue >= 32) {
return new PackedWriter(out, valueCount, 32);
return getWriterByFormat(out, valueCount, 32, PACKED);
} else if (bitsPerValue <= 64 && maxBitsPerValue >= 64) {
return new PackedWriter(out, valueCount, 64);
return getWriterByFormat(out, valueCount, 64, PACKED);
} else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) {
return new PackedWriter(out, valueCount, 24);
return getWriterByFormat(out, valueCount, 24, PACKED);
} else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) {
return new PackedWriter(out, valueCount, bitsPerValue);
return getWriterByFormat(out, valueCount, 48, PACKED);
} else {
for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) {
if (Packed64SingleBlock.isSupported(bpv)) {
float overhead = Packed64SingleBlock.overheadPerValue(bpv);
float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv;
if (overhead <= acceptableOverhead) {
return new Packed64SingleBlockWriter(out, valueCount, bpv);
return getWriterByFormat(out, valueCount, bpv, PACKED_SINGLE_BLOCK);
}
}
}
return new PackedWriter(out, valueCount, bitsPerValue);
return getWriterByFormat(out, valueCount, bitsPerValue, PACKED);
}
}
private static Writer getWriterByFormat(DataOutput out,
int valueCount, int bitsPerValue, int format) throws IOException {
switch (format) {
case PACKED:
return new PackedWriter(out, valueCount, bitsPerValue);
case PACKED_SINGLE_BLOCK:
return new Packed64SingleBlockWriter(out, valueCount, bitsPerValue);
default:
throw new IllegalArgumentException("Unknown format " + format);
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
View File

@ -41,7 +41,7 @@ public class TestRollingUpdates extends LuceneTestCase {
//provider.register(new MemoryCodec());
if (random().nextBoolean()) {
Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random().nextBoolean())));
Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random().nextBoolean(), random.nextFloat())));
}
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));

9
lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
View File

@ -64,6 +64,7 @@ import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
import org.apache.lucene.util.fst.PairOutputs.Pair;
import org.apache.lucene.util.packed.PackedInts;
@SuppressCodecs({ "SimpleText", "Memory" })
public class TestFSTs extends LuceneTestCase {
@ -536,7 +537,7 @@ public class TestFSTs extends LuceneTestCase {
if (VERBOSE) {
System.out.println("TEST: now rewrite");
}
final FST<T> packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
final FST<T> packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000), random.nextFloat());
if (VERBOSE) {
System.out.println("TEST: now verify packed FST");
}
@ -1182,7 +1183,7 @@ public class TestFSTs extends LuceneTestCase {
if (rewriteIter == 1) {
if (doRewrite) {
// Verify again, with packed FST:
fst = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
fst = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000), random.nextFloat());
} else {
break;
}
@ -1324,7 +1325,7 @@ public class TestFSTs extends LuceneTestCase {
if (doPack) {
System.out.println("Pack...");
fst = fst.pack(4, 100000000);
fst = fst.pack(4, 100000000, random().nextFloat());
System.out.println("New size " + fst.sizeInBytes() + " bytes");
}
@ -1927,7 +1928,7 @@ public class TestFSTs extends LuceneTestCase {
final Long nothing = outputs.getNoOutput();
final Builder<Long> b = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
final FST<Long> fst = new FST<Long>(FST.INPUT_TYPE.BYTE1, outputs, false);
final FST<Long> fst = new FST<Long>(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT);
final Builder.UnCompiledNode<Long> rootNode = new Builder.UnCompiledNode<Long>(b, 0);

36
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -560,4 +560,40 @@ public class TestPackedInts extends LuceneTestCase {
assertEquals(1 << 10, wrt.get(valueCount - 1));
}
public void testSave() throws IOException {
final int valueCount = _TestUtil.nextInt(random(), 1, 2048);
for (int bpv = 1; bpv <= 64; ++bpv) {
final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
final RAMDirectory directory = new RAMDirectory();
List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
for (PackedInts.Mutable mutable : packedInts) {
for (int i = 0; i < mutable.size(); ++i) {
mutable.set(i, random().nextInt(maxValue));
}
IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
mutable.save(out);
out.close();
IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
PackedInts.Reader reader = PackedInts.getReader(in);
assertEquals(mutable.getBitsPerValue(), reader.getBitsPerValue());
assertEquals(valueCount, reader.size());
if (mutable instanceof Packed64SingleBlock) {
// make sure that we used the right format so that the reader has
// the same performance characteristics as the mutable that has been
// serialized
assertTrue(reader instanceof Packed64SingleBlock);
} else {
assertFalse(reader instanceof Packed64SingleBlock);
}
for (int i = 0; i < valueCount; ++i) {
assertEquals(mutable.get(i), reader.get(i));
}
in.close();
directory.deleteFile("packed-ints.bin");
}
}
}
}

4
lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
View File

@ -99,8 +99,8 @@ public class RandomCodec extends Lucene40Codec {
new NestedPulsingPostingsFormat(),
new Lucene40WithOrds(),
new SimpleTextPostingsFormat(),
new MemoryPostingsFormat(true),
new MemoryPostingsFormat(false));
new MemoryPostingsFormat(true, random.nextFloat()),
new MemoryPostingsFormat(false, random.nextFloat()));
Collections.shuffle(formats, random);
}