Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3892: fix maxChain bug; support numBits=0 (all ints == 0) 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1360993 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-07-12 23:01:19 +00:00
parent 101ea65be9
commit 2c0f1e2a4f
6 changed files with 233 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lucene/core/src/java/org/apache/lucene/codecs/pfor/ForPostingsFormat.java
View File

@ -17,28 +17,27 @@ package org.apache.lucene.codecs.pfor;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Set;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTreeTermsReader;
import org.apache.lucene.codecs.BlockTreeTermsWriter;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.BlockTreeTermsWriter;
import org.apache.lucene.codecs.BlockTreeTermsReader;
import org.apache.lucene.codecs.TermsIndexReaderBase;
import org.apache.lucene.codecs.TermsIndexWriterBase;
import org.apache.lucene.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.TermsIndexReaderBase;
import org.apache.lucene.codecs.TermsIndexWriterBase;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.sep.SepPostingsReader;
import org.apache.lucene.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/**
* Pass ForFactory to a PostingsWriter/ReaderBase, and get

13
lucene/core/src/java/org/apache/lucene/codecs/pfor/ForUtil.java
View File

@ -63,7 +63,7 @@ public class ForUtil {
int header = intBuffer.get();
int numInts = (header & MASK[8]) + 1;
int numBits = ((header >> 8) & MASK[5]) + 1;
int numBits = ((header >> 8) & MASK[6]);
decompressCore(intBuffer, data, numBits);
@ -77,10 +77,11 @@ public class ForUtil {
*/
static void decompressCore(IntBuffer intBuffer, int[] data, int numBits) {
assert numBits<=32;
assert numBits>=1;
assert numBits>=0;
// TODO: PackedIntsDecompress is hardewired to size==129 only
// TODO: PackedIntsDecompress is hardewired to size==128 only
switch(numBits) {
case 0: PackedIntsDecompress.decode0(intBuffer, data); break;
case 1: PackedIntsDecompress.decode1(intBuffer, data); break;
case 2: PackedIntsDecompress.decode2(intBuffer, data); break;
case 3: PackedIntsDecompress.decode3(intBuffer, data); break;
@ -145,7 +146,7 @@ public class ForUtil {
* Estimate best num of frame bits according to the largest value.
*/
static int getNumBits(final int[] data, int size) {
int optBits=1;
int optBits=0;
for (int i=0; i<size; ++i) {
while ((data[i] & ~MASK[optBits]) != 0) {
optBits++;
@ -158,12 +159,12 @@ public class ForUtil {
* Generate the 4 byte header, which contains (from lsb to msb):
*
* - 8 bits for uncompressed int num - 1 (use up to 7 bits i.e 128 actually)
* - 5 bits for num of frame bits - 1
* - 6 bits for num of frame bits
* - other bits unused
*
*/
static int getHeader(int numInts, int numBits) {
return (numInts-1)
| ((numBits-1) << 8);
| ((numBits) << 8);
}
}

79
lucene/core/src/java/org/apache/lucene/codecs/pfor/PForUtil.java
View File

@ -53,7 +53,9 @@ public final class PForUtil extends ForUtil {
// the max value possible for current exception pointer,
// value of the first pointer is limited by header as 254
long maxChain = (1<<8) - 2;
// (first exception ranges from -1 ~ 254)
long maxChainFirst = 254;
long maxChain = maxChainFirst + 1;
boolean conValue, conForce, conEnd;
int i=0;
@ -77,41 +79,35 @@ public final class PForUtil extends ForUtil {
}
/** encode normal area, record exception positions */
i=0;
excNum = 0;
if (excFirstPos < 0) { // no exception
for (; i<size; ++i) {
for (i=0; i<size; ++i) {
encodeNormalValue(intBuffer,i,data[i], numBits);
}
excLastPos = -1;
} else {
for (; i<excFirstPos; ++i) {
for (i=0; i<excFirstPos; ++i) {
encodeNormalValue(intBuffer,i,data[i], numBits);
}
maxChain = 1L<<numBits;
excLastPos = -1;
for (; i<size; ++i) {
excLastPos = excFirstPos;
excNum = i<size? 1:0;
for (i=excFirstPos+1; i<size; ++i) {
conValue = ((data[i] & MASK[numBits]) != data[i]); // value exception
conForce = (i >= maxChain + excLastPos); // force exception
conEnd = (excNum == excNumBase); // following forced ignored
if ((!conValue && !conForce) || conEnd) {
encodeNormalValue(intBuffer,i,data[i], numBits);
} else {
if (excLastPos >= 0) {
encodeNormalValue(intBuffer, excLastPos, i-excLastPos-1, numBits);
}
excNum++;
excLastPos = i;
}
}
if (excLastPos >= 0) {
encodeNormalValue(intBuffer, excLastPos, (i-excLastPos-1)&MASK[numBits], numBits); // mask out suppressed force exception
}
}
/** encode exception area */
i=0;
for (; i<excNum; ++i) {
for (i=0; i<excNum; ++i) {
if (excBytes < 2 && (excValues[i] & ~MASK[8]) != 0) {
excBytes=2;
}
@ -139,11 +135,11 @@ public final class PForUtil extends ForUtil {
intBuffer.rewind();
int header = intBuffer.get();
int numInts = (header & MASK[8]) + 1;
int numInts = (header & MASK[8]);
int excNum = ((header >> 8) & MASK[8]) + 1;
int excFirstPos = ((header >> 16) & MASK[8]) - 1;
int excBytes = PER_EXCEPTION_SIZE[(header >> 29) & MASK[2]];
int numBits = ((header >> 24) & MASK[5]) + 1;
int excBytes = PER_EXCEPTION_SIZE[(header >> 30) & MASK[2]];
int numBits = ((header >> 24) & MASK[6]);
decompressCore(intBuffer, data, numBits);
@ -157,6 +153,11 @@ public final class PForUtil extends ForUtil {
intBuffer.put(0, header);
}
/**
* Encode exception values into exception area.
* The width for each exception will be fixed as:
* 1, 2, or 4 byte(s).
*/
static void encodeExcValues(IntBuffer intBuffer, int[] values, int num, int perbytes, int byteOffset) {
if (num == 0)
return;
@ -251,8 +252,10 @@ public final class PForUtil extends ForUtil {
* It will run 32 times.
*/
static int getNumBits(final int[] data, int size) {
if (isAllZero(data))
return 0;
int optBits=1;
int optSize=estimateCompressedSize(data,size,1);
int optSize=estimateCompressedSize(data,size,optBits);
for (int i=2; i<=32; ++i) {
int curSize=estimateCompressedSize(data,size,i);
if (curSize<optSize) {
@ -263,6 +266,16 @@ public final class PForUtil extends ForUtil {
return optBits;
}
static boolean isAllZero(final int[] data) {
int len=data.length;
for (int i=0; i<len; i++) {
if (data[i] != 0) {
return false;
}
}
return true;
}
/**
* Iterate the whole block to get maximum exception bits,
* and estimate compressed size without forced exception.
@ -303,16 +316,40 @@ public final class PForUtil extends ForUtil {
*
* 8 bits for the index of the first exception + 1 (when no exception, this is 0)
*
* 5 bits for num of frame bits - 1
* 6 bits for num of frame bits
* 2 bits for the exception code: 00: byte, 01: short, 10: int
* 1 bit unused
*
*/
// TODO: exception num should never be equal with uncompressed int num!!!
// first exception ranges from -1 ~ 255
// the problem is that we don't need first exception to be -1 ...
// it is ok to range from 0~255, and judge exception for exception num (0~255)
// uncompressed int num: (1~256)
static int getHeader(int numInts, int numBits, int excNum, int excFirstPos, int excBytes) {
return (numInts-1)
| (((excNum-1) & MASK[8]) << 8)
| ((excFirstPos+1) << 16)
| ((numBits-1) << 24)
| ((excBytes/2) << 29);
| ((numBits) << 24)
| ((excBytes/2) << 30);
}
/**
* Expert: get metadata from header.
*/
public static int getNumInts(int header) {
return (header & MASK[8]) + 1;
}
public static int getExcNum(int header) {
return ((header >> 8) & MASK[8]) + 1;
}
public static int getFirstPos(int header) {
return ((header >> 16) & MASK[8]) - 1;
}
public static int getExcBytes(int header) {
return PER_EXCEPTION_SIZE[(header >> 30) & MASK[2]];
}
public static int getNumBits(int header) {
return ((header >> 24) & MASK[6]);
}
}

4
lucene/core/src/java/org/apache/lucene/codecs/pfor/PackedIntsDecompress.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.codecs.pfor;
/* This code is generated, do not modify. See gendecompress.py */
import java.nio.IntBuffer;
import java.util.Arrays;
final class PackedIntsDecompress {
@ -26,6 +27,9 @@ final class PackedIntsDecompress {
// NOTE: hardwired to blockSize == 128
public static void decode0(final IntBuffer compressedBuffer, final int[] output) {
Arrays.fill(output, 0);
}
public static void decode1(final IntBuffer compressedBuffer, final int[] output) {
final int numFrameBits = 1;
final int mask = (int) ((1L<<numFrameBits) - 1);

7
lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py
View File

@ -69,13 +69,18 @@ def genDecompress():
w("\n/* This code is generated, do not modify. See gendecompress.py */\n\n")
w("import java.nio.IntBuffer;\n\n")
w("import java.nio.IntBuffer;\n")
w("import java.util.Arrays;\n\n")
w("final class PackedIntsDecompress {\n")
w('\n // nocommit: assess perf of this to see if specializing is really needed\n')
w('\n // NOTE: hardwired to blockSize == 128\n\n')
w(' public static void decode0(final IntBuffer compressedBuffer, final int[] output) {\n')
w(' Arrays.fill(output, 0);\n')
w(' }\n')
for numFrameBits in xrange(1, 33):
w(' public static void decode%d(final IntBuffer compressedBuffer, final int[] output) {\n' % numFrameBits)
w(' final int numFrameBits = %d;\n' % numFrameBits)

163
lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java
View File

@ -28,7 +28,7 @@ import org.apache.lucene.util.LuceneTestCase;
/**
* Test the core utility for PFor compress and decompress
* We don't provide test case for For encoder/decoder, since
* We don't specially provide test case for For encoder/decoder, since
* PFor is a extended version of For, and most methods will be reused
* here.
*/
@ -46,37 +46,157 @@ public class TestPForUtil extends LuceneTestCase {
}
/**
* Test correctness of ignored forced exception.
* The trailing forced exceptions shouldn't be reverted
* since they're not necessary.
* Should not encode extra information other than header
*/
public void testForcedException() throws Exception {
public void testPForAllZeros() throws Exception {
int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE;
int ensz;
int[] data=new int[sz];
byte[] res = new byte[4+sz*8];
int[] copy = new int[sz];
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
Arrays.fill(data,0);
ensz = ForUtil.compress(data,sz,resBuffer); // test For
ForUtil.decompress(resBuffer,copy);
assert ensz == 4;
assert cmp(data,sz,copy,sz)==true;
Arrays.fill(data,0);
ensz = PForUtil.compress(data,sz,resBuffer); // test PFor
PForUtil.decompress(resBuffer,copy);
assert ensz == 4;
assert cmp(data,sz,copy,sz)==true;
}
public void testForAllZeros() throws Exception {
int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE;
int ensz;
int[] data=new int[sz];
byte[] res = new byte[4+sz*8];
int[] copy = new int[sz];
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
ensz = ForUtil.compress(data,sz,resBuffer);
}
/**
* Test correctness of forced exception.
* the forced ones should exactly fit max chain
*/
public void testForcedExceptionDistance() throws Exception {
initRandom();
int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE;
int[] data=new int[sz];
byte[] res = new byte[4+sz*8];
int[] copy = new int[sz];
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
int numBits = gen.nextInt(5)+1;
int i,j;
int pace, ensz, header;
int expect, got;
// fill exception value with same pace, there should
// be no forced exceptions.
createDistribution(data, sz, 1, MASK[numBits], MASK[numBits]);
pace = 1<<numBits;
for (i=0,j=0; i<sz; i+=pace) {
int exc = gen.nextInt();
data[i] = (exc & 0xffff0000) == 0 ? exc | 0xffff0000 : exc;
j++;
}
ensz = PForUtil.compress(data,sz,resBuffer);
header = resBuffer.get(0);
expect = j;
got = PForUtil.getExcNum(header);
assert expect == got: expect+" expected but got "+got;
// there should exactly one forced exception before each
// exception when i>0
createDistribution(data, sz, 1, MASK[numBits], MASK[numBits]);
pace = (1<<numBits)+1;
for (i=0,j=0; i<sz; i+=pace) {
int exc = gen.nextInt();
data[i] = (exc & 0xffff0000) == 0 ? exc | 0xffff0000 : exc;
j++;
}
ensz = PForUtil.compress(data,sz,resBuffer);
header = resBuffer.get(0);
expect = 2*(j-1)+1;
got = PForUtil.getExcNum(header);
assert expect == got: expect+" expected but got "+got;
// two forced exception
createDistribution(data, sz, 1, MASK[numBits], MASK[numBits]);
pace = (1<<numBits)*2+1;
for (i=0,j=0; i<sz; i+=pace) {
int exc = gen.nextInt();
data[i] = (exc & 0xffff0000) == 0 ? exc | 0xffff0000 : exc;
j++;
}
ensz = PForUtil.compress(data,sz,resBuffer);
header = resBuffer.get(0);
expect = 3*(j-1)+1;
got = PForUtil.getExcNum(header);
assert expect == got: expect+" expected but got "+got;
}
/**
* Test correctness of ignored forced exception.
* The trailing forced exceptions should always be reverted
* since they're not necessary.
*/
public void testTrailingForcedException() throws Exception {
initRandom();
int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE;
assert sz % 32 == 0;
Integer[] buff= new Integer[sz];
int[] data = new int[sz];
int[] copy = new int[sz];
byte[] res = new byte[4+sz*8];
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
for (int i=0; i<sz-1; ++i)
buff[i]=gen.nextInt() & 1;
buff[sz-1]=gen.nextInt() & 0xffffffff; // create only one exception
Collections.shuffle(Arrays.asList(buff),gen);
int excIndex = gen.nextInt(sz/2);
int excValue = gen.nextInt();
if ((excValue & 0xffff0000) == 0) {
excValue |= 0xffff0000; // always prepare a 4 bytes exception
}
// make value of numFrameBits to be small,
// thus easy to get forced exceptions
for (int i=0; i<sz; ++i) {
buff[i]=gen.nextInt() & 1;
}
// create only one value exception
buff[excIndex]=excValue;
for (int i=0; i<sz; ++i)
data[i] = buff[i];
int ensz = PForUtil.compress(data,sz,resBuffer);
assert (ensz <= sz*8+4); // must not exceed the loose upperbound
assert (ensz <= sz*8+4): ensz+" > "+sz*8+4; // must not exceed the loose upperbound
assert (ensz >= 8); // at least we have a header along with an exception, right?
resBuffer.rewind();
PForUtil.decompress(resBuffer,copy);
//println(getHex(data,sz)+"\n");
//println(getHex(res,ensz)+"\n");
//println(getHex(copy,sz)+"\n");
// println(getHex(data,sz)+"\n");
// println(getHex(res,ensz)+"\n");
// println(getHex(copy,sz)+"\n");
// fetch the last int, i.e. last exception.
int lastExc = (res[ensz-4] << 24) |
((0xff & res[ensz-3]) << 16) |
((0xff & res[ensz-2]) << 8 ) |
(0xff & res[ensz-1]);
// trailing forced exceptions are suppressed,
// so the last exception should be what we assigned.
assert lastExc==excValue;
assert cmp(data,sz,copy,sz)==true;
}
@ -87,18 +207,18 @@ public class TestPForUtil extends LuceneTestCase {
*/
public void testAllDistribution() throws Exception {
initRandom();
int sz = ForPostingsFormat.DEFAULT_BLOCK_SIZE;
int[] data = new int[sz];
for (int i=0; i<=32; ++i) { // try to test every kinds of distribution
double alpha=gen.nextDouble(); // rate of normal value
for (int j=0; j<=32; ++j) {
tryDistribution(ForPostingsFormat.DEFAULT_BLOCK_SIZE,alpha,MASK[i],MASK[j]);
createDistribution(data,sz,alpha,MASK[i],MASK[j]);
tryCompressAndDecompress(data, sz);
}
}
}
public void tryDistribution(int sz, double alpha, int masknorm, int maskexc) throws Exception {
public void createDistribution(int[] data, int sz, double alpha, int masknorm, int maskexc) {
Integer[] buff= new Integer[sz];
int[] data = new int[sz];
byte[] res = new byte[4+sz*8]; // loosely upperbound
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
int i=0;
for (; i<sz*alpha; ++i)
buff[i]=gen.nextInt() & masknorm;
@ -107,6 +227,10 @@ public class TestPForUtil extends LuceneTestCase {
Collections.shuffle(Arrays.asList(buff),gen);
for (i=0; i<sz; ++i)
data[i] = buff[i];
}
public void tryCompressAndDecompress(final int[] data, int sz) throws Exception {
byte[] res = new byte[4+sz*8]; // loosely upperbound
IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
int ensz = PForUtil.compress(data,sz,resBuffer);
@ -160,6 +284,9 @@ public class TestPForUtil extends LuceneTestCase {
}
return hex.toString();
}
static void eprintln(String format, Object... args) {
System.err.println(String.format(format,args));
}
static void println(String format, Object... args) {
System.out.println(String.format(format,args));
}