Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3892: Ability to decode to int[] arrays (in addition to long[] arrays). 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1370817 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2012-08-08 15:46:28 +00:00
parent 8eaed3c865
commit 5e948be185
4 changed files with 9178 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9056
lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java
View File

File diff suppressed because it is too large Load Diff

52
lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
View File

@ -282,6 +282,32 @@ public class PackedInts {
*/
void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations);
/**
* Read <code>iterations * blockCount()</code> blocks from <code>blocks</code>,
* decode them and write <code>iterations * valueCount()</code> values into
* <code>values</code>.
*
* @param blocks the long blocks that hold packed integer values
* @param blocksOffset the offset where to start reading blocks
* @param values the values buffer
* @param valuesOffset the offset where to start writing values
* @param iterations controls how much data to decode
*/
void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
/**
* Read <code>8 * iterations * blockCount()</code> blocks from <code>blocks</code>,
* decode them and write <code>iterations * valueCount()</code> values into
* <code>values</code>.
*
* @param blocks the long blocks that hold packed integer values
* @param blocksOffset the offset where to start reading blocks
* @param values the values buffer
* @param valuesOffset the offset where to start writing values
* @param iterations controls how much data to decode
*/
void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
}
/**
@ -326,6 +352,32 @@ public class PackedInts {
*/
void encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
/**
* Read <code>iterations * valueCount()</code> values from <code>values</code>,
* encode them and write <code>iterations * blockCount()</code> blocks into
* <code>blocks</code>.
*
* @param blocks the long blocks that hold packed integer values
* @param blocksOffset the offset where to start writing blocks
* @param values the values buffer
* @param valuesOffset the offset where to start reading values
* @param iterations controls how much data to encode
*/
void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations);
/**
* Read <code>iterations * valueCount()</code> values from <code>values</code>,
* encode them and write <code>8 * iterations * blockCount()</code> blocks into
* <code>blocks</code>.
*
* @param blocks the long blocks that hold packed integer values
* @param blocksOffset the offset where to start writing blocks
* @param values the values buffer
* @param valuesOffset the offset where to start reading values
* @param iterations controls how much data to encode
*/
void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
}
/**

50
lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
View File

@ -53,16 +53,27 @@ enum BulkOperation implements PackedInts.Decoder, PackedInts.Encoder {
FOOTER="""
private static long[] toLongArray(byte[] bytes, int offset, int length) {
final int longsLen = length >>> 3;
LongBuffer longs = LongBuffer.allocate(longsLen);
longs.put(ByteBuffer.wrap(bytes, offset, length).asLongBuffer());
return longs.array();
private static long[] toLongArray(int[] ints, int offset, int length) {
long[] arr = new long[length];
for (int i = 0; i < length; ++i) {
arr[i] = ints[offset + i];
}
return arr;
}
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
decode(toLongArray(blocks, blocksOffset, 8 * iterations * blockCount()), 0, values, valuesOffset, iterations);
public void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
throw new UnsupportedOperationException();
}
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
throw new UnsupportedOperationException();
}
@Override
public void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
encode(toLongArray(values, valuesOffset, iterations * valueCount()), 0, blocks, blocksOffset, iterations);
}
@Override
@ -72,6 +83,13 @@ FOOTER="""
ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
}
@Override
public void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations) {
final long[] longBLocks = new long[blockCount() * iterations];
encode(values, valuesOffset, longBLocks, 0, iterations);
ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
}
/**
* For every number of bits per value, there is a minimum number of
* blocks (b) / values (v) you need to write in order to reach the next block
@ -139,9 +157,9 @@ def packed64singleblock(bpv, f):
f.write(" public int valueCount() {\n")
f.write(" return %d;\n" %values)
f.write(" }\n\n")
#p64sb_decode(bpv, 32)
p64sb_decode(bpv, 32)
p64sb_decode(bpv, 64)
#p64sb_encode(bpv, 32)
p64sb_encode(bpv, 32)
p64sb_encode(bpv, 64)
f.write(" }")
@ -178,7 +196,7 @@ def p64sb_decode(bpv, bits):
f.write(" assert blocksOffset + 8 * iterations * blockCount() <= blocks.length;\n")
f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
if bpv >= 32:
if bpv >= 32 and bits > 32:
for i in xrange(7, -1, -1):
f.write(" final long byte%d = blocks[blocksOffset++] & 0xFF;\n" %i)
else:
@ -222,6 +240,10 @@ def p64sb_encode(bpv, bits):
typ = get_type(bits)
mask_start, mask_end = masks(bits)
f.write(" public void encode(%s[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {\n" %typ)
if bits < bpv:
f.write(" throw new UnsupportedOperationException();\n")
f.write(" }\n\n")
return
f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
@ -258,6 +280,10 @@ def packed64(bpv, f):
System.arraycopy(blocks, blocksOffset, values, valuesOffset, valueCount() * iterations);
}
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
LongBuffer.wrap(values, valuesOffset, iterations * valueCount()).put(ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer());
}
public void encode(long[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
System.arraycopy(values, valuesOffset, blocks, blocksOffset, valueCount() * iterations);
}
@ -265,9 +291,9 @@ def packed64(bpv, f):
}
""")
else:
#p64_decode(bpv, 32, values)
p64_decode(bpv, 32, values)
p64_decode(bpv, 64, values)
#p64_encode(bpv, 32, values)
p64_encode(bpv, 32, values)
p64_encode(bpv, 64, values)
f.write(" }\n")

39
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -667,12 +667,27 @@ public class TestPackedInts extends LuceneTestCase {
for (long value : values) {
assertTrue(value <= PackedInts.maxValue(bpv));
}
// test decoding to int[]
final int[] intValues;
if (bpv <= 32) {
intValues = new int[values.length];
decoder.decode(blocks, blocksOffset, intValues, valuesOffset, iterations);
assertTrue(equals(intValues, values));
} else {
intValues = null;
}
// 3. re-encode
final long[] blocks2 = new long[blocksOffset2 + blocksLen];
encoder.encode(values, valuesOffset, blocks2, blocksOffset2, iterations);
assertArrayEquals(msg, Arrays.copyOfRange(blocks, blocksOffset, blocks.length),
Arrays.copyOfRange(blocks2, blocksOffset2, blocks2.length));
// test encoding from int[]
if (bpv <= 32) {
final long[] blocks3 = new long[blocks2.length];
encoder.encode(intValues, valuesOffset, blocks3, blocksOffset2, iterations);
assertArrayEquals(msg, blocks2, blocks3);
}
// 4. byte[] decoding
final byte[] byteBlocks = new byte[8 * blocks.length];
@ -683,13 +698,37 @@ public class TestPackedInts extends LuceneTestCase {
assertTrue(msg, value <= PackedInts.maxValue(bpv));
}
assertArrayEquals(msg, values, values2);
// test decoding to int[]
if (bpv <= 32) {
final int[] intValues2 = new int[values2.length];
decoder.decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, iterations);
assertTrue(msg, equals(intValues2, values2));
}
// 5. byte[] encoding
final byte[] blocks3 = new byte[8 * (blocksOffset2 + blocksLen)];
encoder.encode(values, valuesOffset, blocks3, 8 * blocksOffset2, iterations);
assertEquals(msg, LongBuffer.wrap(blocks2), ByteBuffer.wrap(blocks3).asLongBuffer());
// test encoding from int[]
if (bpv <= 32) {
final byte[] blocks4 = new byte[blocks3.length];
encoder.encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, iterations);
assertArrayEquals(msg, blocks3, blocks4);
}
}
}
}
private static boolean equals(int[] ints, long[] longs) {
if (ints.length != longs.length) {
return false;
}
for (int i = 0; i < ints.length; ++i) {
if ((ints[i] & 0xFFFFFFFFL) != longs[i]) {
return false;
}
}
return true;
}
}