From 4a5496ac97d66e937e26cb200a647dae735992f3 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Thu, 9 Aug 2012 22:29:36 +0000 Subject: [PATCH] LUCENE-3892: replace Block with BlockPacked git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1371519 13f79535-47bb-0310-9956-ffa450edef68 --- .../codecs/block/BlockPostingsFormat.java | 3 +- .../codecs/block/BlockPostingsReader.java | 102 +- .../codecs/block/BlockPostingsWriter.java | 63 +- .../lucene/codecs/block/BlockSkipReader.java | 3 +- .../lucene/codecs/block/BlockSkipWriter.java | 2 +- .../apache/lucene/codecs/block/ForUtil.java | 326 +-- .../codecs/block/PackedIntsDecompress.java | 1769 ----------------- .../lucene/codecs/block/gendecompress.py | 107 - .../BlockPackedPostingsFormat.java | 110 - .../BlockPackedPostingsReader.java | 1549 --------------- .../BlockPackedPostingsWriter.java | 562 ------ .../blockpacked/BlockPackedSkipReader.java | 244 --- .../blockpacked/BlockPackedSkipWriter.java | 163 -- .../lucene/codecs/blockpacked/ForUtil.java | 240 --- .../org.apache.lucene.codecs.PostingsFormat | 1 - .../{blockpacked => block}/TestForUtil.java | 8 +- 16 files changed, 284 insertions(+), 4968 deletions(-) delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/block/PackedIntsDecompress.java delete mode 100755 lucene/core/src/java/org/apache/lucene/codecs/block/gendecompress.py delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsFormat.java delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java rename lucene/core/src/test/org/apache/lucene/codecs/{blockpacked => block}/TestForUtil.java (91%) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java index 6628bc35383..31955241301 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java @@ -43,7 +43,8 @@ public final class BlockPostingsFormat extends PostingsFormat { private final int maxTermBlockSize; // nocommit is this right?: - // NOTE: must be factor of .... 32? + // NOTE: should be at least 64 because of PackedInts long-aligned encoding/decoding + // NOTE: must be factor of ... 64? public final static int BLOCK_SIZE = 128; public BlockPostingsFormat() { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java index c6f26f44201..7286c86ab10 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java @@ -17,9 +17,11 @@ package org.apache.lucene.codecs.block; * limitations under the License. */ +import static org.apache.lucene.codecs.block.BlockPostingsFormat.BLOCK_SIZE; +import static org.apache.lucene.codecs.block.ForUtil.MAX_DATA_SIZE; +import static org.apache.lucene.codecs.block.ForUtil.MAX_ENCODED_SIZE; + import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; import java.util.Arrays; import org.apache.lucene.codecs.BlockTermState; @@ -27,8 +29,8 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; @@ -43,8 +45,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; -import static org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat.BLOCK_SIZE; - /** * Concrete class that reads docId(maybe frq,pos,offset,payloads) list * with postings format. @@ -58,6 +58,8 @@ public final class BlockPostingsReader extends PostingsReaderBase { private final IndexInput posIn; private final IndexInput payIn; + private final ForUtil forUtil; + public static boolean DEBUG = false; // nocommit @@ -76,6 +78,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { BlockPostingsWriter.DOC_CODEC, BlockPostingsWriter.VERSION_START, BlockPostingsWriter.VERSION_START); + forUtil = new ForUtil(docIn); if (fieldInfos.hasProx()) { posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockPostingsFormat.POS_EXTENSION), @@ -119,7 +122,11 @@ public final class BlockPostingsReader extends PostingsReaderBase { } } - static void readVIntBlock(IndexInput docIn, int[] docBuffer, int[] freqBuffer, int num, boolean indexHasFreq) throws IOException { + /** + * Read values that have been written using variable-length encoding instead of bit-packing. + */ + private static void readVIntBlock(IndexInput docIn, int[] docBuffer, + int[] freqBuffer, int num, boolean indexHasFreq) throws IOException { if (indexHasFreq) { for(int i=0;i= 0; indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; indexHasPayloads = fieldInfo.hasPayloads(); - encoded = new byte[BLOCK_SIZE*4]; - encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); + encoded = new byte[MAX_ENCODED_SIZE]; } public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { @@ -419,14 +413,16 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, docDeltaBuffer); + forUtil.readBlock(docIn, encoded, docDeltaBuffer); + if (indexHasFreq) { if (DEBUG) { System.out.println(" fill freq block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, freqBuffer); + forUtil.readBlock(docIn, encoded, freqBuffer); } } else { + // Read vInts: if (DEBUG) { System.out.println(" fill last vInt block from fp=" + docIn.getFilePointer()); } @@ -444,6 +440,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" docUpto=" + docUpto + " (of df=" + docFreq + ") docBufferUpto=" + docBufferUpto); } + if (docUpto == docFreq) { if (DEBUG) { System.out.println(" return doc=END"); @@ -453,6 +450,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (docBufferUpto == BLOCK_SIZE) { refillDocs(); } + if (DEBUG) { System.out.println(" accum=" + accum + " docDeltaBuffer[" + docBufferUpto + "]=" + docDeltaBuffer[docBufferUpto]); } @@ -571,11 +569,10 @@ public final class BlockPostingsReader extends PostingsReaderBase { final class BlockDocsAndPositionsEnum extends DocsAndPositionsEnum { private final byte[] encoded; - private final IntBuffer encodedBuffer; - private final int[] docDeltaBuffer = new int[BLOCK_SIZE]; - private final int[] freqBuffer = new int[BLOCK_SIZE]; - private final int[] posDeltaBuffer = new int[BLOCK_SIZE]; + private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE]; + private final int[] freqBuffer = new int[MAX_DATA_SIZE]; + private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE]; private int docBufferUpto; private int posBufferUpto; @@ -634,8 +631,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { this.startDocIn = BlockPostingsReader.this.docIn; this.docIn = (IndexInput) startDocIn.clone(); this.posIn = (IndexInput) BlockPostingsReader.this.posIn.clone(); - encoded = new byte[BLOCK_SIZE*4]; - encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); + encoded = new byte[MAX_ENCODED_SIZE]; indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; indexHasPayloads = fieldInfo.hasPayloads(); } @@ -694,12 +690,13 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, docDeltaBuffer); + forUtil.readBlock(docIn, encoded, docDeltaBuffer); if (DEBUG) { System.out.println(" fill freq block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, freqBuffer); + forUtil.readBlock(docIn, encoded, freqBuffer); } else { + // Read vInts: if (DEBUG) { System.out.println(" fill last vInt doc block from fp=" + docIn.getFilePointer()); } @@ -740,7 +737,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" bulk pos block @ fp=" + posIn.getFilePointer()); } - readBlock(posIn, encoded, encodedBuffer, posDeltaBuffer); + forUtil.readBlock(posIn, encoded, posDeltaBuffer); } } @@ -905,7 +902,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { System.out.println(" skip whole block @ fp=" + posIn.getFilePointer()); } assert posIn.getFilePointer() != lastPosBlockFP; - skipBlock(posIn); + forUtil.skipBlock(posIn); toSkip -= BLOCK_SIZE; } refillPositions(); @@ -976,11 +973,10 @@ public final class BlockPostingsReader extends PostingsReaderBase { final class EverythingEnum extends DocsAndPositionsEnum { private final byte[] encoded; - private final IntBuffer encodedBuffer; - private final int[] docDeltaBuffer = new int[BLOCK_SIZE]; - private final int[] freqBuffer = new int[BLOCK_SIZE]; - private final int[] posDeltaBuffer = new int[BLOCK_SIZE]; + private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE]; + private final int[] freqBuffer = new int[MAX_DATA_SIZE]; + private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE]; private final int[] payloadLengthBuffer; private final int[] offsetStartDeltaBuffer; @@ -1058,12 +1054,11 @@ public final class BlockPostingsReader extends PostingsReaderBase { this.docIn = (IndexInput) startDocIn.clone(); this.posIn = (IndexInput) BlockPostingsReader.this.posIn.clone(); this.payIn = (IndexInput) BlockPostingsReader.this.payIn.clone(); - encoded = new byte[BLOCK_SIZE*4]; - encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); + encoded = new byte[MAX_ENCODED_SIZE]; indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; if (indexHasOffsets) { - offsetStartDeltaBuffer = new int[BLOCK_SIZE]; - offsetLengthBuffer = new int[BLOCK_SIZE]; + offsetStartDeltaBuffer = new int[MAX_DATA_SIZE]; + offsetLengthBuffer = new int[MAX_DATA_SIZE]; } else { offsetStartDeltaBuffer = null; offsetLengthBuffer = null; @@ -1073,7 +1068,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { indexHasPayloads = fieldInfo.hasPayloads(); if (indexHasPayloads) { - payloadLengthBuffer = new int[BLOCK_SIZE]; + payloadLengthBuffer = new int[MAX_DATA_SIZE]; payloadBytes = new byte[128]; payload = new BytesRef(); } else { @@ -1138,11 +1133,11 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, docDeltaBuffer); + forUtil.readBlock(docIn, encoded, docDeltaBuffer); if (DEBUG) { System.out.println(" fill freq block from fp=" + docIn.getFilePointer()); } - readBlock(docIn, encoded, encodedBuffer, freqBuffer); + forUtil.readBlock(docIn, encoded, freqBuffer); } else { if (DEBUG) { System.out.println(" fill last vInt doc block from fp=" + docIn.getFilePointer()); @@ -1202,13 +1197,13 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" bulk pos block @ fp=" + posIn.getFilePointer()); } - readBlock(posIn, encoded, encodedBuffer, posDeltaBuffer); + forUtil.readBlock(posIn, encoded, posDeltaBuffer); if (indexHasPayloads) { if (DEBUG) { System.out.println(" bulk payload block @ pay.fp=" + payIn.getFilePointer()); } - readBlock(payIn, encoded, encodedBuffer, payloadLengthBuffer); + forUtil.readBlock(payIn, encoded, payloadLengthBuffer); int numBytes = payIn.readVInt(); if (DEBUG) { System.out.println(" " + numBytes + " payload bytes @ pay.fp=" + payIn.getFilePointer()); @@ -1224,8 +1219,8 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (DEBUG) { System.out.println(" bulk offset block @ pay.fp=" + payIn.getFilePointer()); } - readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaBuffer); - readBlock(payIn, encoded, encodedBuffer, offsetLengthBuffer); + forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer); + forUtil.readBlock(payIn, encoded, offsetLengthBuffer); } } } @@ -1268,6 +1263,7 @@ public final class BlockPostingsReader extends PostingsReaderBase { lastStartOffset = 0; return doc; } + if (DEBUG) { System.out.println(" doc=" + accum + " is deleted; try next doc"); } @@ -1412,11 +1408,11 @@ public final class BlockPostingsReader extends PostingsReaderBase { System.out.println(" skip whole block @ fp=" + posIn.getFilePointer()); } assert posIn.getFilePointer() != lastPosBlockFP; - skipBlock(posIn); + forUtil.skipBlock(posIn); if (indexHasPayloads) { // Skip payloadLength block: - skipBlock(payIn); + forUtil.skipBlock(payIn); // Skip payloadBytes block: int numBytes = payIn.readVInt(); @@ -1426,8 +1422,8 @@ public final class BlockPostingsReader extends PostingsReaderBase { if (indexHasOffsets) { // Must load offset blocks merely to sum // up into lastStartOffset: - readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaBuffer); - readBlock(payIn, encoded, encodedBuffer, offsetLengthBuffer); + forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer); + forUtil.readBlock(payIn, encoded, offsetLengthBuffer); for(int i=0;i= 0; - encodeNormalValue(intBuffer, i, data[i], numBits); - } - - return numBits; - } +final class ForUtil { /** - * Save only one int when the whole block equals to a - * single value. + * Special number of bits per value used whenever all values to encode are equal. */ - static int compressDuplicateBlock(final int[] data, IntBuffer intBuffer) { - intBuffer.put(0, data[0]); - return 0; - } - - /** Decompress given Integer buffer into int array. - * - * @param intBuffer integer buffer to hold compressed data - * @param data int array to hold uncompressed data - * @param header header of current block, which contains numFrameBits - */ - static void decompress(IntBuffer intBuffer, int[] data, int header) { - // since this buffer is reused at upper level, rewind first - intBuffer.rewind(); - - // NOTE: header == numBits now, but we may change that - final int numBits = header; - assert numBits >=0 && numBits < 32; - decompressCore(intBuffer, data, numBits); - } - - public static void decompressCore(IntBuffer intBuffer, int[] data, int numBits) { - switch(numBits) { - case 0: PackedIntsDecompress.decode0(intBuffer, data); break; - case 1: PackedIntsDecompress.decode1(intBuffer, data); break; - case 2: PackedIntsDecompress.decode2(intBuffer, data); break; - case 3: PackedIntsDecompress.decode3(intBuffer, data); break; - case 4: PackedIntsDecompress.decode4(intBuffer, data); break; - case 5: PackedIntsDecompress.decode5(intBuffer, data); break; - case 6: PackedIntsDecompress.decode6(intBuffer, data); break; - case 7: PackedIntsDecompress.decode7(intBuffer, data); break; - case 8: PackedIntsDecompress.decode8(intBuffer, data); break; - case 9: PackedIntsDecompress.decode9(intBuffer, data); break; - case 10: PackedIntsDecompress.decode10(intBuffer, data); break; - case 11: PackedIntsDecompress.decode11(intBuffer, data); break; - case 12: PackedIntsDecompress.decode12(intBuffer, data); break; - case 13: PackedIntsDecompress.decode13(intBuffer, data); break; - case 14: PackedIntsDecompress.decode14(intBuffer, data); break; - case 15: PackedIntsDecompress.decode15(intBuffer, data); break; - case 16: PackedIntsDecompress.decode16(intBuffer, data); break; - case 17: PackedIntsDecompress.decode17(intBuffer, data); break; - case 18: PackedIntsDecompress.decode18(intBuffer, data); break; - case 19: PackedIntsDecompress.decode19(intBuffer, data); break; - case 20: PackedIntsDecompress.decode20(intBuffer, data); break; - case 21: PackedIntsDecompress.decode21(intBuffer, data); break; - case 22: PackedIntsDecompress.decode22(intBuffer, data); break; - case 23: PackedIntsDecompress.decode23(intBuffer, data); break; - case 24: PackedIntsDecompress.decode24(intBuffer, data); break; - case 25: PackedIntsDecompress.decode25(intBuffer, data); break; - case 26: PackedIntsDecompress.decode26(intBuffer, data); break; - case 27: PackedIntsDecompress.decode27(intBuffer, data); break; - case 28: PackedIntsDecompress.decode28(intBuffer, data); break; - case 29: PackedIntsDecompress.decode29(intBuffer, data); break; - case 30: PackedIntsDecompress.decode30(intBuffer, data); break; - case 31: PackedIntsDecompress.decode31(intBuffer, data); break; - // nocommit have default throw exc? or add assert up above - } - } - - static void encodeNormalValue(IntBuffer intBuffer, int pos, int value, int numBits) { - final int globalBitPos = numBits*pos; // position in bit stream - final int localBitPos = globalBitPos & 31; // position inside an int - int intPos = globalBitPos/32; // which integer to locate - setBufferIntBits(intBuffer, intPos, localBitPos, numBits, value); - if ((localBitPos + numBits) > 32) { // value does not fit in this int, fill tail - setBufferIntBits(intBuffer, intPos+1, 0, - (localBitPos+numBits-32), - (value >>> (32-localBitPos))); - } - } - - static void setBufferIntBits(IntBuffer intBuffer, int intPos, int firstBitPos, int numBits, int value) { - assert (value & ~MASK[numBits]) == 0; - // safely discards those msb parts when firstBitPos+numBits>32 - intBuffer.put(intPos, - (intBuffer.get(intPos) & ~(MASK[numBits] << firstBitPos)) - | (value << firstBitPos)); - } + private static final int ALL_VALUES_EQUAL = 0; + private static final int PACKED_INTS_VERSION = 0; // nocommit: encode in the stream? /** - * Returns number of bits necessary to represent max value. + * Upper limit of the number of bytes that might be required to stored + * BLOCK_SIZE encoded values. */ - static int getNumBits(final int[] data) { - if (isAllEqual(data)) { - return 0; - } - int size=data.length; - int optBits=1; - for (int i=0; iBLOCK_SIZE are garbage, it is necessary to allocate value buffers + * whose size is >= MAX_DATA_SIZE to avoid {@link ArrayIndexOutOfBoundsException}s. + */ + static final int MAX_DATA_SIZE; + static { + int minDataSize = 0; + for (PackedInts.Format format : PackedInts.Format.values()) { + for (int bpv = 1; bpv <= 32; ++bpv) { + if (!format.isSupported(bpv)) { + continue; + } + final PackedInts.Decoder decoder = PackedInts.getDecoder(format, PACKED_INTS_VERSION, bpv); + final int iterations = (int) Math.ceil((float) BLOCK_SIZE / decoder.valueCount()); + minDataSize = Math.max(minDataSize, iterations * decoder.valueCount()); } } - assert optBits < 32; - return optBits; + MAX_DATA_SIZE = minDataSize; + } + + /** + * Compute the number of iterations required to decode BLOCK_SIZE + * values with the provided {@link Decoder}. + */ + private static int computeIterations(PackedInts.Decoder decoder) { + return (int) Math.ceil((float) BLOCK_SIZE / decoder.valueCount()); + } + + /** + * Compute the number of bytes required to encode a block of values that require + * bitsPerValue bits per value with format format. + */ + private static int encodedSize(PackedInts.Format format, int bitsPerValue) { + return format.nblocks(bitsPerValue, BLOCK_SIZE) << 3; + } + + private final int[] encodedSizes; + private final PackedInts.Encoder[] encoders; + private final PackedInts.Decoder[] decoders; + private final int[] iterations; + + /** + * Create a new {@link ForUtil} instance and save state into out. + */ + ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { + encodedSizes = new int[33]; + encoders = new PackedInts.Encoder[33]; + decoders = new PackedInts.Decoder[33]; + iterations = new int[33]; + + for (int bpv = 1; bpv <= 32; ++bpv) { + final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( + BLOCK_SIZE, bpv, acceptableOverheadRatio); + assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); + assert formatAndBits.bitsPerValue <= 32; + encodedSizes[bpv] = encodedSize(formatAndBits.format, formatAndBits.bitsPerValue); + encoders[bpv] = PackedInts.getEncoder( + formatAndBits.format, PACKED_INTS_VERSION, formatAndBits.bitsPerValue); + decoders[bpv] = PackedInts.getDecoder( + formatAndBits.format, PACKED_INTS_VERSION, formatAndBits.bitsPerValue); + iterations[bpv] = computeIterations(decoders[bpv]); + + out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); + } + } + + /** + * Restore a {@link ForUtil} from a {@link DataInput}. + */ + ForUtil(DataInput in) throws IOException { + encodedSizes = new int[33]; + encoders = new PackedInts.Encoder[33]; + decoders = new PackedInts.Decoder[33]; + iterations = new int[33]; + + for (int bpv = 1; bpv <= 32; ++bpv) { + final int code = in.readVInt(); + final int formatId = code >>> 5; + final int bitsPerValue = (code & 31) + 1; + + final PackedInts.Format format = PackedInts.Format.byId(formatId); + assert format.isSupported(bitsPerValue); + encodedSizes[bpv] = encodedSize(format, bitsPerValue); + encoders[bpv] = PackedInts.getEncoder( + format, PACKED_INTS_VERSION, bitsPerValue); + decoders[bpv] = PackedInts.getDecoder( + format, PACKED_INTS_VERSION, bitsPerValue); + iterations[bpv] = computeIterations(decoders[bpv]); + } + } + + /** + * Write a block of data (For format). + * + * @param data the data to write + * @param encoded a buffer to use to encode data + * @param out the destination output + * @throws IOException + */ + void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { + if (isAllEqual(data)) { + out.writeVInt(ALL_VALUES_EQUAL); + out.writeInt(data[0]); + return; + } + + final int numBits = bitsRequired(data); + assert numBits > 0 && numBits <= 32 : numBits; + final PackedInts.Encoder encoder = encoders[numBits]; + final int iters = iterations[numBits]; + assert iters * encoder.valueCount() >= BLOCK_SIZE; + final int encodedSize = encodedSizes[numBits]; + assert (iters * encoder.blockCount()) << 3 >= encodedSize; + + out.writeVInt(numBits); + + encoder.encode(data, 0, encoded, 0, iters); + out.writeBytes(encoded, encodedSize); + } + + /** + * Read the next block of data (For format). + * + * @param in the input to use to read data + * @param encoded a buffer that can be used to store encoded data + * @param decoded where to write decoded data + * @throws IOException + */ + void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { + final int numBits = in.readVInt(); + assert numBits <= 32 : numBits; + + if (numBits == ALL_VALUES_EQUAL) { + final int value = in.readInt(); + Arrays.fill(decoded, 0, BLOCK_SIZE, value); + return; + } + + final int encodedSize = encodedSizes[numBits]; + in.readBytes(encoded, 0, encodedSize); + + final PackedInts.Decoder decoder = decoders[numBits]; + final int iters = iterations[numBits]; + assert iters * decoder.valueCount() >= BLOCK_SIZE; + + decoder.decode(encoded, 0, decoded, 0, iters); + } + + /** + * Skip the next block of data. + * + * @param in the input where to read data + * @throws IOException + */ + void skipBlock(IndexInput in) throws IOException { + final int numBits = in.readVInt(); + if (numBits == ALL_VALUES_EQUAL) { + in.seek(in.getFilePointer() + 4); + return; + } + assert numBits > 0 && numBits <= 32 : numBits; + final int encodedSize = encodedSizes[numBits]; + in.seek(in.getFilePointer() + encodedSize); } // nocommit: we must have a util function for this, hmm? - protected static boolean isAllEqual(final int[] data) { - int len = data.length; - int v = data[0]; - for (int i=1; idata. */ - static int getEncodedSize(int numBits) { - // NOTE: works only because BLOCK_SIZE is 0 mod 8: - return numBits == 0 ? 4 : numBits*BLOCK_SIZE/8; + private static int bitsRequired(final int[] data) { + long or = 0; + for (int i = 0; i < BLOCK_SIZE; ++i) { + or |= data[i]; + } + return PackedInts.bitsRequired(or); } + } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/block/PackedIntsDecompress.java b/lucene/core/src/java/org/apache/lucene/codecs/block/PackedIntsDecompress.java deleted file mode 100644 index e0b82a415bd..00000000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/block/PackedIntsDecompress.java +++ /dev/null @@ -1,1769 +0,0 @@ -package org.apache.lucene.codecs.block; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* This code is generated, do not modify. See gendecompress.py */ - -import java.nio.IntBuffer; -import java.util.Arrays; - -final class PackedIntsDecompress { - - // nocommit: assess perf of this to see if specializing is really needed - - // NOTE: hardwired to blockSize == 128 - - public static void decode0(final IntBuffer compressedBuffer, final int[] output) { - Arrays.fill(output, compressedBuffer.get()); - } - public static void decode1(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 1; - final int mask = (int) ((1L<>> 1) & mask; - output[2 + outputOffset] = (intValue0 >>> 2) & mask; - output[3 + outputOffset] = (intValue0 >>> 3) & mask; - output[4 + outputOffset] = (intValue0 >>> 4) & mask; - output[5 + outputOffset] = (intValue0 >>> 5) & mask; - output[6 + outputOffset] = (intValue0 >>> 6) & mask; - output[7 + outputOffset] = (intValue0 >>> 7) & mask; - output[8 + outputOffset] = (intValue0 >>> 8) & mask; - output[9 + outputOffset] = (intValue0 >>> 9) & mask; - output[10 + outputOffset] = (intValue0 >>> 10) & mask; - output[11 + outputOffset] = (intValue0 >>> 11) & mask; - output[12 + outputOffset] = (intValue0 >>> 12) & mask; - output[13 + outputOffset] = (intValue0 >>> 13) & mask; - output[14 + outputOffset] = (intValue0 >>> 14) & mask; - output[15 + outputOffset] = (intValue0 >>> 15) & mask; - output[16 + outputOffset] = (intValue0 >>> 16) & mask; - output[17 + outputOffset] = (intValue0 >>> 17) & mask; - output[18 + outputOffset] = (intValue0 >>> 18) & mask; - output[19 + outputOffset] = (intValue0 >>> 19) & mask; - output[20 + outputOffset] = (intValue0 >>> 20) & mask; - output[21 + outputOffset] = (intValue0 >>> 21) & mask; - output[22 + outputOffset] = (intValue0 >>> 22) & mask; - output[23 + outputOffset] = (intValue0 >>> 23) & mask; - output[24 + outputOffset] = (intValue0 >>> 24) & mask; - output[25 + outputOffset] = (intValue0 >>> 25) & mask; - output[26 + outputOffset] = (intValue0 >>> 26) & mask; - output[27 + outputOffset] = (intValue0 >>> 27) & mask; - output[28 + outputOffset] = (intValue0 >>> 28) & mask; - output[29 + outputOffset] = (intValue0 >>> 29) & mask; - output[30 + outputOffset] = (intValue0 >>> 30) & mask; - output[31 + outputOffset] = intValue0 >>> 31; - outputOffset += 32; - } - } - public static void decode2(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 2; - final int mask = (int) ((1L<>> 2) & mask; - output[2 + outputOffset] = (intValue0 >>> 4) & mask; - output[3 + outputOffset] = (intValue0 >>> 6) & mask; - output[4 + outputOffset] = (intValue0 >>> 8) & mask; - output[5 + outputOffset] = (intValue0 >>> 10) & mask; - output[6 + outputOffset] = (intValue0 >>> 12) & mask; - output[7 + outputOffset] = (intValue0 >>> 14) & mask; - output[8 + outputOffset] = (intValue0 >>> 16) & mask; - output[9 + outputOffset] = (intValue0 >>> 18) & mask; - output[10 + outputOffset] = (intValue0 >>> 20) & mask; - output[11 + outputOffset] = (intValue0 >>> 22) & mask; - output[12 + outputOffset] = (intValue0 >>> 24) & mask; - output[13 + outputOffset] = (intValue0 >>> 26) & mask; - output[14 + outputOffset] = (intValue0 >>> 28) & mask; - output[15 + outputOffset] = intValue0 >>> 30; - output[16 + outputOffset] = intValue1 & mask; - output[17 + outputOffset] = (intValue1 >>> 2) & mask; - output[18 + outputOffset] = (intValue1 >>> 4) & mask; - output[19 + outputOffset] = (intValue1 >>> 6) & mask; - output[20 + outputOffset] = (intValue1 >>> 8) & mask; - output[21 + outputOffset] = (intValue1 >>> 10) & mask; - output[22 + outputOffset] = (intValue1 >>> 12) & mask; - output[23 + outputOffset] = (intValue1 >>> 14) & mask; - output[24 + outputOffset] = (intValue1 >>> 16) & mask; - output[25 + outputOffset] = (intValue1 >>> 18) & mask; - output[26 + outputOffset] = (intValue1 >>> 20) & mask; - output[27 + outputOffset] = (intValue1 >>> 22) & mask; - output[28 + outputOffset] = (intValue1 >>> 24) & mask; - output[29 + outputOffset] = (intValue1 >>> 26) & mask; - output[30 + outputOffset] = (intValue1 >>> 28) & mask; - output[31 + outputOffset] = intValue1 >>> 30; - outputOffset += 32; - } - } - public static void decode3(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 3; - final int mask = (int) ((1L<>> 3) & mask; - output[2 + outputOffset] = (intValue0 >>> 6) & mask; - output[3 + outputOffset] = (intValue0 >>> 9) & mask; - output[4 + outputOffset] = (intValue0 >>> 12) & mask; - output[5 + outputOffset] = (intValue0 >>> 15) & mask; - output[6 + outputOffset] = (intValue0 >>> 18) & mask; - output[7 + outputOffset] = (intValue0 >>> 21) & mask; - output[8 + outputOffset] = (intValue0 >>> 24) & mask; - output[9 + outputOffset] = (intValue0 >>> 27) & mask; - output[10 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; - output[11 + outputOffset] = (intValue1 >>> 1) & mask; - output[12 + outputOffset] = (intValue1 >>> 4) & mask; - output[13 + outputOffset] = (intValue1 >>> 7) & mask; - output[14 + outputOffset] = (intValue1 >>> 10) & mask; - output[15 + outputOffset] = (intValue1 >>> 13) & mask; - output[16 + outputOffset] = (intValue1 >>> 16) & mask; - output[17 + outputOffset] = (intValue1 >>> 19) & mask; - output[18 + outputOffset] = (intValue1 >>> 22) & mask; - output[19 + outputOffset] = (intValue1 >>> 25) & mask; - output[20 + outputOffset] = (intValue1 >>> 28) & mask; - output[21 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; - output[22 + outputOffset] = (intValue2 >>> 2) & mask; - output[23 + outputOffset] = (intValue2 >>> 5) & mask; - output[24 + outputOffset] = (intValue2 >>> 8) & mask; - output[25 + outputOffset] = (intValue2 >>> 11) & mask; - output[26 + outputOffset] = (intValue2 >>> 14) & mask; - output[27 + outputOffset] = (intValue2 >>> 17) & mask; - output[28 + outputOffset] = (intValue2 >>> 20) & mask; - output[29 + outputOffset] = (intValue2 >>> 23) & mask; - output[30 + outputOffset] = (intValue2 >>> 26) & mask; - output[31 + outputOffset] = intValue2 >>> 29; - outputOffset += 32; - } - } - public static void decode4(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 4; - final int mask = (int) ((1L<>> 4) & mask; - output[2 + outputOffset] = (intValue0 >>> 8) & mask; - output[3 + outputOffset] = (intValue0 >>> 12) & mask; - output[4 + outputOffset] = (intValue0 >>> 16) & mask; - output[5 + outputOffset] = (intValue0 >>> 20) & mask; - output[6 + outputOffset] = (intValue0 >>> 24) & mask; - output[7 + outputOffset] = intValue0 >>> 28; - output[8 + outputOffset] = intValue1 & mask; - output[9 + outputOffset] = (intValue1 >>> 4) & mask; - output[10 + outputOffset] = (intValue1 >>> 8) & mask; - output[11 + outputOffset] = (intValue1 >>> 12) & mask; - output[12 + outputOffset] = (intValue1 >>> 16) & mask; - output[13 + outputOffset] = (intValue1 >>> 20) & mask; - output[14 + outputOffset] = (intValue1 >>> 24) & mask; - output[15 + outputOffset] = intValue1 >>> 28; - output[16 + outputOffset] = intValue2 & mask; - output[17 + outputOffset] = (intValue2 >>> 4) & mask; - output[18 + outputOffset] = (intValue2 >>> 8) & mask; - output[19 + outputOffset] = (intValue2 >>> 12) & mask; - output[20 + outputOffset] = (intValue2 >>> 16) & mask; - output[21 + outputOffset] = (intValue2 >>> 20) & mask; - output[22 + outputOffset] = (intValue2 >>> 24) & mask; - output[23 + outputOffset] = intValue2 >>> 28; - output[24 + outputOffset] = intValue3 & mask; - output[25 + outputOffset] = (intValue3 >>> 4) & mask; - output[26 + outputOffset] = (intValue3 >>> 8) & mask; - output[27 + outputOffset] = (intValue3 >>> 12) & mask; - output[28 + outputOffset] = (intValue3 >>> 16) & mask; - output[29 + outputOffset] = (intValue3 >>> 20) & mask; - output[30 + outputOffset] = (intValue3 >>> 24) & mask; - output[31 + outputOffset] = intValue3 >>> 28; - outputOffset += 32; - } - } - public static void decode5(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 5; - final int mask = (int) ((1L<>> 5) & mask; - output[2 + outputOffset] = (intValue0 >>> 10) & mask; - output[3 + outputOffset] = (intValue0 >>> 15) & mask; - output[4 + outputOffset] = (intValue0 >>> 20) & mask; - output[5 + outputOffset] = (intValue0 >>> 25) & mask; - output[6 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; - output[7 + outputOffset] = (intValue1 >>> 3) & mask; - output[8 + outputOffset] = (intValue1 >>> 8) & mask; - output[9 + outputOffset] = (intValue1 >>> 13) & mask; - output[10 + outputOffset] = (intValue1 >>> 18) & mask; - output[11 + outputOffset] = (intValue1 >>> 23) & mask; - output[12 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[13 + outputOffset] = (intValue2 >>> 1) & mask; - output[14 + outputOffset] = (intValue2 >>> 6) & mask; - output[15 + outputOffset] = (intValue2 >>> 11) & mask; - output[16 + outputOffset] = (intValue2 >>> 16) & mask; - output[17 + outputOffset] = (intValue2 >>> 21) & mask; - output[18 + outputOffset] = (intValue2 >>> 26) & mask; - output[19 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; - output[20 + outputOffset] = (intValue3 >>> 4) & mask; - output[21 + outputOffset] = (intValue3 >>> 9) & mask; - output[22 + outputOffset] = (intValue3 >>> 14) & mask; - output[23 + outputOffset] = (intValue3 >>> 19) & mask; - output[24 + outputOffset] = (intValue3 >>> 24) & mask; - output[25 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; - output[26 + outputOffset] = (intValue4 >>> 2) & mask; - output[27 + outputOffset] = (intValue4 >>> 7) & mask; - output[28 + outputOffset] = (intValue4 >>> 12) & mask; - output[29 + outputOffset] = (intValue4 >>> 17) & mask; - output[30 + outputOffset] = (intValue4 >>> 22) & mask; - output[31 + outputOffset] = intValue4 >>> 27; - outputOffset += 32; - } - } - public static void decode6(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 6; - final int mask = (int) ((1L<>> 6) & mask; - output[2 + outputOffset] = (intValue0 >>> 12) & mask; - output[3 + outputOffset] = (intValue0 >>> 18) & mask; - output[4 + outputOffset] = (intValue0 >>> 24) & mask; - output[5 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; - output[6 + outputOffset] = (intValue1 >>> 4) & mask; - output[7 + outputOffset] = (intValue1 >>> 10) & mask; - output[8 + outputOffset] = (intValue1 >>> 16) & mask; - output[9 + outputOffset] = (intValue1 >>> 22) & mask; - output[10 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[11 + outputOffset] = (intValue2 >>> 2) & mask; - output[12 + outputOffset] = (intValue2 >>> 8) & mask; - output[13 + outputOffset] = (intValue2 >>> 14) & mask; - output[14 + outputOffset] = (intValue2 >>> 20) & mask; - output[15 + outputOffset] = intValue2 >>> 26; - output[16 + outputOffset] = intValue3 & mask; - output[17 + outputOffset] = (intValue3 >>> 6) & mask; - output[18 + outputOffset] = (intValue3 >>> 12) & mask; - output[19 + outputOffset] = (intValue3 >>> 18) & mask; - output[20 + outputOffset] = (intValue3 >>> 24) & mask; - output[21 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[22 + outputOffset] = (intValue4 >>> 4) & mask; - output[23 + outputOffset] = (intValue4 >>> 10) & mask; - output[24 + outputOffset] = (intValue4 >>> 16) & mask; - output[25 + outputOffset] = (intValue4 >>> 22) & mask; - output[26 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; - output[27 + outputOffset] = (intValue5 >>> 2) & mask; - output[28 + outputOffset] = (intValue5 >>> 8) & mask; - output[29 + outputOffset] = (intValue5 >>> 14) & mask; - output[30 + outputOffset] = (intValue5 >>> 20) & mask; - output[31 + outputOffset] = intValue5 >>> 26; - outputOffset += 32; - } - } - public static void decode7(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 7; - final int mask = (int) ((1L<>> 7) & mask; - output[2 + outputOffset] = (intValue0 >>> 14) & mask; - output[3 + outputOffset] = (intValue0 >>> 21) & mask; - output[4 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; - output[5 + outputOffset] = (intValue1 >>> 3) & mask; - output[6 + outputOffset] = (intValue1 >>> 10) & mask; - output[7 + outputOffset] = (intValue1 >>> 17) & mask; - output[8 + outputOffset] = (intValue1 >>> 24) & mask; - output[9 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; - output[10 + outputOffset] = (intValue2 >>> 6) & mask; - output[11 + outputOffset] = (intValue2 >>> 13) & mask; - output[12 + outputOffset] = (intValue2 >>> 20) & mask; - output[13 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; - output[14 + outputOffset] = (intValue3 >>> 2) & mask; - output[15 + outputOffset] = (intValue3 >>> 9) & mask; - output[16 + outputOffset] = (intValue3 >>> 16) & mask; - output[17 + outputOffset] = (intValue3 >>> 23) & mask; - output[18 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[19 + outputOffset] = (intValue4 >>> 5) & mask; - output[20 + outputOffset] = (intValue4 >>> 12) & mask; - output[21 + outputOffset] = (intValue4 >>> 19) & mask; - output[22 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; - output[23 + outputOffset] = (intValue5 >>> 1) & mask; - output[24 + outputOffset] = (intValue5 >>> 8) & mask; - output[25 + outputOffset] = (intValue5 >>> 15) & mask; - output[26 + outputOffset] = (intValue5 >>> 22) & mask; - output[27 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; - output[28 + outputOffset] = (intValue6 >>> 4) & mask; - output[29 + outputOffset] = (intValue6 >>> 11) & mask; - output[30 + outputOffset] = (intValue6 >>> 18) & mask; - output[31 + outputOffset] = intValue6 >>> 25; - outputOffset += 32; - } - } - public static void decode8(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 8; - final int mask = (int) ((1L<>> 8) & mask; - output[2 + outputOffset] = (intValue0 >>> 16) & mask; - output[3 + outputOffset] = intValue0 >>> 24; - output[4 + outputOffset] = intValue1 & mask; - output[5 + outputOffset] = (intValue1 >>> 8) & mask; - output[6 + outputOffset] = (intValue1 >>> 16) & mask; - output[7 + outputOffset] = intValue1 >>> 24; - output[8 + outputOffset] = intValue2 & mask; - output[9 + outputOffset] = (intValue2 >>> 8) & mask; - output[10 + outputOffset] = (intValue2 >>> 16) & mask; - output[11 + outputOffset] = intValue2 >>> 24; - output[12 + outputOffset] = intValue3 & mask; - output[13 + outputOffset] = (intValue3 >>> 8) & mask; - output[14 + outputOffset] = (intValue3 >>> 16) & mask; - output[15 + outputOffset] = intValue3 >>> 24; - output[16 + outputOffset] = intValue4 & mask; - output[17 + outputOffset] = (intValue4 >>> 8) & mask; - output[18 + outputOffset] = (intValue4 >>> 16) & mask; - output[19 + outputOffset] = intValue4 >>> 24; - output[20 + outputOffset] = intValue5 & mask; - output[21 + outputOffset] = (intValue5 >>> 8) & mask; - output[22 + outputOffset] = (intValue5 >>> 16) & mask; - output[23 + outputOffset] = intValue5 >>> 24; - output[24 + outputOffset] = intValue6 & mask; - output[25 + outputOffset] = (intValue6 >>> 8) & mask; - output[26 + outputOffset] = (intValue6 >>> 16) & mask; - output[27 + outputOffset] = intValue6 >>> 24; - output[28 + outputOffset] = intValue7 & mask; - output[29 + outputOffset] = (intValue7 >>> 8) & mask; - output[30 + outputOffset] = (intValue7 >>> 16) & mask; - output[31 + outputOffset] = intValue7 >>> 24; - outputOffset += 32; - } - } - public static void decode9(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 9; - final int mask = (int) ((1L<>> 9) & mask; - output[2 + outputOffset] = (intValue0 >>> 18) & mask; - output[3 + outputOffset] = ((intValue0 >>> 27) | (intValue1 << 5)) & mask; - output[4 + outputOffset] = (intValue1 >>> 4) & mask; - output[5 + outputOffset] = (intValue1 >>> 13) & mask; - output[6 + outputOffset] = (intValue1 >>> 22) & mask; - output[7 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; - output[8 + outputOffset] = (intValue2 >>> 8) & mask; - output[9 + outputOffset] = (intValue2 >>> 17) & mask; - output[10 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; - output[11 + outputOffset] = (intValue3 >>> 3) & mask; - output[12 + outputOffset] = (intValue3 >>> 12) & mask; - output[13 + outputOffset] = (intValue3 >>> 21) & mask; - output[14 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[15 + outputOffset] = (intValue4 >>> 7) & mask; - output[16 + outputOffset] = (intValue4 >>> 16) & mask; - output[17 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; - output[18 + outputOffset] = (intValue5 >>> 2) & mask; - output[19 + outputOffset] = (intValue5 >>> 11) & mask; - output[20 + outputOffset] = (intValue5 >>> 20) & mask; - output[21 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; - output[22 + outputOffset] = (intValue6 >>> 6) & mask; - output[23 + outputOffset] = (intValue6 >>> 15) & mask; - output[24 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; - output[25 + outputOffset] = (intValue7 >>> 1) & mask; - output[26 + outputOffset] = (intValue7 >>> 10) & mask; - output[27 + outputOffset] = (intValue7 >>> 19) & mask; - output[28 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[29 + outputOffset] = (intValue8 >>> 5) & mask; - output[30 + outputOffset] = (intValue8 >>> 14) & mask; - output[31 + outputOffset] = intValue8 >>> 23; - outputOffset += 32; - } - } - public static void decode10(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 10; - final int mask = (int) ((1L<>> 10) & mask; - output[2 + outputOffset] = (intValue0 >>> 20) & mask; - output[3 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; - output[4 + outputOffset] = (intValue1 >>> 8) & mask; - output[5 + outputOffset] = (intValue1 >>> 18) & mask; - output[6 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[7 + outputOffset] = (intValue2 >>> 6) & mask; - output[8 + outputOffset] = (intValue2 >>> 16) & mask; - output[9 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; - output[10 + outputOffset] = (intValue3 >>> 4) & mask; - output[11 + outputOffset] = (intValue3 >>> 14) & mask; - output[12 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[13 + outputOffset] = (intValue4 >>> 2) & mask; - output[14 + outputOffset] = (intValue4 >>> 12) & mask; - output[15 + outputOffset] = intValue4 >>> 22; - output[16 + outputOffset] = intValue5 & mask; - output[17 + outputOffset] = (intValue5 >>> 10) & mask; - output[18 + outputOffset] = (intValue5 >>> 20) & mask; - output[19 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; - output[20 + outputOffset] = (intValue6 >>> 8) & mask; - output[21 + outputOffset] = (intValue6 >>> 18) & mask; - output[22 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; - output[23 + outputOffset] = (intValue7 >>> 6) & mask; - output[24 + outputOffset] = (intValue7 >>> 16) & mask; - output[25 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; - output[26 + outputOffset] = (intValue8 >>> 4) & mask; - output[27 + outputOffset] = (intValue8 >>> 14) & mask; - output[28 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; - output[29 + outputOffset] = (intValue9 >>> 2) & mask; - output[30 + outputOffset] = (intValue9 >>> 12) & mask; - output[31 + outputOffset] = intValue9 >>> 22; - outputOffset += 32; - } - } - public static void decode11(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 11; - final int mask = (int) ((1L<>> 11) & mask; - output[2 + outputOffset] = ((intValue0 >>> 22) | (intValue1 << 10)) & mask; - output[3 + outputOffset] = (intValue1 >>> 1) & mask; - output[4 + outputOffset] = (intValue1 >>> 12) & mask; - output[5 + outputOffset] = ((intValue1 >>> 23) | (intValue2 << 9)) & mask; - output[6 + outputOffset] = (intValue2 >>> 2) & mask; - output[7 + outputOffset] = (intValue2 >>> 13) & mask; - output[8 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; - output[9 + outputOffset] = (intValue3 >>> 3) & mask; - output[10 + outputOffset] = (intValue3 >>> 14) & mask; - output[11 + outputOffset] = ((intValue3 >>> 25) | (intValue4 << 7)) & mask; - output[12 + outputOffset] = (intValue4 >>> 4) & mask; - output[13 + outputOffset] = (intValue4 >>> 15) & mask; - output[14 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; - output[15 + outputOffset] = (intValue5 >>> 5) & mask; - output[16 + outputOffset] = (intValue5 >>> 16) & mask; - output[17 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; - output[18 + outputOffset] = (intValue6 >>> 6) & mask; - output[19 + outputOffset] = (intValue6 >>> 17) & mask; - output[20 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; - output[21 + outputOffset] = (intValue7 >>> 7) & mask; - output[22 + outputOffset] = (intValue7 >>> 18) & mask; - output[23 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; - output[24 + outputOffset] = (intValue8 >>> 8) & mask; - output[25 + outputOffset] = (intValue8 >>> 19) & mask; - output[26 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; - output[27 + outputOffset] = (intValue9 >>> 9) & mask; - output[28 + outputOffset] = (intValue9 >>> 20) & mask; - output[29 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; - output[30 + outputOffset] = (intValue10 >>> 10) & mask; - output[31 + outputOffset] = intValue10 >>> 21; - outputOffset += 32; - } - } - public static void decode12(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 12; - final int mask = (int) ((1L<>> 12) & mask; - output[2 + outputOffset] = ((intValue0 >>> 24) | (intValue1 << 8)) & mask; - output[3 + outputOffset] = (intValue1 >>> 4) & mask; - output[4 + outputOffset] = (intValue1 >>> 16) & mask; - output[5 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[6 + outputOffset] = (intValue2 >>> 8) & mask; - output[7 + outputOffset] = intValue2 >>> 20; - output[8 + outputOffset] = intValue3 & mask; - output[9 + outputOffset] = (intValue3 >>> 12) & mask; - output[10 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[11 + outputOffset] = (intValue4 >>> 4) & mask; - output[12 + outputOffset] = (intValue4 >>> 16) & mask; - output[13 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; - output[14 + outputOffset] = (intValue5 >>> 8) & mask; - output[15 + outputOffset] = intValue5 >>> 20; - output[16 + outputOffset] = intValue6 & mask; - output[17 + outputOffset] = (intValue6 >>> 12) & mask; - output[18 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; - output[19 + outputOffset] = (intValue7 >>> 4) & mask; - output[20 + outputOffset] = (intValue7 >>> 16) & mask; - output[21 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[22 + outputOffset] = (intValue8 >>> 8) & mask; - output[23 + outputOffset] = intValue8 >>> 20; - output[24 + outputOffset] = intValue9 & mask; - output[25 + outputOffset] = (intValue9 >>> 12) & mask; - output[26 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; - output[27 + outputOffset] = (intValue10 >>> 4) & mask; - output[28 + outputOffset] = (intValue10 >>> 16) & mask; - output[29 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; - output[30 + outputOffset] = (intValue11 >>> 8) & mask; - output[31 + outputOffset] = intValue11 >>> 20; - outputOffset += 32; - } - } - public static void decode13(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 13; - final int mask = (int) ((1L<>> 13) & mask; - output[2 + outputOffset] = ((intValue0 >>> 26) | (intValue1 << 6)) & mask; - output[3 + outputOffset] = (intValue1 >>> 7) & mask; - output[4 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; - output[5 + outputOffset] = (intValue2 >>> 1) & mask; - output[6 + outputOffset] = (intValue2 >>> 14) & mask; - output[7 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; - output[8 + outputOffset] = (intValue3 >>> 8) & mask; - output[9 + outputOffset] = ((intValue3 >>> 21) | (intValue4 << 11)) & mask; - output[10 + outputOffset] = (intValue4 >>> 2) & mask; - output[11 + outputOffset] = (intValue4 >>> 15) & mask; - output[12 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; - output[13 + outputOffset] = (intValue5 >>> 9) & mask; - output[14 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; - output[15 + outputOffset] = (intValue6 >>> 3) & mask; - output[16 + outputOffset] = (intValue6 >>> 16) & mask; - output[17 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; - output[18 + outputOffset] = (intValue7 >>> 10) & mask; - output[19 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; - output[20 + outputOffset] = (intValue8 >>> 4) & mask; - output[21 + outputOffset] = (intValue8 >>> 17) & mask; - output[22 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; - output[23 + outputOffset] = (intValue9 >>> 11) & mask; - output[24 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; - output[25 + outputOffset] = (intValue10 >>> 5) & mask; - output[26 + outputOffset] = (intValue10 >>> 18) & mask; - output[27 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; - output[28 + outputOffset] = (intValue11 >>> 12) & mask; - output[29 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; - output[30 + outputOffset] = (intValue12 >>> 6) & mask; - output[31 + outputOffset] = intValue12 >>> 19; - outputOffset += 32; - } - } - public static void decode14(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 14; - final int mask = (int) ((1L<>> 14) & mask; - output[2 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; - output[3 + outputOffset] = (intValue1 >>> 10) & mask; - output[4 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; - output[5 + outputOffset] = (intValue2 >>> 6) & mask; - output[6 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; - output[7 + outputOffset] = (intValue3 >>> 2) & mask; - output[8 + outputOffset] = (intValue3 >>> 16) & mask; - output[9 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[10 + outputOffset] = (intValue4 >>> 12) & mask; - output[11 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; - output[12 + outputOffset] = (intValue5 >>> 8) & mask; - output[13 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; - output[14 + outputOffset] = (intValue6 >>> 4) & mask; - output[15 + outputOffset] = intValue6 >>> 18; - output[16 + outputOffset] = intValue7 & mask; - output[17 + outputOffset] = (intValue7 >>> 14) & mask; - output[18 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[19 + outputOffset] = (intValue8 >>> 10) & mask; - output[20 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; - output[21 + outputOffset] = (intValue9 >>> 6) & mask; - output[22 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; - output[23 + outputOffset] = (intValue10 >>> 2) & mask; - output[24 + outputOffset] = (intValue10 >>> 16) & mask; - output[25 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; - output[26 + outputOffset] = (intValue11 >>> 12) & mask; - output[27 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; - output[28 + outputOffset] = (intValue12 >>> 8) & mask; - output[29 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; - output[30 + outputOffset] = (intValue13 >>> 4) & mask; - output[31 + outputOffset] = intValue13 >>> 18; - outputOffset += 32; - } - } - public static void decode15(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 15; - final int mask = (int) ((1L<>> 15) & mask; - output[2 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; - output[3 + outputOffset] = (intValue1 >>> 13) & mask; - output[4 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[5 + outputOffset] = (intValue2 >>> 11) & mask; - output[6 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; - output[7 + outputOffset] = (intValue3 >>> 9) & mask; - output[8 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[9 + outputOffset] = (intValue4 >>> 7) & mask; - output[10 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; - output[11 + outputOffset] = (intValue5 >>> 5) & mask; - output[12 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; - output[13 + outputOffset] = (intValue6 >>> 3) & mask; - output[14 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; - output[15 + outputOffset] = (intValue7 >>> 1) & mask; - output[16 + outputOffset] = (intValue7 >>> 16) & mask; - output[17 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; - output[18 + outputOffset] = (intValue8 >>> 14) & mask; - output[19 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; - output[20 + outputOffset] = (intValue9 >>> 12) & mask; - output[21 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; - output[22 + outputOffset] = (intValue10 >>> 10) & mask; - output[23 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; - output[24 + outputOffset] = (intValue11 >>> 8) & mask; - output[25 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; - output[26 + outputOffset] = (intValue12 >>> 6) & mask; - output[27 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; - output[28 + outputOffset] = (intValue13 >>> 4) & mask; - output[29 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; - output[30 + outputOffset] = (intValue14 >>> 2) & mask; - output[31 + outputOffset] = intValue14 >>> 17; - outputOffset += 32; - } - } - public static void decode16(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 16; - final int mask = (int) ((1L<>> 16; - output[2 + outputOffset] = intValue1 & mask; - output[3 + outputOffset] = intValue1 >>> 16; - output[4 + outputOffset] = intValue2 & mask; - output[5 + outputOffset] = intValue2 >>> 16; - output[6 + outputOffset] = intValue3 & mask; - output[7 + outputOffset] = intValue3 >>> 16; - output[8 + outputOffset] = intValue4 & mask; - output[9 + outputOffset] = intValue4 >>> 16; - output[10 + outputOffset] = intValue5 & mask; - output[11 + outputOffset] = intValue5 >>> 16; - output[12 + outputOffset] = intValue6 & mask; - output[13 + outputOffset] = intValue6 >>> 16; - output[14 + outputOffset] = intValue7 & mask; - output[15 + outputOffset] = intValue7 >>> 16; - output[16 + outputOffset] = intValue8 & mask; - output[17 + outputOffset] = intValue8 >>> 16; - output[18 + outputOffset] = intValue9 & mask; - output[19 + outputOffset] = intValue9 >>> 16; - output[20 + outputOffset] = intValue10 & mask; - output[21 + outputOffset] = intValue10 >>> 16; - output[22 + outputOffset] = intValue11 & mask; - output[23 + outputOffset] = intValue11 >>> 16; - output[24 + outputOffset] = intValue12 & mask; - output[25 + outputOffset] = intValue12 >>> 16; - output[26 + outputOffset] = intValue13 & mask; - output[27 + outputOffset] = intValue13 >>> 16; - output[28 + outputOffset] = intValue14 & mask; - output[29 + outputOffset] = intValue14 >>> 16; - output[30 + outputOffset] = intValue15 & mask; - output[31 + outputOffset] = intValue15 >>> 16; - outputOffset += 32; - } - } - public static void decode17(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 17; - final int mask = (int) ((1L<>> 17) | (intValue1 << 15)) & mask; - output[2 + outputOffset] = (intValue1 >>> 2) & mask; - output[3 + outputOffset] = ((intValue1 >>> 19) | (intValue2 << 13)) & mask; - output[4 + outputOffset] = (intValue2 >>> 4) & mask; - output[5 + outputOffset] = ((intValue2 >>> 21) | (intValue3 << 11)) & mask; - output[6 + outputOffset] = (intValue3 >>> 6) & mask; - output[7 + outputOffset] = ((intValue3 >>> 23) | (intValue4 << 9)) & mask; - output[8 + outputOffset] = (intValue4 >>> 8) & mask; - output[9 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; - output[10 + outputOffset] = (intValue5 >>> 10) & mask; - output[11 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; - output[12 + outputOffset] = (intValue6 >>> 12) & mask; - output[13 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; - output[14 + outputOffset] = (intValue7 >>> 14) & mask; - output[15 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; - output[16 + outputOffset] = ((intValue8 >>> 16) | (intValue9 << 16)) & mask; - output[17 + outputOffset] = (intValue9 >>> 1) & mask; - output[18 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; - output[19 + outputOffset] = (intValue10 >>> 3) & mask; - output[20 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; - output[21 + outputOffset] = (intValue11 >>> 5) & mask; - output[22 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; - output[23 + outputOffset] = (intValue12 >>> 7) & mask; - output[24 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; - output[25 + outputOffset] = (intValue13 >>> 9) & mask; - output[26 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; - output[27 + outputOffset] = (intValue14 >>> 11) & mask; - output[28 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; - output[29 + outputOffset] = (intValue15 >>> 13) & mask; - output[30 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; - output[31 + outputOffset] = intValue16 >>> 15; - outputOffset += 32; - } - } - public static void decode18(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 18; - final int mask = (int) ((1L<>> 18) | (intValue1 << 14)) & mask; - output[2 + outputOffset] = (intValue1 >>> 4) & mask; - output[3 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; - output[4 + outputOffset] = (intValue2 >>> 8) & mask; - output[5 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; - output[6 + outputOffset] = (intValue3 >>> 12) & mask; - output[7 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[8 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; - output[9 + outputOffset] = (intValue5 >>> 2) & mask; - output[10 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; - output[11 + outputOffset] = (intValue6 >>> 6) & mask; - output[12 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; - output[13 + outputOffset] = (intValue7 >>> 10) & mask; - output[14 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[15 + outputOffset] = intValue8 >>> 14; - output[16 + outputOffset] = intValue9 & mask; - output[17 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; - output[18 + outputOffset] = (intValue10 >>> 4) & mask; - output[19 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; - output[20 + outputOffset] = (intValue11 >>> 8) & mask; - output[21 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; - output[22 + outputOffset] = (intValue12 >>> 12) & mask; - output[23 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; - output[24 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; - output[25 + outputOffset] = (intValue14 >>> 2) & mask; - output[26 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; - output[27 + outputOffset] = (intValue15 >>> 6) & mask; - output[28 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; - output[29 + outputOffset] = (intValue16 >>> 10) & mask; - output[30 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; - output[31 + outputOffset] = intValue17 >>> 14; - outputOffset += 32; - } - } - public static void decode19(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 19; - final int mask = (int) ((1L<>> 19) | (intValue1 << 13)) & mask; - output[2 + outputOffset] = (intValue1 >>> 6) & mask; - output[3 + outputOffset] = ((intValue1 >>> 25) | (intValue2 << 7)) & mask; - output[4 + outputOffset] = (intValue2 >>> 12) & mask; - output[5 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; - output[6 + outputOffset] = ((intValue3 >>> 18) | (intValue4 << 14)) & mask; - output[7 + outputOffset] = (intValue4 >>> 5) & mask; - output[8 + outputOffset] = ((intValue4 >>> 24) | (intValue5 << 8)) & mask; - output[9 + outputOffset] = (intValue5 >>> 11) & mask; - output[10 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; - output[11 + outputOffset] = ((intValue6 >>> 17) | (intValue7 << 15)) & mask; - output[12 + outputOffset] = (intValue7 >>> 4) & mask; - output[13 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; - output[14 + outputOffset] = (intValue8 >>> 10) & mask; - output[15 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; - output[16 + outputOffset] = ((intValue9 >>> 16) | (intValue10 << 16)) & mask; - output[17 + outputOffset] = (intValue10 >>> 3) & mask; - output[18 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; - output[19 + outputOffset] = (intValue11 >>> 9) & mask; - output[20 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; - output[21 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; - output[22 + outputOffset] = (intValue13 >>> 2) & mask; - output[23 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; - output[24 + outputOffset] = (intValue14 >>> 8) & mask; - output[25 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; - output[26 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; - output[27 + outputOffset] = (intValue16 >>> 1) & mask; - output[28 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; - output[29 + outputOffset] = (intValue17 >>> 7) & mask; - output[30 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; - output[31 + outputOffset] = intValue18 >>> 13; - outputOffset += 32; - } - } - public static void decode20(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 20; - final int mask = (int) ((1L<>> 20) | (intValue1 << 12)) & mask; - output[2 + outputOffset] = (intValue1 >>> 8) & mask; - output[3 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[4 + outputOffset] = ((intValue2 >>> 16) | (intValue3 << 16)) & mask; - output[5 + outputOffset] = (intValue3 >>> 4) & mask; - output[6 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[7 + outputOffset] = intValue4 >>> 12; - output[8 + outputOffset] = intValue5 & mask; - output[9 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; - output[10 + outputOffset] = (intValue6 >>> 8) & mask; - output[11 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; - output[12 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; - output[13 + outputOffset] = (intValue8 >>> 4) & mask; - output[14 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; - output[15 + outputOffset] = intValue9 >>> 12; - output[16 + outputOffset] = intValue10 & mask; - output[17 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; - output[18 + outputOffset] = (intValue11 >>> 8) & mask; - output[19 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; - output[20 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; - output[21 + outputOffset] = (intValue13 >>> 4) & mask; - output[22 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; - output[23 + outputOffset] = intValue14 >>> 12; - output[24 + outputOffset] = intValue15 & mask; - output[25 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; - output[26 + outputOffset] = (intValue16 >>> 8) & mask; - output[27 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; - output[28 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; - output[29 + outputOffset] = (intValue18 >>> 4) & mask; - output[30 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; - output[31 + outputOffset] = intValue19 >>> 12; - outputOffset += 32; - } - } - public static void decode21(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 21; - final int mask = (int) ((1L<>> 21) | (intValue1 << 11)) & mask; - output[2 + outputOffset] = (intValue1 >>> 10) & mask; - output[3 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; - output[4 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; - output[5 + outputOffset] = (intValue3 >>> 9) & mask; - output[6 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; - output[7 + outputOffset] = ((intValue4 >>> 19) | (intValue5 << 13)) & mask; - output[8 + outputOffset] = (intValue5 >>> 8) & mask; - output[9 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; - output[10 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; - output[11 + outputOffset] = (intValue7 >>> 7) & mask; - output[12 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[13 + outputOffset] = ((intValue8 >>> 17) | (intValue9 << 15)) & mask; - output[14 + outputOffset] = (intValue9 >>> 6) & mask; - output[15 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; - output[16 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; - output[17 + outputOffset] = (intValue11 >>> 5) & mask; - output[18 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; - output[19 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; - output[20 + outputOffset] = (intValue13 >>> 4) & mask; - output[21 + outputOffset] = ((intValue13 >>> 25) | (intValue14 << 7)) & mask; - output[22 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; - output[23 + outputOffset] = (intValue15 >>> 3) & mask; - output[24 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; - output[25 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; - output[26 + outputOffset] = (intValue17 >>> 2) & mask; - output[27 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; - output[28 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; - output[29 + outputOffset] = (intValue19 >>> 1) & mask; - output[30 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; - output[31 + outputOffset] = intValue20 >>> 11; - outputOffset += 32; - } - } - public static void decode22(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 22; - final int mask = (int) ((1L<>> 22) | (intValue1 << 10)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 12) | (intValue2 << 20)) & mask; - output[3 + outputOffset] = (intValue2 >>> 2) & mask; - output[4 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; - output[5 + outputOffset] = ((intValue3 >>> 14) | (intValue4 << 18)) & mask; - output[6 + outputOffset] = (intValue4 >>> 4) & mask; - output[7 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; - output[8 + outputOffset] = ((intValue5 >>> 16) | (intValue6 << 16)) & mask; - output[9 + outputOffset] = (intValue6 >>> 6) & mask; - output[10 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; - output[11 + outputOffset] = ((intValue7 >>> 18) | (intValue8 << 14)) & mask; - output[12 + outputOffset] = (intValue8 >>> 8) & mask; - output[13 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; - output[14 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; - output[15 + outputOffset] = intValue10 >>> 10; - output[16 + outputOffset] = intValue11 & mask; - output[17 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; - output[18 + outputOffset] = ((intValue12 >>> 12) | (intValue13 << 20)) & mask; - output[19 + outputOffset] = (intValue13 >>> 2) & mask; - output[20 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; - output[21 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; - output[22 + outputOffset] = (intValue15 >>> 4) & mask; - output[23 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; - output[24 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; - output[25 + outputOffset] = (intValue17 >>> 6) & mask; - output[26 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; - output[27 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; - output[28 + outputOffset] = (intValue19 >>> 8) & mask; - output[29 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; - output[30 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; - output[31 + outputOffset] = intValue21 >>> 10; - outputOffset += 32; - } - } - public static void decode23(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 23; - final int mask = (int) ((1L<>> 23) | (intValue1 << 9)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 14) | (intValue2 << 18)) & mask; - output[3 + outputOffset] = (intValue2 >>> 5) & mask; - output[4 + outputOffset] = ((intValue2 >>> 28) | (intValue3 << 4)) & mask; - output[5 + outputOffset] = ((intValue3 >>> 19) | (intValue4 << 13)) & mask; - output[6 + outputOffset] = ((intValue4 >>> 10) | (intValue5 << 22)) & mask; - output[7 + outputOffset] = (intValue5 >>> 1) & mask; - output[8 + outputOffset] = ((intValue5 >>> 24) | (intValue6 << 8)) & mask; - output[9 + outputOffset] = ((intValue6 >>> 15) | (intValue7 << 17)) & mask; - output[10 + outputOffset] = (intValue7 >>> 6) & mask; - output[11 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; - output[12 + outputOffset] = ((intValue8 >>> 20) | (intValue9 << 12)) & mask; - output[13 + outputOffset] = ((intValue9 >>> 11) | (intValue10 << 21)) & mask; - output[14 + outputOffset] = (intValue10 >>> 2) & mask; - output[15 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; - output[16 + outputOffset] = ((intValue11 >>> 16) | (intValue12 << 16)) & mask; - output[17 + outputOffset] = (intValue12 >>> 7) & mask; - output[18 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; - output[19 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; - output[20 + outputOffset] = ((intValue14 >>> 12) | (intValue15 << 20)) & mask; - output[21 + outputOffset] = (intValue15 >>> 3) & mask; - output[22 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; - output[23 + outputOffset] = ((intValue16 >>> 17) | (intValue17 << 15)) & mask; - output[24 + outputOffset] = (intValue17 >>> 8) & mask; - output[25 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; - output[26 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; - output[27 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; - output[28 + outputOffset] = (intValue20 >>> 4) & mask; - output[29 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; - output[30 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; - output[31 + outputOffset] = intValue22 >>> 9; - outputOffset += 32; - } - } - public static void decode24(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 24; - final int mask = (int) ((1L<>> 24) | (intValue1 << 8)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 16) | (intValue2 << 16)) & mask; - output[3 + outputOffset] = intValue2 >>> 8; - output[4 + outputOffset] = intValue3 & mask; - output[5 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[6 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; - output[7 + outputOffset] = intValue5 >>> 8; - output[8 + outputOffset] = intValue6 & mask; - output[9 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; - output[10 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; - output[11 + outputOffset] = intValue8 >>> 8; - output[12 + outputOffset] = intValue9 & mask; - output[13 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; - output[14 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; - output[15 + outputOffset] = intValue11 >>> 8; - output[16 + outputOffset] = intValue12 & mask; - output[17 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; - output[18 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; - output[19 + outputOffset] = intValue14 >>> 8; - output[20 + outputOffset] = intValue15 & mask; - output[21 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; - output[22 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; - output[23 + outputOffset] = intValue17 >>> 8; - output[24 + outputOffset] = intValue18 & mask; - output[25 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; - output[26 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; - output[27 + outputOffset] = intValue20 >>> 8; - output[28 + outputOffset] = intValue21 & mask; - output[29 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; - output[30 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; - output[31 + outputOffset] = intValue23 >>> 8; - outputOffset += 32; - } - } - public static void decode25(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 25; - final int mask = (int) ((1L<>> 25) | (intValue1 << 7)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 18) | (intValue2 << 14)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 11) | (intValue3 << 21)) & mask; - output[4 + outputOffset] = (intValue3 >>> 4) & mask; - output[5 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; - output[6 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; - output[7 + outputOffset] = ((intValue5 >>> 15) | (intValue6 << 17)) & mask; - output[8 + outputOffset] = ((intValue6 >>> 8) | (intValue7 << 24)) & mask; - output[9 + outputOffset] = (intValue7 >>> 1) & mask; - output[10 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; - output[11 + outputOffset] = ((intValue8 >>> 19) | (intValue9 << 13)) & mask; - output[12 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; - output[13 + outputOffset] = (intValue10 >>> 5) & mask; - output[14 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; - output[15 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; - output[16 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; - output[17 + outputOffset] = ((intValue13 >>> 9) | (intValue14 << 23)) & mask; - output[18 + outputOffset] = (intValue14 >>> 2) & mask; - output[19 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; - output[20 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; - output[21 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; - output[22 + outputOffset] = (intValue17 >>> 6) & mask; - output[23 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; - output[24 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; - output[25 + outputOffset] = ((intValue19 >>> 17) | (intValue20 << 15)) & mask; - output[26 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; - output[27 + outputOffset] = (intValue21 >>> 3) & mask; - output[28 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; - output[29 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; - output[30 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; - output[31 + outputOffset] = intValue24 >>> 7; - outputOffset += 32; - } - } - public static void decode26(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 26; - final int mask = (int) ((1L<>> 26) | (intValue1 << 6)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 14) | (intValue3 << 18)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 8) | (intValue4 << 24)) & mask; - output[5 + outputOffset] = (intValue4 >>> 2) & mask; - output[6 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; - output[7 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; - output[8 + outputOffset] = ((intValue6 >>> 16) | (intValue7 << 16)) & mask; - output[9 + outputOffset] = ((intValue7 >>> 10) | (intValue8 << 22)) & mask; - output[10 + outputOffset] = (intValue8 >>> 4) & mask; - output[11 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; - output[12 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; - output[13 + outputOffset] = ((intValue10 >>> 18) | (intValue11 << 14)) & mask; - output[14 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; - output[15 + outputOffset] = intValue12 >>> 6; - output[16 + outputOffset] = intValue13 & mask; - output[17 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; - output[18 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; - output[19 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; - output[20 + outputOffset] = ((intValue16 >>> 8) | (intValue17 << 24)) & mask; - output[21 + outputOffset] = (intValue17 >>> 2) & mask; - output[22 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; - output[23 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; - output[24 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; - output[25 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; - output[26 + outputOffset] = (intValue21 >>> 4) & mask; - output[27 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; - output[28 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; - output[29 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; - output[30 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; - output[31 + outputOffset] = intValue25 >>> 6; - outputOffset += 32; - } - } - public static void decode27(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 27; - final int mask = (int) ((1L<>> 27) | (intValue1 << 5)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 17) | (intValue3 << 15)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 12) | (intValue4 << 20)) & mask; - output[5 + outputOffset] = ((intValue4 >>> 7) | (intValue5 << 25)) & mask; - output[6 + outputOffset] = (intValue5 >>> 2) & mask; - output[7 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; - output[8 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; - output[9 + outputOffset] = ((intValue7 >>> 19) | (intValue8 << 13)) & mask; - output[10 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; - output[11 + outputOffset] = ((intValue9 >>> 9) | (intValue10 << 23)) & mask; - output[12 + outputOffset] = (intValue10 >>> 4) & mask; - output[13 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; - output[14 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; - output[15 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; - output[16 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; - output[17 + outputOffset] = ((intValue14 >>> 11) | (intValue15 << 21)) & mask; - output[18 + outputOffset] = ((intValue15 >>> 6) | (intValue16 << 26)) & mask; - output[19 + outputOffset] = (intValue16 >>> 1) & mask; - output[20 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; - output[21 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; - output[22 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; - output[23 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; - output[24 + outputOffset] = ((intValue20 >>> 8) | (intValue21 << 24)) & mask; - output[25 + outputOffset] = (intValue21 >>> 3) & mask; - output[26 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; - output[27 + outputOffset] = ((intValue22 >>> 25) | (intValue23 << 7)) & mask; - output[28 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; - output[29 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; - output[30 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; - output[31 + outputOffset] = intValue26 >>> 5; - outputOffset += 32; - } - } - public static void decode28(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 28; - final int mask = (int) ((1L<>> 28) | (intValue1 << 4)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 16) | (intValue4 << 16)) & mask; - output[5 + outputOffset] = ((intValue4 >>> 12) | (intValue5 << 20)) & mask; - output[6 + outputOffset] = ((intValue5 >>> 8) | (intValue6 << 24)) & mask; - output[7 + outputOffset] = intValue6 >>> 4; - output[8 + outputOffset] = intValue7 & mask; - output[9 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; - output[10 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; - output[11 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; - output[12 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; - output[13 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; - output[14 + outputOffset] = ((intValue12 >>> 8) | (intValue13 << 24)) & mask; - output[15 + outputOffset] = intValue13 >>> 4; - output[16 + outputOffset] = intValue14 & mask; - output[17 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; - output[18 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; - output[19 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; - output[20 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; - output[21 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; - output[22 + outputOffset] = ((intValue19 >>> 8) | (intValue20 << 24)) & mask; - output[23 + outputOffset] = intValue20 >>> 4; - output[24 + outputOffset] = intValue21 & mask; - output[25 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; - output[26 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; - output[27 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; - output[28 + outputOffset] = ((intValue24 >>> 16) | (intValue25 << 16)) & mask; - output[29 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; - output[30 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; - output[31 + outputOffset] = intValue27 >>> 4; - outputOffset += 32; - } - } - public static void decode29(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 29; - final int mask = (int) ((1L<>> 29) | (intValue1 << 3)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 26) | (intValue2 << 6)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 23) | (intValue3 << 9)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 20) | (intValue4 << 12)) & mask; - output[5 + outputOffset] = ((intValue4 >>> 17) | (intValue5 << 15)) & mask; - output[6 + outputOffset] = ((intValue5 >>> 14) | (intValue6 << 18)) & mask; - output[7 + outputOffset] = ((intValue6 >>> 11) | (intValue7 << 21)) & mask; - output[8 + outputOffset] = ((intValue7 >>> 8) | (intValue8 << 24)) & mask; - output[9 + outputOffset] = ((intValue8 >>> 5) | (intValue9 << 27)) & mask; - output[10 + outputOffset] = (intValue9 >>> 2) & mask; - output[11 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; - output[12 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; - output[13 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; - output[14 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; - output[15 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; - output[16 + outputOffset] = ((intValue14 >>> 16) | (intValue15 << 16)) & mask; - output[17 + outputOffset] = ((intValue15 >>> 13) | (intValue16 << 19)) & mask; - output[18 + outputOffset] = ((intValue16 >>> 10) | (intValue17 << 22)) & mask; - output[19 + outputOffset] = ((intValue17 >>> 7) | (intValue18 << 25)) & mask; - output[20 + outputOffset] = ((intValue18 >>> 4) | (intValue19 << 28)) & mask; - output[21 + outputOffset] = (intValue19 >>> 1) & mask; - output[22 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; - output[23 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; - output[24 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; - output[25 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; - output[26 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; - output[27 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; - output[28 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; - output[29 + outputOffset] = ((intValue26 >>> 9) | (intValue27 << 23)) & mask; - output[30 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; - output[31 + outputOffset] = intValue28 >>> 3; - outputOffset += 32; - } - } - public static void decode30(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 30; - final int mask = (int) ((1L<>> 30) | (intValue1 << 2)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; - output[5 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; - output[6 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; - output[7 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; - output[8 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; - output[9 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; - output[10 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; - output[11 + outputOffset] = ((intValue10 >>> 10) | (intValue11 << 22)) & mask; - output[12 + outputOffset] = ((intValue11 >>> 8) | (intValue12 << 24)) & mask; - output[13 + outputOffset] = ((intValue12 >>> 6) | (intValue13 << 26)) & mask; - output[14 + outputOffset] = ((intValue13 >>> 4) | (intValue14 << 28)) & mask; - output[15 + outputOffset] = intValue14 >>> 2; - output[16 + outputOffset] = intValue15 & mask; - output[17 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; - output[18 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; - output[19 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; - output[20 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; - output[21 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; - output[22 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; - output[23 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; - output[24 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; - output[25 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; - output[26 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; - output[27 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; - output[28 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; - output[29 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; - output[30 + outputOffset] = ((intValue28 >>> 4) | (intValue29 << 28)) & mask; - output[31 + outputOffset] = intValue29 >>> 2; - outputOffset += 32; - } - } - public static void decode31(final IntBuffer compressedBuffer, final int[] output) { - final int numFrameBits = 31; - final int mask = (int) ((1L<>> 31) | (intValue1 << 1)) & mask; - output[2 + outputOffset] = ((intValue1 >>> 30) | (intValue2 << 2)) & mask; - output[3 + outputOffset] = ((intValue2 >>> 29) | (intValue3 << 3)) & mask; - output[4 + outputOffset] = ((intValue3 >>> 28) | (intValue4 << 4)) & mask; - output[5 + outputOffset] = ((intValue4 >>> 27) | (intValue5 << 5)) & mask; - output[6 + outputOffset] = ((intValue5 >>> 26) | (intValue6 << 6)) & mask; - output[7 + outputOffset] = ((intValue6 >>> 25) | (intValue7 << 7)) & mask; - output[8 + outputOffset] = ((intValue7 >>> 24) | (intValue8 << 8)) & mask; - output[9 + outputOffset] = ((intValue8 >>> 23) | (intValue9 << 9)) & mask; - output[10 + outputOffset] = ((intValue9 >>> 22) | (intValue10 << 10)) & mask; - output[11 + outputOffset] = ((intValue10 >>> 21) | (intValue11 << 11)) & mask; - output[12 + outputOffset] = ((intValue11 >>> 20) | (intValue12 << 12)) & mask; - output[13 + outputOffset] = ((intValue12 >>> 19) | (intValue13 << 13)) & mask; - output[14 + outputOffset] = ((intValue13 >>> 18) | (intValue14 << 14)) & mask; - output[15 + outputOffset] = ((intValue14 >>> 17) | (intValue15 << 15)) & mask; - output[16 + outputOffset] = ((intValue15 >>> 16) | (intValue16 << 16)) & mask; - output[17 + outputOffset] = ((intValue16 >>> 15) | (intValue17 << 17)) & mask; - output[18 + outputOffset] = ((intValue17 >>> 14) | (intValue18 << 18)) & mask; - output[19 + outputOffset] = ((intValue18 >>> 13) | (intValue19 << 19)) & mask; - output[20 + outputOffset] = ((intValue19 >>> 12) | (intValue20 << 20)) & mask; - output[21 + outputOffset] = ((intValue20 >>> 11) | (intValue21 << 21)) & mask; - output[22 + outputOffset] = ((intValue21 >>> 10) | (intValue22 << 22)) & mask; - output[23 + outputOffset] = ((intValue22 >>> 9) | (intValue23 << 23)) & mask; - output[24 + outputOffset] = ((intValue23 >>> 8) | (intValue24 << 24)) & mask; - output[25 + outputOffset] = ((intValue24 >>> 7) | (intValue25 << 25)) & mask; - output[26 + outputOffset] = ((intValue25 >>> 6) | (intValue26 << 26)) & mask; - output[27 + outputOffset] = ((intValue26 >>> 5) | (intValue27 << 27)) & mask; - output[28 + outputOffset] = ((intValue27 >>> 4) | (intValue28 << 28)) & mask; - output[29 + outputOffset] = ((intValue28 >>> 3) | (intValue29 << 29)) & mask; - output[30 + outputOffset] = ((intValue29 >>> 2) | (intValue30 << 30)) & mask; - output[31 + outputOffset] = intValue30 >>> 1; - outputOffset += 32; - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/block/gendecompress.py b/lucene/core/src/java/org/apache/lucene/codecs/block/gendecompress.py deleted file mode 100755 index 7a77aa4d0cb..00000000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/block/gendecompress.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python2 -""" - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -""" -Generate source code for java classes for For or PFor decompression. -""" - -def bitsExpr(i, numFrameBits): - framePos = i * numFrameBits - intValNum = (framePos / 32) - bitPos = framePos % 32 - bitsInInt = "intValue" + str(intValNum) - needBrackets = 0 - if bitPos > 0: - bitsInInt += " >>> " + str(bitPos) - needBrackets = 1 - if bitPos + numFrameBits > 32: - if needBrackets: - bitsInInt = "(" + bitsInInt + ")" - bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")" - needBrackets = 1 - if bitPos + numFrameBits != 32: - if needBrackets: - bitsInInt = "(" + bitsInInt + ")" - bitsInInt += " & mask" - return bitsInInt - - -def genDecompress(): - className = "PackedIntsDecompress" - fileName = className + ".java" - imports = "import java.nio.IntBuffer;\n" - f = open(fileName, 'w') - w = f.write - try: - w("package org.apache.lucene.codecs.block;\n") - w("""/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - """) - - w("\n/* This code is generated, do not modify. See gendecompress.py */\n\n") - - w("import java.nio.IntBuffer;\n") - w("import java.util.Arrays;\n\n") - - w("final class PackedIntsDecompress {\n") - - w('\n // nocommit: assess perf of this to see if specializing is really needed\n') - w('\n // NOTE: hardwired to blockSize == 128\n\n') - - w(' public static void decode0(final IntBuffer compressedBuffer, final int[] output) {\n') - w(' Arrays.fill(output, compressedBuffer.get());\n') - w(' }\n') - - for numFrameBits in xrange(1, 32): - w(' public static void decode%d(final IntBuffer compressedBuffer, final int[] output) {\n' % numFrameBits) - w(' final int numFrameBits = %d;\n' % numFrameBits) - w(' final int mask = (int) ((1L< 1; - this.maxTermBlockSize = maxTermBlockSize; - assert minTermBlockSize <= maxTermBlockSize; - } - - @Override - public String toString() { - return getName() + "(blocksize=" + BLOCK_SIZE + ")"; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase postingsWriter = new BlockPackedPostingsWriter(state); - - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, - postingsWriter, - minTermBlockSize, - maxTermBlockSize); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsWriter); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postingsReader = new BlockPackedPostingsReader(state.dir, - state.fieldInfos, - state.segmentInfo, - state.context, - state.segmentSuffix); - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - state.segmentSuffix, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsReader); - } - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java deleted file mode 100644 index e5493a37e87..00000000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java +++ /dev/null @@ -1,1549 +0,0 @@ -package org.apache.lucene.codecs.blockpacked; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat.BLOCK_SIZE; -import static org.apache.lucene.codecs.blockpacked.ForUtil.MAX_DATA_SIZE; -import static org.apache.lucene.codecs.blockpacked.ForUtil.MAX_ENCODED_SIZE; - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.codecs.BlockTermState; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.PostingsReaderBase; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.TermState; -import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** - * Concrete class that reads docId(maybe frq,pos,offset,payloads) list - * with postings format. - * - * @see BlockPackedSkipReader for details - * - */ -public final class BlockPackedPostingsReader extends PostingsReaderBase { - - private final IndexInput docIn; - private final IndexInput posIn; - private final IndexInput payIn; - - private final ForUtil forUtil; - - public static boolean DEBUG = false; - - // nocommit - final String segment; - - public BlockPackedPostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException { - boolean success = false; - segment = segmentInfo.name; - IndexInput docIn = null; - IndexInput posIn = null; - IndexInput payIn = null; - try { - docIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockPackedPostingsFormat.DOC_EXTENSION), - ioContext); - CodecUtil.checkHeader(docIn, - BlockPackedPostingsWriter.DOC_CODEC, - BlockPackedPostingsWriter.VERSION_START, - BlockPackedPostingsWriter.VERSION_START); - forUtil = new ForUtil(docIn); - - if (fieldInfos.hasProx()) { - posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockPackedPostingsFormat.POS_EXTENSION), - ioContext); - CodecUtil.checkHeader(posIn, - BlockPackedPostingsWriter.POS_CODEC, - BlockPackedPostingsWriter.VERSION_START, - BlockPackedPostingsWriter.VERSION_START); - - if (fieldInfos.hasPayloads() || fieldInfos.hasOffsets()) { - payIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockPackedPostingsFormat.PAY_EXTENSION), - ioContext); - CodecUtil.checkHeader(payIn, - BlockPackedPostingsWriter.PAY_CODEC, - BlockPackedPostingsWriter.VERSION_START, - BlockPackedPostingsWriter.VERSION_START); - } - } - - this.docIn = docIn; - this.posIn = posIn; - this.payIn = payIn; - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(docIn, posIn, payIn); - } - } - } - - @Override - public void init(IndexInput termsIn) throws IOException { - // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, - BlockPackedPostingsWriter.TERMS_CODEC, - BlockPackedPostingsWriter.VERSION_START, - BlockPackedPostingsWriter.VERSION_START); - final int indexBlockSize = termsIn.readVInt(); - if (indexBlockSize != BLOCK_SIZE) { - throw new IllegalStateException("index-time blockSize (" + indexBlockSize + ") != read-time blockSize (" + BLOCK_SIZE + ")"); - } - } - - /** - * Read values that have been written using variable-length encoding instead of bit-packing. - */ - private static void readVIntBlock(IndexInput docIn, int[] docBuffer, - int[] freqBuffer, int num, boolean indexHasFreq) throws IOException { - if (indexHasFreq) { - for(int i=0;i>> 1; - if ((code & 1) != 0) { - freqBuffer[i] = 1; - } else { - freqBuffer[i] = docIn.readVInt(); - } - } - } else { - for(int i=0;i docUpto) { - // Skipper moved - if (DEBUG) { - System.out.println("skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer()); - } - assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto; - docUpto = newDocUpto; - - // Force to read next block - docBufferUpto = BLOCK_SIZE; - accum = skipper.getDoc(); // actually, this is just lastSkipEntry - docIn.seek(skipper.getDocPointer()); // now point to the block we want to search - } - nextSkipDoc = skipper.getNextSkipDoc(); - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - if (docBufferUpto == BLOCK_SIZE) { - refillDocs(); - } - - // Now scan... this is an inlined/pared down version - // of nextDoc(): - while (true) { - if (DEBUG) { - System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); - } - accum += docDeltaBuffer[docBufferUpto]; - docUpto++; - - if (accum >= target) { - break; - } - docBufferUpto++; - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - } - - if (liveDocs == null || liveDocs.get(accum)) { - if (DEBUG) { - System.out.println(" return doc=" + accum); - } - freq = freqBuffer[docBufferUpto]; - docBufferUpto++; - return doc = accum; - } else { - if (DEBUG) { - System.out.println(" now do nextDoc()"); - } - docBufferUpto++; - return nextDoc(); - } - } - } - - - final class BlockDocsAndPositionsEnum extends DocsAndPositionsEnum { - - private final byte[] encoded; - - private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE]; - private final int[] freqBuffer = new int[MAX_DATA_SIZE]; - private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE]; - - private int docBufferUpto; - private int posBufferUpto; - - private BlockPackedSkipReader skipper; - private boolean skipped; - - final IndexInput startDocIn; - - final IndexInput docIn; - final IndexInput posIn; - - final boolean indexHasOffsets; - final boolean indexHasPayloads; - - private int docFreq; // number of docs in this posting list - private int docUpto; // how many docs we've read - private int doc; // doc we last read - private int accum; // accumulator for doc deltas - private int freq; // freq we last read - private int position; // current position - - // how many positions "behind" we are; nextPosition must - // skip these to "catch up": - private int posPendingCount; - - // Lazy pos seek: if != -1 then we must seek to this FP - // before reading positions: - private long posPendingFP; - - // Where this term's postings start in the .doc file: - private long docTermStartFP; - - // Where this term's postings start in the .pos file: - private long posTermStartFP; - - // Where this term's payloads/offsets start in the .pay - // file: - private long payTermStartFP; - - // File pointer where the last (vInt encoded) pos delta - // block is. We need this to know whether to bulk - // decode vs vInt decode the block: - private long lastPosBlockFP; - - // Where this term's skip data starts (after - // docTermStartFP) in the .doc file (or -1 if there is - // no skip data for this term): - private int skipOffset; - - private int nextSkipDoc; - - private Bits liveDocs; - - public BlockDocsAndPositionsEnum(FieldInfo fieldInfo) throws IOException { - this.startDocIn = BlockPackedPostingsReader.this.docIn; - this.docIn = (IndexInput) startDocIn.clone(); - this.posIn = (IndexInput) BlockPackedPostingsReader.this.posIn.clone(); - encoded = new byte[MAX_ENCODED_SIZE]; - indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - indexHasPayloads = fieldInfo.hasPayloads(); - } - - public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { - return docIn == startDocIn && - indexHasOffsets == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) && - indexHasPayloads == fieldInfo.hasPayloads(); - } - - public DocsAndPositionsEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException { - this.liveDocs = liveDocs; - if (DEBUG) { - System.out.println(" FPR.reset: termState=" + termState); - } - docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; - docIn.seek(docTermStartFP); - skipOffset = termState.skipOffset; - posPendingFP = posTermStartFP; - posPendingCount = 0; - if (termState.totalTermFreq < BLOCK_SIZE) { - lastPosBlockFP = posTermStartFP; - } else if (termState.totalTermFreq == BLOCK_SIZE) { - lastPosBlockFP = -1; - } else { - lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; - } - - doc = -1; - accum = 0; - docUpto = 0; - nextSkipDoc = BLOCK_SIZE - 1; - docBufferUpto = BLOCK_SIZE; - skipped = false; - return this; - } - - @Override - public int freq() throws IOException { - return freq; - } - - @Override - public int docID() { - return doc; - } - - private void refillDocs() throws IOException { - final int left = docFreq - docUpto; - assert left > 0; - - if (left >= BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); - } - forUtil.readBlock(docIn, encoded, docDeltaBuffer); - if (DEBUG) { - System.out.println(" fill freq block from fp=" + docIn.getFilePointer()); - } - forUtil.readBlock(docIn, encoded, freqBuffer); - } else { - // Read vInts: - if (DEBUG) { - System.out.println(" fill last vInt doc block from fp=" + docIn.getFilePointer()); - } - readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true); - } - docBufferUpto = 0; - } - - private void refillPositions() throws IOException { - if (DEBUG) { - System.out.println(" refillPositions"); - } - if (posIn.getFilePointer() == lastPosBlockFP) { - if (DEBUG) { - System.out.println(" vInt pos block @ fp=" + posIn.getFilePointer() + " hasPayloads=" + indexHasPayloads + " hasOffsets=" + indexHasOffsets); - } - final int count = posIn.readVInt(); - int payloadLength = 0; - for(int i=0;i>> 1; - if (payloadLength != 0) { - posIn.seek(posIn.getFilePointer() + payloadLength); - } - } else { - posDeltaBuffer[i] = code; - } - if (indexHasOffsets) { - posIn.readVInt(); - posIn.readVInt(); - } - } - } else { - if (DEBUG) { - System.out.println(" bulk pos block @ fp=" + posIn.getFilePointer()); - } - forUtil.readBlock(posIn, encoded, posDeltaBuffer); - } - } - - @Override - public int nextDoc() throws IOException { - if (DEBUG) { - System.out.println(" FPR.nextDoc"); - } - while (true) { - if (DEBUG) { - System.out.println(" docUpto=" + docUpto + " (of df=" + docFreq + ") docBufferUpto=" + docBufferUpto); - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - if (docBufferUpto == BLOCK_SIZE) { - refillDocs(); - } - if (DEBUG) { - System.out.println(" accum=" + accum + " docDeltaBuffer[" + docBufferUpto + "]=" + docDeltaBuffer[docBufferUpto]); - } - accum += docDeltaBuffer[docBufferUpto]; - freq = freqBuffer[docBufferUpto]; - posPendingCount += freq; - docBufferUpto++; - docUpto++; - - if (liveDocs == null || liveDocs.get(accum)) { - doc = accum; - position = 0; - if (DEBUG) { - System.out.println(" return doc=" + doc + " freq=" + freq + " posPendingCount=" + posPendingCount); - } - return doc; - } - if (DEBUG) { - System.out.println(" doc=" + accum + " is deleted; try next doc"); - } - } - } - - @Override - public int advance(int target) throws IOException { - // nocommit make frq block load lazy/skippable - if (DEBUG) { - System.out.println(" FPR.advance target=" + target); - } - - if (docFreq > BLOCK_SIZE && target > nextSkipDoc) { - if (DEBUG) { - System.out.println(" try skipper"); - } - if (skipper == null) { - // Lazy init: first time this enum has ever been used for skipping - if (DEBUG) { - System.out.println(" create skipper"); - } - skipper = new BlockPackedSkipReader((IndexInput) docIn.clone(), - BlockPackedPostingsWriter.maxSkipLevels, - BLOCK_SIZE, - true, - indexHasOffsets, - indexHasPayloads); - } - - if (!skipped) { - assert skipOffset != -1; - // This is the first time this enum has skipped - // since reset() was called; load the skip data: - if (DEBUG) { - System.out.println(" init skipper"); - } - skipper.init(docTermStartFP+skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq); - skipped = true; - } - - final int newDocUpto = skipper.skipTo(target) + 1; - - if (newDocUpto > docUpto) { - // Skipper moved - if (DEBUG) { - System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto()); - } - - assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto; - docUpto = newDocUpto; - - // Force to read next block - docBufferUpto = BLOCK_SIZE; - accum = skipper.getDoc(); - docIn.seek(skipper.getDocPointer()); - posPendingFP = skipper.getPosPointer(); - posPendingCount = skipper.getPosBufferUpto(); - } - nextSkipDoc = skipper.getNextSkipDoc(); - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - if (docBufferUpto == BLOCK_SIZE) { - refillDocs(); - } - - // Now scan... this is an inlined/pared down version - // of nextDoc(): - while (true) { - if (DEBUG) { - System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - accum += docDeltaBuffer[docBufferUpto]; - freq = freqBuffer[docBufferUpto]; - posPendingCount += freq; - docBufferUpto++; - docUpto++; - - if (accum >= target) { - break; - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - } - - if (liveDocs == null || liveDocs.get(accum)) { - if (DEBUG) { - System.out.println(" return doc=" + accum); - } - position = 0; - return doc = accum; - } else { - if (DEBUG) { - System.out.println(" now do nextDoc()"); - } - return nextDoc(); - } - } - - // nocommit in theory we could avoid loading frq block - // when not needed, ie, use skip data to load how far to - // seek the pos pointe ... instead of having to load frq - // blocks only to sum up how many positions to skip - private void skipPositions() throws IOException { - // Skip positions now: - int toSkip = posPendingCount - freq; - if (DEBUG) { - System.out.println(" FPR.skipPositions: toSkip=" + toSkip); - } - - final int leftInBlock = BLOCK_SIZE - posBufferUpto; - if (toSkip < leftInBlock) { - posBufferUpto += toSkip; - if (DEBUG) { - System.out.println(" skip w/in block to posBufferUpto=" + posBufferUpto); - } - } else { - toSkip -= leftInBlock; - while(toSkip >= BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" skip whole block @ fp=" + posIn.getFilePointer()); - } - assert posIn.getFilePointer() != lastPosBlockFP; - forUtil.skipBlock(posIn); - toSkip -= BLOCK_SIZE; - } - refillPositions(); - posBufferUpto = toSkip; - if (DEBUG) { - System.out.println(" skip w/in block to posBufferUpto=" + posBufferUpto); - } - } - - position = 0; - } - - @Override - public int nextPosition() throws IOException { - if (DEBUG) { - System.out.println(" FPR.nextPosition posPendingCount=" + posPendingCount + " posBufferUpto=" + posBufferUpto); - } - if (posPendingFP != -1) { - if (DEBUG) { - System.out.println(" seek to pendingFP=" + posPendingFP); - } - posIn.seek(posPendingFP); - posPendingFP = -1; - - // Force buffer refill: - posBufferUpto = BLOCK_SIZE; - } - - if (posPendingCount > freq) { - skipPositions(); - posPendingCount = freq; - } - - if (posBufferUpto == BLOCK_SIZE) { - refillPositions(); - posBufferUpto = 0; - } - position += posDeltaBuffer[posBufferUpto++]; - posPendingCount--; - if (DEBUG) { - System.out.println(" return pos=" + position); - } - return position; - } - - @Override - public int startOffset() { - return -1; - } - - @Override - public int endOffset() { - return -1; - } - - @Override - public boolean hasPayload() { - return false; - } - - @Override - public BytesRef getPayload() { - return null; - } - } - - // Also handles payloads + offsets - final class EverythingEnum extends DocsAndPositionsEnum { - - private final byte[] encoded; - - private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE]; - private final int[] freqBuffer = new int[MAX_DATA_SIZE]; - private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE]; - - private final int[] payloadLengthBuffer; - private final int[] offsetStartDeltaBuffer; - private final int[] offsetLengthBuffer; - - private byte[] payloadBytes; - private int payloadByteUpto; - private int payloadLength; - - private int lastStartOffset; - private int startOffset; - private int endOffset; - - private int docBufferUpto; - private int posBufferUpto; - - private BlockPackedSkipReader skipper; - private boolean skipped; - - final IndexInput startDocIn; - - final IndexInput docIn; - final IndexInput posIn; - final IndexInput payIn; - final BytesRef payload; - - final boolean indexHasOffsets; - final boolean indexHasPayloads; - - private int docFreq; // number of docs in this posting list - private int docUpto; // how many docs we've read - private int doc; // doc we last read - private int accum; // accumulator for doc deltas - private int freq; // freq we last read - private int position; // current position - - // how many positions "behind" we are; nextPosition must - // skip these to "catch up": - private int posPendingCount; - - // Lazy pos seek: if != -1 then we must seek to this FP - // before reading positions: - private long posPendingFP; - - // Lazy pay seek: if != -1 then we must seek to this FP - // before reading payloads/offsets: - private long payPendingFP; - - // Where this term's postings start in the .doc file: - private long docTermStartFP; - - // Where this term's postings start in the .pos file: - private long posTermStartFP; - - // Where this term's payloads/offsets start in the .pay - // file: - private long payTermStartFP; - - // File pointer where the last (vInt encoded) pos delta - // block is. We need this to know whether to bulk - // decode vs vInt decode the block: - private long lastPosBlockFP; - - // Where this term's skip data starts (after - // docTermStartFP) in the .doc file (or -1 if there is - // no skip data for this term): - private int skipOffset; - - private int nextSkipDoc; - - private Bits liveDocs; - - public EverythingEnum(FieldInfo fieldInfo) throws IOException { - this.startDocIn = BlockPackedPostingsReader.this.docIn; - this.docIn = (IndexInput) startDocIn.clone(); - this.posIn = (IndexInput) BlockPackedPostingsReader.this.posIn.clone(); - this.payIn = (IndexInput) BlockPackedPostingsReader.this.payIn.clone(); - encoded = new byte[MAX_ENCODED_SIZE]; - indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - if (indexHasOffsets) { - offsetStartDeltaBuffer = new int[MAX_DATA_SIZE]; - offsetLengthBuffer = new int[MAX_DATA_SIZE]; - } else { - offsetStartDeltaBuffer = null; - offsetLengthBuffer = null; - startOffset = -1; - endOffset = -1; - } - - indexHasPayloads = fieldInfo.hasPayloads(); - if (indexHasPayloads) { - payloadLengthBuffer = new int[MAX_DATA_SIZE]; - payloadBytes = new byte[128]; - payload = new BytesRef(); - } else { - payloadLengthBuffer = null; - payloadBytes = null; - payload = null; - } - } - - public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { - return docIn == startDocIn && - indexHasOffsets == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) && - indexHasPayloads == fieldInfo.hasPayloads(); - } - - public EverythingEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException { - this.liveDocs = liveDocs; - if (DEBUG) { - System.out.println(" FPR.reset: termState=" + termState); - } - docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; - docIn.seek(docTermStartFP); - skipOffset = termState.skipOffset; - posPendingFP = posTermStartFP; - payPendingFP = payTermStartFP; - posPendingCount = 0; - if (termState.totalTermFreq < BLOCK_SIZE) { - lastPosBlockFP = posTermStartFP; - } else if (termState.totalTermFreq == BLOCK_SIZE) { - lastPosBlockFP = -1; - } else { - lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; - } - - doc = -1; - accum = 0; - docUpto = 0; - nextSkipDoc = BLOCK_SIZE - 1; - docBufferUpto = BLOCK_SIZE; - skipped = false; - return this; - } - - @Override - public int freq() throws IOException { - return freq; - } - - @Override - public int docID() { - return doc; - } - - private void refillDocs() throws IOException { - final int left = docFreq - docUpto; - assert left > 0; - - if (left >= BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); - } - forUtil.readBlock(docIn, encoded, docDeltaBuffer); - if (DEBUG) { - System.out.println(" fill freq block from fp=" + docIn.getFilePointer()); - } - forUtil.readBlock(docIn, encoded, freqBuffer); - } else { - if (DEBUG) { - System.out.println(" fill last vInt doc block from fp=" + docIn.getFilePointer()); - } - readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true); - } - docBufferUpto = 0; - } - - private void refillPositions() throws IOException { - if (DEBUG) { - System.out.println(" refillPositions"); - } - if (posIn.getFilePointer() == lastPosBlockFP) { - if (DEBUG) { - System.out.println(" vInt pos block @ fp=" + posIn.getFilePointer() + " hasPayloads=" + indexHasPayloads + " hasOffsets=" + indexHasOffsets); - } - final int count = posIn.readVInt(); - int payloadLength = 0; - payloadByteUpto = 0; - for(int i=0;i docUpto) { - // Skipper moved - if (DEBUG) { - System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset); - } - assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto; - docUpto = newDocUpto; - - // Force to read next block - docBufferUpto = BLOCK_SIZE; - accum = skipper.getDoc(); - docIn.seek(skipper.getDocPointer()); - posPendingFP = skipper.getPosPointer(); - payPendingFP = skipper.getPayPointer(); - posPendingCount = skipper.getPosBufferUpto(); - lastStartOffset = skipper.getStartOffset(); - payloadByteUpto = skipper.getPayloadByteUpto(); - } - nextSkipDoc = skipper.getNextSkipDoc(); - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - if (docBufferUpto == BLOCK_SIZE) { - refillDocs(); - } - - // Now scan: - // Now scan: - while (true) { - if (DEBUG) { - System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); - } - accum += docDeltaBuffer[docBufferUpto]; - freq = freqBuffer[docBufferUpto]; - posPendingCount += freq; - docBufferUpto++; - docUpto++; - - if (accum >= target) { - break; - } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - } - - if (liveDocs == null || liveDocs.get(accum)) { - if (DEBUG) { - System.out.println(" return doc=" + accum); - } - if (indexHasPayloads) { - payloadByteUpto += payloadLength; - payloadLength = 0; - } - position = 0; - payloadLength = 0; - lastStartOffset = 0; - return doc = accum; - } else { - if (DEBUG) { - System.out.println(" now do nextDoc()"); - } - return nextDoc(); - } - } - - // nocommit in theory we could avoid loading frq block - // when not needed, ie, use skip data to load how far to - // seek the pos pointe ... instead of having to load frq - // blocks only to sum up how many positions to skip - private void skipPositions() throws IOException { - // Skip positions now: - int toSkip = posPendingCount - freq; - if (DEBUG) { - System.out.println(" FPR.skipPositions: toSkip=" + toSkip); - } - - final int leftInBlock = BLOCK_SIZE - posBufferUpto; - if (toSkip < leftInBlock) { - int end = posBufferUpto + toSkip; - while(posBufferUpto < end) { - if (indexHasPayloads) { - payloadByteUpto += payloadLengthBuffer[posBufferUpto]; - } - if (indexHasOffsets) { - lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto]; - } - posBufferUpto++; - } - if (DEBUG) { - System.out.println(" skip w/in block to posBufferUpto=" + posBufferUpto); - } - } else { - toSkip -= leftInBlock; - while(toSkip >= BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" skip whole block @ fp=" + posIn.getFilePointer()); - } - assert posIn.getFilePointer() != lastPosBlockFP; - forUtil.skipBlock(posIn); - - if (indexHasPayloads) { - // Skip payloadLength block: - forUtil.skipBlock(payIn); - - // Skip payloadBytes block: - int numBytes = payIn.readVInt(); - payIn.seek(payIn.getFilePointer() + numBytes); - } - - if (indexHasOffsets) { - // Must load offset blocks merely to sum - // up into lastStartOffset: - forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer); - forUtil.readBlock(payIn, encoded, offsetLengthBuffer); - for(int i=0;i= 0; - fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; - fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - fieldHasPayloads = fieldInfo.hasPayloads(); - skipWriter.setField(fieldHasPositions, fieldHasOffsets, fieldHasPayloads); - } - - @Override - public void startTerm() { - docTermStartFP = docOut.getFilePointer(); - if (fieldHasPositions) { - posTermStartFP = posOut.getFilePointer(); - if (fieldHasPayloads || fieldHasOffsets) { - payTermStartFP = payOut.getFilePointer(); - } - } - lastDocID = 0; - lastBlockDocID = -1; - if (DEBUG) { - System.out.println("FPW.startTerm startFP=" + docTermStartFP); - } - skipWriter.resetSkip(); - } - - @Override - public void startDoc(int docID, int termDocFreq) throws IOException { - if (DEBUG) { - System.out.println("FPW.startDoc docID["+docBufferUpto+"]=" + docID); - } - - final int docDelta = docID - lastDocID; - - if (docID < 0 || (docCount > 0 && docDelta <= 0)) { - throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " ) (docOut: " + docOut + ")"); - } - - docDeltaBuffer[docBufferUpto] = docDelta; -// if (DEBUG) { -// System.out.println(" docDeltaBuffer[" + docBufferUpto + "]=" + docDelta); -// } - if (fieldHasFreqs) { - freqBuffer[docBufferUpto] = termDocFreq; - } - docBufferUpto++; - docCount++; - - if (docBufferUpto == BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" write docDelta block @ fp=" + docOut.getFilePointer()); - } - forUtil.writeBlock(docDeltaBuffer, encoded, docOut); - if (fieldHasFreqs) { - if (DEBUG) { - System.out.println(" write freq block @ fp=" + docOut.getFilePointer()); - } - forUtil.writeBlock(freqBuffer, encoded, docOut); - } - // NOTE: don't set docBufferUpto back to 0 here; - // finishDoc will do so (because it needs to see that - // the block was filled so it can save skip data) - } - - lastDocID = docID; - lastPosition = 0; - lastStartOffset = 0; - } - - /** Add a new position & payload */ - @Override - public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException { -// if (DEBUG) { -// System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (fieldHasPayloads ? " payloadByteUpto=" + payloadByteUpto: "")); -// } - posDeltaBuffer[posBufferUpto] = position - lastPosition; - if (fieldHasPayloads) { - if (payload == null || payload.length == 0) { - // no payload - payloadLengthBuffer[posBufferUpto] = 0; - } else { - payloadLengthBuffer[posBufferUpto] = payload.length; - if (payloadByteUpto + payload.length > payloadBytes.length) { - payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length); - } - System.arraycopy(payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length); - payloadByteUpto += payload.length; - } - } - - if (fieldHasOffsets) { - assert startOffset >= lastStartOffset; - assert endOffset >= startOffset; - offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset; - offsetLengthBuffer[posBufferUpto] = endOffset - startOffset; - lastStartOffset = startOffset; - } - - posBufferUpto++; - lastPosition = position; - if (posBufferUpto == BLOCK_SIZE) { - if (DEBUG) { - System.out.println(" write pos bulk block @ fp=" + posOut.getFilePointer()); - } - forUtil.writeBlock(posDeltaBuffer, encoded, posOut); - - if (fieldHasPayloads) { - forUtil.writeBlock(payloadLengthBuffer, encoded, payOut); - payOut.writeVInt(payloadByteUpto); - payOut.writeBytes(payloadBytes, 0, payloadByteUpto); - payloadByteUpto = 0; - } - if (fieldHasOffsets) { - forUtil.writeBlock(offsetStartDeltaBuffer, encoded, payOut); - forUtil.writeBlock(offsetLengthBuffer, encoded, payOut); - } - posBufferUpto = 0; - } - } - - @Override - public void finishDoc() throws IOException { - // Have collected a block of docs, and get a new doc. - // Should write skip data as well as postings list for - // current block - - if (lastBlockDocID != -1 && docBufferUpto == 1) { - // nocomit move to startDoc? ie we can write skip - // data as soon as the next doc starts... - if (DEBUG) { - System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-1)); - } - skipWriter.bufferSkip(lastBlockDocID, docCount-1, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto); - } - - // Since we don't know df for current term, we had to buffer - // those skip data for each block, and when a new doc comes, - // write them to skip file. - if (docBufferUpto == BLOCK_SIZE) { - lastBlockDocID = lastDocID; - if (posOut != null) { - if (payOut != null) { - lastBlockPayFP = payOut.getFilePointer(); - } - lastBlockPosFP = posOut.getFilePointer(); - lastBlockPosBufferUpto = posBufferUpto; - lastBlockStartOffset = lastStartOffset; - lastBlockPayloadByteUpto = payloadByteUpto; - } - if (DEBUG) { - System.out.println(" docBufferUpto="+docBufferUpto+" now get lastBlockDocID="+lastBlockDocID+" lastBlockPosFP=" + lastBlockPosFP + " lastBlockPosBufferUpto=" + lastBlockPosBufferUpto + " lastBlockPayloadByteUpto=" + lastBlockPayloadByteUpto); - } - docBufferUpto = 0; - } - } - - private static class PendingTerm { - public final long docStartFP; - public final long posStartFP; - public final long payStartFP; - public final int skipOffset; - public final int lastPosBlockOffset; - - public PendingTerm(long docStartFP, long posStartFP, long payStartFP, int skipOffset, int lastPosBlockOffset) { - this.docStartFP = docStartFP; - this.posStartFP = posStartFP; - this.payStartFP = payStartFP; - this.skipOffset = skipOffset; - this.lastPosBlockOffset = lastPosBlockOffset; - } - } - - private final List pendingTerms = new ArrayList(); - - /** Called when we are done adding docs to this term */ - @Override - public void finishTerm(TermStats stats) throws IOException { - assert stats.docFreq > 0; - - // TODO: wasteful we are counting this (counting # docs - // for this term) in two places? - assert stats.docFreq == docCount: stats.docFreq + " vs " + docCount; - - if (DEBUG) { - System.out.println("FPW.finishTerm docFreq=" + stats.docFreq); - } - - if (DEBUG) { - if (docBufferUpto > 0) { - System.out.println(" write doc/freq vInt block (count=" + docBufferUpto + ") at fp=" + docOut.getFilePointer() + " docTermStartFP=" + docTermStartFP); - } - } - - // vInt encode the remaining doc deltas and freqs: - for(int i=0;i 0) { - System.out.println(" write pos vInt block (count=" + posBufferUpto + ") at fp=" + posOut.getFilePointer() + " posTermStartFP=" + posTermStartFP + " hasPayloads=" + fieldHasPayloads + " hasOffsets=" + fieldHasOffsets); - } - } - - assert stats.totalTermFreq != -1; - if (stats.totalTermFreq > BLOCK_SIZE) { - lastPosBlockOffset = (int) (posOut.getFilePointer() - posTermStartFP); - } else { - lastPosBlockOffset = -1; - } - if (posBufferUpto > 0) { - posOut.writeVInt(posBufferUpto); - - // nocommit should we send offsets/payloads to - // .pay...? seems wasteful (have to store extra - // vLong for low (< BLOCK_SIZE) DF terms = vast vast - // majority) - - // vInt encode the remaining positions/payloads/offsets: - int lastPayloadLength = -1; - int payloadBytesReadUpto = 0; - for(int i=0;iBLOCK_SIZE encoded values. - */ - static final int MAX_ENCODED_SIZE = BLOCK_SIZE * 4; - - /** - * Upper limit of the number of values that might be decoded in a single call to - * {@link #readBlock(IndexInput, byte[], int[])}. Although values after - * BLOCK_SIZE are garbage, it is necessary to allocate value buffers - * whose size is >= MAX_DATA_SIZE to avoid {@link ArrayIndexOutOfBoundsException}s. - */ - static final int MAX_DATA_SIZE; - static { - int minDataSize = 0; - for (PackedInts.Format format : PackedInts.Format.values()) { - for (int bpv = 1; bpv <= 32; ++bpv) { - if (!format.isSupported(bpv)) { - continue; - } - final PackedInts.Decoder decoder = PackedInts.getDecoder(format, PACKED_INTS_VERSION, bpv); - final int iterations = (int) Math.ceil((float) BLOCK_SIZE / decoder.valueCount()); - minDataSize = Math.max(minDataSize, iterations * decoder.valueCount()); - } - } - MAX_DATA_SIZE = minDataSize; - } - - /** - * Compute the number of iterations required to decode BLOCK_SIZE - * values with the provided {@link Decoder}. - */ - private static int computeIterations(PackedInts.Decoder decoder) { - return (int) Math.ceil((float) BLOCK_SIZE / decoder.valueCount()); - } - - /** - * Compute the number of bytes required to encode a block of values that require - * bitsPerValue bits per value with format format. - */ - private static int encodedSize(PackedInts.Format format, int bitsPerValue) { - return format.nblocks(bitsPerValue, BLOCK_SIZE) << 3; - } - - private final int[] encodedSizes; - private final PackedInts.Encoder[] encoders; - private final PackedInts.Decoder[] decoders; - private final int[] iterations; - - /** - * Create a new {@link ForUtil} instance and save state into out. - */ - ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { - encodedSizes = new int[33]; - encoders = new PackedInts.Encoder[33]; - decoders = new PackedInts.Decoder[33]; - iterations = new int[33]; - - for (int bpv = 1; bpv <= 32; ++bpv) { - final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( - BLOCK_SIZE, bpv, acceptableOverheadRatio); - assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); - assert formatAndBits.bitsPerValue <= 32; - encodedSizes[bpv] = encodedSize(formatAndBits.format, formatAndBits.bitsPerValue); - encoders[bpv] = PackedInts.getEncoder( - formatAndBits.format, PACKED_INTS_VERSION, formatAndBits.bitsPerValue); - decoders[bpv] = PackedInts.getDecoder( - formatAndBits.format, PACKED_INTS_VERSION, formatAndBits.bitsPerValue); - iterations[bpv] = computeIterations(decoders[bpv]); - - out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); - } - } - - /** - * Restore a {@link ForUtil} from a {@link DataInput}. - */ - ForUtil(DataInput in) throws IOException { - encodedSizes = new int[33]; - encoders = new PackedInts.Encoder[33]; - decoders = new PackedInts.Decoder[33]; - iterations = new int[33]; - - for (int bpv = 1; bpv <= 32; ++bpv) { - final int code = in.readVInt(); - final int formatId = code >>> 5; - final int bitsPerValue = (code & 31) + 1; - - final PackedInts.Format format = PackedInts.Format.byId(formatId); - assert format.isSupported(bitsPerValue); - encodedSizes[bpv] = encodedSize(format, bitsPerValue); - encoders[bpv] = PackedInts.getEncoder( - format, PACKED_INTS_VERSION, bitsPerValue); - decoders[bpv] = PackedInts.getDecoder( - format, PACKED_INTS_VERSION, bitsPerValue); - iterations[bpv] = computeIterations(decoders[bpv]); - } - } - - /** - * Write a block of data (For format). - * - * @param data the data to write - * @param encoded a buffer to use to encode data - * @param out the destination output - * @throws IOException - */ - void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { - if (isAllEqual(data)) { - out.writeVInt(ALL_VALUES_EQUAL); - out.writeInt(data[0]); - return; - } - - final int numBits = bitsRequired(data); - assert numBits > 0 && numBits <= 32 : numBits; - final PackedInts.Encoder encoder = encoders[numBits]; - final int iters = iterations[numBits]; - assert iters * encoder.valueCount() >= BLOCK_SIZE; - final int encodedSize = encodedSizes[numBits]; - assert (iters * encoder.blockCount()) << 3 >= encodedSize; - - out.writeVInt(numBits); - - encoder.encode(data, 0, encoded, 0, iters); - out.writeBytes(encoded, encodedSize); - } - - /** - * Read the next block of data (For format). - * - * @param in the input to use to read data - * @param encoded a buffer that can be used to store encoded data - * @param decoded where to write decoded data - * @throws IOException - */ - void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { - final int numBits = in.readVInt(); - assert numBits <= 32 : numBits; - - if (numBits == ALL_VALUES_EQUAL) { - final int value = in.readInt(); - Arrays.fill(decoded, 0, BLOCK_SIZE, value); - return; - } - - final int encodedSize = encodedSizes[numBits]; - in.readBytes(encoded, 0, encodedSize); - - final PackedInts.Decoder decoder = decoders[numBits]; - final int iters = iterations[numBits]; - assert iters * decoder.valueCount() >= BLOCK_SIZE; - - decoder.decode(encoded, 0, decoded, 0, iters); - } - - /** - * Skip the next block of data. - * - * @param in the input where to read data - * @throws IOException - */ - void skipBlock(IndexInput in) throws IOException { - final int numBits = in.readVInt(); - if (numBits == ALL_VALUES_EQUAL) { - in.seek(in.getFilePointer() + 4); - return; - } - assert numBits > 0 && numBits <= 32 : numBits; - final int encodedSize = encodedSizes[numBits]; - in.seek(in.getFilePointer() + encodedSize); - } - - // nocommit: we must have a util function for this, hmm? - private static boolean isAllEqual(final int[] data) { - final long v = data[0]; - for (int i = 1; i < BLOCK_SIZE; ++i) { - if (data[i] != v) { - return false; - } - } - return true; - } - - /** - * Compute the number of bits required to serialize any of the longs in - * data. - */ - private static int bitsRequired(final int[] data) { - long or = 0; - for (int i = 0; i < BLOCK_SIZE; ++i) { - or |= data[i]; - } - return PackedInts.bitsRequired(or); - } - -} diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat index 5a0096d4437..8a246a8a29d 100644 --- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat +++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat @@ -21,4 +21,3 @@ org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat org.apache.lucene.codecs.memory.DirectPostingsFormat org.apache.lucene.codecs.block.BlockPostingsFormat -org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat diff --git a/lucene/core/src/test/org/apache/lucene/codecs/blockpacked/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/block/TestForUtil.java similarity index 91% rename from lucene/core/src/test/org/apache/lucene/codecs/blockpacked/TestForUtil.java rename to lucene/core/src/test/org/apache/lucene/codecs/block/TestForUtil.java index 983b8df7b2f..025a6348164 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/blockpacked/TestForUtil.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/block/TestForUtil.java @@ -1,4 +1,4 @@ -package org.apache.lucene.codecs.blockpacked; +package org.apache.lucene.codecs.block; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,9 +17,9 @@ package org.apache.lucene.codecs.blockpacked; * limitations under the License. */ -import static org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat.BLOCK_SIZE; -import static org.apache.lucene.codecs.blockpacked.ForUtil.MAX_DATA_SIZE; -import static org.apache.lucene.codecs.blockpacked.ForUtil.MAX_ENCODED_SIZE; +import static org.apache.lucene.codecs.block.BlockPostingsFormat.BLOCK_SIZE; +import static org.apache.lucene.codecs.block.ForUtil.MAX_DATA_SIZE; +import static org.apache.lucene.codecs.block.ForUtil.MAX_ENCODED_SIZE; import java.io.IOException; import java.util.Arrays;