From 0ec217b753a78e81892f12e8076313e9072f01b0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 2 Dec 2021 14:00:46 +0100 Subject: [PATCH] Make EndiannessReverser(Data|Index)Input always reverse byte order. (#502) Currently EndiannessReverser(Data|Index)Input doesn't reverse the byte order for `readLongs` and `readFloats`. The reasoning is that these two method replaced `readLELongs` and `readLEFloats`, so the byte order doesn't need changing. However this is creating some confusing situations where new code expects a consistent byte order on the write and read sides and gets longs in the wrong byte order. So this commit suggests a different approach, where EndiannessReverser(Data|Index)Input always changes the byte order, and former call sites of `readLELongs` and `readLEFloats` are changed to manually reverse the byte order on top of `readLongs` and `readFloats`. This is making old codecs a bit slower, but I think it's fair since these are old codecs. And this makes the endianness reversing backward compatibility layer easier to reason about? --- .../generated/checksums/generateForUtil.json | 4 +- .../backward_codecs/lucene84/ForUtil.java | 58 ++++--- .../backward_codecs/lucene84/gen_ForUtil.py | 152 ++++++++++-------- .../store/EndiannessReverserIndexInput.java | 9 +- 4 files changed, 123 insertions(+), 100 deletions(-) diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil.json index e9c6ee64a18..543f4f5101b 100644 --- a/lucene/backward-codecs/src/generated/checksums/generateForUtil.json +++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil.json @@ -1,4 +1,4 @@ { - "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java": "59319f493441c05e84ca9077a9f3bf98b6eb8c7e", - "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py": "96635fbbf3270b6cb25237b8c01e068ecad0d2ce" + "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java": "e91aafa414018b34a39c8f0947ff58c1f1dde78d", + "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py": "285ed73b9763c541edaf40072667d68bd2537c56" } \ No newline at end of file diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java index 25408e46ffe..ec9963c2d0b 100644 --- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java @@ -220,6 +220,14 @@ final class ForUtil { arr[63] += arr[62]; } + private static void readLELongs(DataInput in, long[] dst, int offset, int length) + throws IOException { + in.readLongs(dst, offset, length); + for (int i = 0; i < length; ++i) { + dst[offset + i] = Long.reverseBytes(dst[offset + i]); + } + } + private final long[] tmp = new long[BLOCK_SIZE / 2]; /** Encode 128 integers from {@code longs} into {@code out}. */ @@ -306,7 +314,7 @@ final class ForUtil { private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs) throws IOException { final int numLongs = bitsPerValue << 1; - in.readLongs(tmp, 0, numLongs); + readLELongs(in, tmp, 0, numLongs); final long mask = MASKS32[bitsPerValue]; int longsIdx = 0; int shift = 32 - bitsPerValue; @@ -621,7 +629,7 @@ final class ForUtil { } private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 2); + readLELongs(in, tmp, 0, 2); shiftLongs(tmp, 2, longs, 0, 7, MASK8_1); shiftLongs(tmp, 2, longs, 2, 6, MASK8_1); shiftLongs(tmp, 2, longs, 4, 5, MASK8_1); @@ -633,7 +641,7 @@ final class ForUtil { } private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 4); + readLELongs(in, tmp, 0, 4); shiftLongs(tmp, 4, longs, 0, 6, MASK8_2); shiftLongs(tmp, 4, longs, 4, 4, MASK8_2); shiftLongs(tmp, 4, longs, 8, 2, MASK8_2); @@ -641,7 +649,7 @@ final class ForUtil { } private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 6); + readLELongs(in, tmp, 0, 6); shiftLongs(tmp, 6, longs, 0, 5, MASK8_3); shiftLongs(tmp, 6, longs, 6, 2, MASK8_3); for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) { @@ -655,13 +663,13 @@ final class ForUtil { } private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 8); + readLELongs(in, tmp, 0, 8); shiftLongs(tmp, 8, longs, 0, 4, MASK8_4); shiftLongs(tmp, 8, longs, 8, 0, MASK8_4); } private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 10); + readLELongs(in, tmp, 0, 10); shiftLongs(tmp, 10, longs, 0, 3, MASK8_5); for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) { long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2; @@ -678,7 +686,7 @@ final class ForUtil { } private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 12); + readLELongs(in, tmp, 0, 12); shiftLongs(tmp, 12, longs, 0, 2, MASK8_6); shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2); for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) { @@ -690,7 +698,7 @@ final class ForUtil { } private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 14); + readLELongs(in, tmp, 0, 14); shiftLongs(tmp, 14, longs, 0, 1, MASK8_7); shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1); for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) { @@ -706,11 +714,11 @@ final class ForUtil { } private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(longs, 0, 16); + readLELongs(in, longs, 0, 16); } private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 18); + readLELongs(in, tmp, 0, 18); shiftLongs(tmp, 18, longs, 0, 7, MASK16_9); for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) { long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2; @@ -739,7 +747,7 @@ final class ForUtil { } private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 20); + readLELongs(in, tmp, 0, 20); shiftLongs(tmp, 20, longs, 0, 6, MASK16_10); for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) { long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4; @@ -756,7 +764,7 @@ final class ForUtil { } private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 22); + readLELongs(in, tmp, 0, 22); shiftLongs(tmp, 22, longs, 0, 5, MASK16_11); for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) { long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6; @@ -783,7 +791,7 @@ final class ForUtil { } private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 24); + readLELongs(in, tmp, 0, 24); shiftLongs(tmp, 24, longs, 0, 4, MASK16_12); shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4); for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) { @@ -795,7 +803,7 @@ final class ForUtil { } private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 26); + readLELongs(in, tmp, 0, 26); shiftLongs(tmp, 26, longs, 0, 3, MASK16_13); for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) { long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10; @@ -820,7 +828,7 @@ final class ForUtil { } private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 28); + readLELongs(in, tmp, 0, 28); shiftLongs(tmp, 28, longs, 0, 2, MASK16_14); shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2); for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) { @@ -836,7 +844,7 @@ final class ForUtil { } private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 30); + readLELongs(in, tmp, 0, 30); shiftLongs(tmp, 30, longs, 0, 1, MASK16_15); shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1); for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) { @@ -860,11 +868,11 @@ final class ForUtil { } private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(longs, 0, 32); + readLELongs(in, longs, 0, 32); } private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 34); + readLELongs(in, tmp, 0, 34); shiftLongs(tmp, 34, longs, 0, 15, MASK32_17); for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) { long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2; @@ -917,7 +925,7 @@ final class ForUtil { } private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 36); + readLELongs(in, tmp, 0, 36); shiftLongs(tmp, 36, longs, 0, 14, MASK32_18); for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) { long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4; @@ -946,7 +954,7 @@ final class ForUtil { } private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 38); + readLELongs(in, tmp, 0, 38); shiftLongs(tmp, 38, longs, 0, 13, MASK32_19); for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) { long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6; @@ -997,7 +1005,7 @@ final class ForUtil { } private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 40); + readLELongs(in, tmp, 0, 40); shiftLongs(tmp, 40, longs, 0, 12, MASK32_20); for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) { long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8; @@ -1014,7 +1022,7 @@ final class ForUtil { } private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 42); + readLELongs(in, tmp, 0, 42); shiftLongs(tmp, 42, longs, 0, 11, MASK32_21); for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) { long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10; @@ -1063,7 +1071,7 @@ final class ForUtil { } private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 44); + readLELongs(in, tmp, 0, 44); shiftLongs(tmp, 44, longs, 0, 10, MASK32_22); for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) { long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12; @@ -1090,7 +1098,7 @@ final class ForUtil { } private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 46); + readLELongs(in, tmp, 0, 46); shiftLongs(tmp, 46, longs, 0, 9, MASK32_23); for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) { long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14; @@ -1137,7 +1145,7 @@ final class ForUtil { } private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException { - in.readLongs(tmp, 0, 48); + readLELongs(in, tmp, 0, 48); shiftLongs(tmp, 48, longs, 0, 8, MASK32_24); shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8); for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) { diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py index 30256182122..40776450425 100644 --- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py +++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py @@ -40,10 +40,9 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.codecs.lucene84; +package org.apache.lucene.backward_codecs.lucene84; import java.io.IOException; - import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; @@ -85,13 +84,13 @@ final class ForUtil { for (int i = 0; i < 16; ++i) { long l = arr[i]; arr[i] = (l >>> 56) & 0xFFL; - arr[16+i] = (l >>> 48) & 0xFFL; - arr[32+i] = (l >>> 40) & 0xFFL; - arr[48+i] = (l >>> 32) & 0xFFL; - arr[64+i] = (l >>> 24) & 0xFFL; - arr[80+i] = (l >>> 16) & 0xFFL; - arr[96+i] = (l >>> 8) & 0xFFL; - arr[112+i] = l & 0xFFL; + arr[16 + i] = (l >>> 48) & 0xFFL; + arr[32 + i] = (l >>> 40) & 0xFFL; + arr[48 + i] = (l >>> 32) & 0xFFL; + arr[64 + i] = (l >>> 24) & 0xFFL; + arr[80 + i] = (l >>> 16) & 0xFFL; + arr[96 + i] = (l >>> 8) & 0xFFL; + arr[112 + i] = l & 0xFFL; } } @@ -99,15 +98,23 @@ final class ForUtil { for (int i = 0; i < 16; ++i) { long l = arr[i]; arr[i] = (l >>> 24) & 0x000000FF000000FFL; - arr[16+i] = (l >>> 16) & 0x000000FF000000FFL; - arr[32+i] = (l >>> 8) & 0x000000FF000000FFL; - arr[48+i] = l & 0x000000FF000000FFL; + arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL; + arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL; + arr[48 + i] = l & 0x000000FF000000FFL; } } private static void collapse8(long[] arr) { for (int i = 0; i < 16; ++i) { - arr[i] = (arr[i] << 56) | (arr[16+i] << 48) | (arr[32+i] << 40) | (arr[48+i] << 32) | (arr[64+i] << 24) | (arr[80+i] << 16) | (arr[96+i] << 8) | arr[112+i]; + arr[i] = + (arr[i] << 56) + | (arr[16 + i] << 48) + | (arr[32 + i] << 40) + | (arr[48 + i] << 32) + | (arr[64 + i] << 24) + | (arr[80 + i] << 16) + | (arr[96 + i] << 8) + | arr[112 + i]; } } @@ -115,9 +122,9 @@ final class ForUtil { for (int i = 0; i < 32; ++i) { long l = arr[i]; arr[i] = (l >>> 48) & 0xFFFFL; - arr[32+i] = (l >>> 32) & 0xFFFFL; - arr[64+i] = (l >>> 16) & 0xFFFFL; - arr[96+i] = l & 0xFFFFL; + arr[32 + i] = (l >>> 32) & 0xFFFFL; + arr[64 + i] = (l >>> 16) & 0xFFFFL; + arr[96 + i] = l & 0xFFFFL; } } @@ -125,13 +132,13 @@ final class ForUtil { for (int i = 0; i < 32; ++i) { long l = arr[i]; arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL; - arr[32+i] = l & 0x0000FFFF0000FFFFL; + arr[32 + i] = l & 0x0000FFFF0000FFFFL; } } private static void collapse16(long[] arr) { for (int i = 0; i < 32; ++i) { - arr[i] = (arr[i] << 48) | (arr[32+i] << 32) | (arr[64+i] << 16) | arr[96+i]; + arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i]; } } @@ -145,7 +152,7 @@ final class ForUtil { private static void collapse32(long[] arr) { for (int i = 0; i < 64; ++i) { - arr[i] = (arr[i] << 32) | arr[64+i]; + arr[i] = (arr[i] << 32) | arr[64 + i]; } } @@ -164,8 +171,8 @@ final class ForUtil { arr[0] += base << 32; innerPrefixSum32(arr); expand32(arr); - final long l = arr[BLOCK_SIZE/2-1]; - for (int i = BLOCK_SIZE/2; i < BLOCK_SIZE; ++i) { + final long l = arr[BLOCK_SIZE / 2 - 1]; + for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) { arr[i] += l; } } @@ -237,11 +244,17 @@ final class ForUtil { arr[63] += arr[62]; } - private final long[] tmp = new long[BLOCK_SIZE/2]; + private static void readLELongs(DataInput in, long[] dst, int offset, int length) + throws IOException { + in.readLongs(dst, offset, length); + for (int i = 0; i < length; ++i) { + dst[offset + i] = Long.reverseBytes(dst[offset + i]); + } + } - /** - * Encode 128 integers from {@code longs} into {@code out}. - */ + private final long[] tmp = new long[BLOCK_SIZE / 2]; + + /** Encode 128 integers from {@code longs} into {@code out}. */ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException { final int nextPrimitive; final int numLongs; @@ -310,22 +323,22 @@ final class ForUtil { } for (int i = 0; i < numLongsPerShift; ++i) { - // Java longs are big endian and we want to read little endian longs, so we need to reverse bytes + // Java longs are big endian and we want to read little endian longs, so we need to reverse + // bytes long l = Long.reverseBytes(tmp[i]); out.writeLong(l); } } - /** - * Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. - */ + /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */ int numBytes(int bitsPerValue) throws IOException { return bitsPerValue << (BLOCK_SIZE_LOG2 - 3); } - private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs) throws IOException { + private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs) + throws IOException { final int numLongs = bitsPerValue << 1; - in.readLELongs(tmp, 0, numLongs); + readLELongs(in, tmp, 0, numLongs); final long mask = MASKS32[bitsPerValue]; int longsIdx = 0; int shift = 32 - bitsPerValue; @@ -345,7 +358,7 @@ final class ForUtil { l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b; } if (b > 0) { - l |= (tmp[tmpIdx] >>> (remainingBitsPerLong-b)) & MASKS32[b]; + l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b]; remainingBits = remainingBitsPerLong - b; } else { remainingBits = remainingBitsPerLong; @@ -355,13 +368,12 @@ final class ForUtil { } /** - * The pattern that this shiftLongs method applies is recognized by the C2 - * compiler, which generates SIMD instructions for it in order to shift - * multiple longs at once. + * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which + * generates SIMD instructions for it in order to shift multiple longs at once. */ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) { for (int i = 0; i < count; ++i) { - b[bi+i] = (a[i] >>> shift) & mask; + b[bi + i] = (a[i] >>> shift) & mask; } } @@ -381,13 +393,13 @@ def writeRemainderWithSIMDOptimize(bpv, next_primitive, remaining_bits_per_long, tmp_idx = 0 b = bpv b -= remaining_bits_per_long - f.write(' long l0 = tmp[tmpIdx+%d] << %d;\n' %(tmp_idx, b)) + f.write(' long l0 = tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b)) tmp_idx += 1 while b >= remaining_bits_per_long: b -= remaining_bits_per_long - f.write(' l0 |= tmp[tmpIdx+%d] << %d;\n' %(tmp_idx, b)) + f.write(' l0 |= tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b)) tmp_idx += 1 - f.write(' longs[longsIdx+0] = l0;\n') + f.write(' longs[longsIdx + 0] = l0;\n') f.write(' }\n') @@ -406,19 +418,19 @@ def writeRemainder(bpv, next_primitive, remaining_bits_per_long, o, num_values, b = bpv if remaining_bits == 0: b -= remaining_bits_per_long - f.write(' long l%d = (tmp[tmpIdx+%d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b)) + f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b)) else: b -= remaining_bits - f.write(' long l%d = (tmp[tmpIdx+%d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b)) + f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b)) tmp_idx += 1 while b >= remaining_bits_per_long: b -= remaining_bits_per_long - f.write(' l%d |= (tmp[tmpIdx+%d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b)) + f.write(' l%d |= (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b)) tmp_idx += 1 if b > 0: - f.write(' l%d |= (tmp[tmpIdx+%d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_long-b, next_primitive, b)) + f.write(' l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_long-b, next_primitive, b)) remaining_bits = remaining_bits_per_long-b - f.write(' longs[longsIdx+%d] = l%d;\n' %(i, i)) + f.write(' longs[longsIdx + %d] = l%d;\n' %(i, i)) f.write(' }\n') @@ -432,9 +444,9 @@ def writeDecode(bpv, f): f.write(' private static void decode%d(DataInput in, long[] tmp, long[] longs) throws IOException {\n' %bpv) num_values_per_long = 64 / next_primitive if bpv == next_primitive: - f.write(' in.readLELongs(longs, 0, %d);\n' %(bpv*2)) + f.write(' readLELongs(in, longs, 0, %d);\n' %(bpv*2)) else: - f.write(' in.readLELongs(tmp, 0, %d);\n' %(bpv*2)) + f.write(' readLELongs(in, tmp, 0, %d);\n' %(bpv*2)) shift = next_primitive - bpv o = 0 while shift >= 0: @@ -454,23 +466,22 @@ if __name__ == '__main__': f.write(HEADER) for primitive_size in PRIMITIVE_SIZE: f.write(' private static final long[] MASKS%d = new long[%d];\n' %(primitive_size, primitive_size)) + f.write('\n') f.write(' static {\n') for primitive_size in PRIMITIVE_SIZE: f.write(' for (int i = 0; i < %d; ++i) {\n' %primitive_size) f.write(' MASKS%d[i] = mask%d(i);\n' %(primitive_size, primitive_size)) f.write(' }\n') f.write(' }\n') - f.write(' //mark values in array as final longs to avoid the cost of reading array, arrays should only be used when the idx is a variable\n') + f.write(' // mark values in array as final longs to avoid the cost of reading array, arrays should only be\n') + f.write(' // used when the idx is a variable\n') for primitive_size in PRIMITIVE_SIZE: for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)): if bpv * 2 != primitive_size or primitive_size == 8: f.write(' private static final long MASK%d_%d = MASKS%d[%d];\n' %(primitive_size, bpv, primitive_size, bpv)) - f.write('\n') f.write(""" - /** - * Decode 128 integers into {@code longs}. - */ + /** Decode 128 integers into {@code longs}. */ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException { switch (bitsPerValue) { """) @@ -480,22 +491,21 @@ if __name__ == '__main__': next_primitive = 8 elif bpv <= 16: next_primitive = 16 - f.write(' case %d:\n' %bpv) - f.write(' decode%d(in, tmp, longs);\n' %bpv) - f.write(' expand%d(longs);\n' %next_primitive) - f.write(' break;\n') - f.write(' default:\n') - f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n') - f.write(' expand32(longs);\n') - f.write(' break;\n') + f.write(' case %d:\n' %bpv) + f.write(' decode%d(in, tmp, longs);\n' %bpv) + f.write(' expand%d(longs);\n' %next_primitive) + f.write(' break;\n') + f.write(' default:\n') + f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n') + f.write(' expand32(longs);\n') + f.write(' break;\n') f.write(' }\n') f.write(' }\n') f.write(""" - /** - * Delta-decode 128 integers into {@code longs}. - */ - void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs) throws IOException { + /** Delta-decode 128 integers into {@code longs}. */ + void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs) + throws IOException { switch (bitsPerValue) { """) for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1): @@ -504,14 +514,14 @@ if __name__ == '__main__': next_primitive = 8 elif bpv <= 16: next_primitive = 16 - f.write(' case %d:\n' %bpv) - f.write(' decode%d(in, tmp, longs);\n' %bpv) - f.write(' prefixSum%d(longs, base);\n' %next_primitive) - f.write(' break;\n') - f.write(' default:\n') - f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n') - f.write(' prefixSum32(longs, base);\n') - f.write(' break;\n') + f.write(' case %d:\n' %bpv) + f.write(' decode%d(in, tmp, longs);\n' %bpv) + f.write(' prefixSum%d(longs, base);\n' %next_primitive) + f.write(' break;\n') + f.write(' default:\n') + f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n') + f.write(' prefixSum32(longs, base);\n') + f.write(' break;\n') f.write(' }\n') f.write(' }\n') diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java index b02232115c9..d7d0b76853e 100644 --- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java @@ -74,14 +74,19 @@ final class EndiannessReverserIndexInput extends IndexInput { @Override public void readLongs(long[] dst, int offset, int length) throws IOException { - // used to be called readLELongs in.readLongs(dst, offset, length); + for (int i = 0; i < length; ++i) { + dst[offset + i] = Long.reverseBytes(dst[offset + i]); + } } @Override public void readFloats(float[] dst, int offset, int length) throws IOException { - // used to be called readLEFloats in.readFloats(dst, offset, length); + for (int i = 0; i < length; ++i) { + dst[offset + i] = + Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(dst[offset + i]))); + } } @Override