Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added Bit32Encoder and moved the unrolled version of Bit21Encoder to DocIdsWriter

This commit is contained in:
expani 2024-08-29 19:41:32 +05:30
parent 5fb5b2599d
commit 5af1e6043f
2 changed files with 55 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DocIdEncodingBenchmark.java
View File

@ -74,7 +74,7 @@ public class DocIdEncodingBenchmark {
}
}
@Param({"Bit21With3StepsEncoder", "Bit21With2StepsEncoder", "Bit24Encoder"})
@Param({"Bit21With3StepsEncoder", "Bit21With2StepsEncoder", "Bit24Encoder", "Bit32Encoder"})
String encoderName;
private static final int INPUT_SCALE_FACTOR = 2_00_000;
@ -148,11 +148,14 @@ public class DocIdEncodingBenchmark {
static final Map<String, DocIdEncoder> ENCODER_NAME_TO_INSTANCE_MAPPING =
Map.of(
Bit24Encoder.class.getSimpleName().toLowerCase(Locale.ROOT), new Bit24Encoder(),
Bit24Encoder.class.getSimpleName().toLowerCase(Locale.ROOT),
new Bit24Encoder(),
Bit21With2StepsEncoder.class.getSimpleName().toLowerCase(Locale.ROOT),
new Bit21With2StepsEncoder(),
new Bit21With2StepsEncoder(),
Bit21With3StepsEncoder.class.getSimpleName().toLowerCase(Locale.ROOT),
new Bit21With3StepsEncoder());
new Bit21With3StepsEncoder(),
Bit32Encoder.class.getSimpleName().toLowerCase(Locale.ROOT),
new Bit32Encoder());
public static DocIdEncoder fromName(String encoderName) {
String parsedEncoderName = encoderName.trim().toLowerCase(Locale.ROOT);
@ -314,4 +317,21 @@ public class DocIdEncodingBenchmark {
}
}
}
static class Bit32Encoder implements DocIdEncoder {
@Override
public void encode(IndexOutput out, int start, int count, int[] docIds) throws IOException {
for (int i = 0; i < count; i++) {
out.writeInt(docIds[i]);
}
}
@Override
public void decode(IndexInput in, int start, int count, int[] docIds) throws IOException {
for (int i = 0; i < count; i++) {
docIds[i] = in.readInt();
}
}
}
}

31
lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
View File

@ -112,6 +112,23 @@ final class DocIdsWriter {
if (max <= 0x001FFFFF) {
out.writeByte(BPV_21);
int i = 0;
for (; i < count - 8; i += 9) {
long l1 =
((docIds[i] & 0x001FFFFFL) << 42)
| ((docIds[i + 1] & 0x001FFFFFL) << 21)
| (docIds[i + 2] & 0x001FFFFFL);
long l2 =
((docIds[i + 3] & 0x001FFFFFL) << 42)
| ((docIds[i + 4] & 0x001FFFFFL) << 21)
| (docIds[i + 5] & 0x001FFFFFL);
long l3 =
((docIds[i + 6] & 0x001FFFFFL) << 42)
| ((docIds[i + 7] & 0x001FFFFFL) << 21)
| (docIds[i + 8] & 0x001FFFFFL);
out.writeLong(l1);
out.writeLong(l2);
out.writeLong(l3);
}
for (; i < count - 2; i += 3) {
long packedLong =
((docIds[i] & 0x001FFFFFL) << 42)
@ -272,6 +289,20 @@ final class DocIdsWriter {
private void readInts21(IndexInput in, int count, int[] docIDs) throws IOException {
int i = 0;
for (; i < count - 8; i += 9) {
long l1 = in.readLong();
long l2 = in.readLong();
long l3 = in.readLong();
docIDs[i] = (int) (l1 >>> 42);
docIDs[i + 1] = (int) ((l1 & 0x000003FFFFE00000L) >>> 21);
docIDs[i + 2] = (int) (l1 & 0x001FFFFFL);
docIDs[i + 3] = (int) (l2 >>> 42);
docIDs[i + 4] = (int) ((l2 & 0x000003FFFFE00000L) >>> 21);
docIDs[i + 5] = (int) (l2 & 0x001FFFFFL);
docIDs[i + 6] = (int) (l3 >>> 42);
docIDs[i + 7] = (int) ((l3 & 0x000003FFFFE00000L) >>> 21);
docIDs[i + 8] = (int) (l3 & 0x001FFFFFL);
}
for (; i < count - 2; i += 3) {
long packedLong = in.readLong();
docIDs[i] = (int) (packedLong >>> 42);