Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-10376: Roll up the loop in vint/vlong in DataInput (#602)

This commit is contained in:
gf2121 2022-10-24 17:39:22 +08:00 committed by GitHub
parent 0f525bfb14
commit 8cfbc18497
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 27 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -796,6 +796,8 @@ Bug Fixes
Other
---------------------
* LUCENE-10376: Roll up the loop in VInt/VLong in DataInput. (Guo Feng)
* LUCENE-10273: Deprecate SpanishMinimalStemFilter in favor of SpanishPluralStemFilter. (Robert Muir)
* LUCENE-10284: Upgrade morfologik-stemming to 2.1.8. (Dawid Weiss)

61
lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
View File

@ -159,67 +159,6 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAcc
}
}
@Override
public final int readVInt() throws IOException {
if (5 <= buffer.remaining()) {
byte b = buffer.get();
if (b >= 0) return b;
int i = b & 0x7F;
b = buffer.get();
i |= (b & 0x7F) << 7;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7F) << 14;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7F) << 21;
if (b >= 0) return i;
b = buffer.get();
// Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
i |= (b & 0x0F) << 28;
if ((b & 0xF0) == 0) return i;
throw new IOException("Invalid vInt detected (too many bits)");
} else {
return super.readVInt();
}
}
@Override
public final long readVLong() throws IOException {
if (9 <= buffer.remaining()) {
byte b = buffer.get();
if (b >= 0) return b;
long i = b & 0x7FL;
b = buffer.get();
i |= (b & 0x7FL) << 7;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 14;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 21;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 28;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 35;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 42;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 49;
if (b >= 0) return i;
b = buffer.get();
i |= (b & 0x7FL) << 56;
if (b >= 0) return i;
throw new IOException("Invalid vLong detected (negative values disallowed)");
} else {
return super.readVLong();
}
}
@Override
public final byte readByte(long pos) throws IOException {
long index = pos - bufferStart;

60
lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.lucene.store;
import java.io.IOException;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
@ -109,55 +110,24 @@ public final class ByteArrayDataInput extends DataInput {
@Override
public int readVInt() {
byte b = bytes[pos++];
if (b >= 0) return b;
int i = b & 0x7F;
b = bytes[pos++];
i |= (b & 0x7F) << 7;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7F) << 14;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7F) << 21;
if (b >= 0) return i;
b = bytes[pos++];
// Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
i |= (b & 0x0F) << 28;
if ((b & 0xF0) == 0) return i;
throw new RuntimeException("Invalid vInt detected (too many bits)");
try {
return super.readVInt();
} catch (
@SuppressWarnings("unused")
IOException e) {
throw new AssertionError("ByteArrayDataInput#readByte should not throw IOException");
}
}
@Override
public long readVLong() {
byte b = bytes[pos++];
if (b >= 0) return b;
long i = b & 0x7FL;
b = bytes[pos++];
i |= (b & 0x7FL) << 7;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 14;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 21;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 28;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 35;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 42;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 49;
if (b >= 0) return i;
b = bytes[pos++];
i |= (b & 0x7FL) << 56;
if (b >= 0) return i;
throw new RuntimeException("Invalid vLong detected (negative values disallowed)");
try {
return super.readVLong();
} catch (
@SuppressWarnings("unused")
IOException e) {
throw new AssertionError("ByteArrayDataInput#readByte should not throw IOException");
}
}
// NOTE: AIOOBE not EOF if you read too much

66
lucene/core/src/java/org/apache/lucene/store/DataInput.java
View File

@ -107,9 +107,6 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeVInt(int)
*/
public int readVInt() throws IOException {
/* This is the original code of this method,
* but a Hotspot bug (see LUCENE-2975) corrupts the for-loop if
* readByte() is inlined. So the loop was unwinded!
byte b = readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@ -117,24 +114,6 @@ public abstract class DataInput implements Cloneable {
i |= (b & 0x7F) << shift;
}
return i;
*/
byte b = readByte();
if (b >= 0) return b;
int i = b & 0x7F;
b = readByte();
i |= (b & 0x7F) << 7;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7F) << 14;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7F) << 21;
if (b >= 0) return i;
b = readByte();
// Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
i |= (b & 0x0F) << 28;
if ((b & 0xF0) == 0) return i;
throw new IOException("Invalid vInt detected (too many bits)");
}
/**
@ -206,13 +185,6 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeVLong(long)
*/
public long readVLong() throws IOException {
return readVLong(false);
}
private long readVLong(boolean allowNegative) throws IOException {
/* This is the original code of this method,
* but a Hotspot bug (see LUCENE-2975) corrupts the for-loop if
* readByte() is inlined. So the loop was unwinded!
byte b = readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@ -220,42 +192,6 @@ public abstract class DataInput implements Cloneable {
i |= (b & 0x7FL) << shift;
}
return i;
*/
byte b = readByte();
if (b >= 0) return b;
long i = b & 0x7FL;
b = readByte();
i |= (b & 0x7FL) << 7;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 14;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 21;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 28;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 35;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 42;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 49;
if (b >= 0) return i;
b = readByte();
i |= (b & 0x7FL) << 56;
if (b >= 0) return i;
if (allowNegative) {
b = readByte();
i |= (b & 0x7FL) << 63;
if (b == 0 || b == 1) return i;
throw new IOException("Invalid vLong detected (more than 64 bits)");
} else {
throw new IOException("Invalid vLong detected (negative values disallowed)");
}
}
/**
@ -265,7 +201,7 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeZLong(long)
*/
public long readZLong() throws IOException {
return BitUtil.zigZagDecode(readVLong(true));
return BitUtil.zigZagDecode(readVLong());
}
/**

36
lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java
View File

@ -40,33 +40,15 @@ public final class BlockPackedReaderIterator {
// same as DataInput.readVLong but supports negative values
static long readVLong(DataInput in) throws IOException {
byte b = in.readByte();
if (b >= 0) return b;
long i = b & 0x7FL;
b = in.readByte();
i |= (b & 0x7FL) << 7;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 14;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 21;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 28;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 35;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 42;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0x7FL) << 49;
if (b >= 0) return i;
b = in.readByte();
i |= (b & 0xFFL) << 56;
return i;
long l = 0L;
for (int shift = 0; shift < 56; shift += 7) {
byte b = in.readByte();
l |= (b & 0x7FL) << shift;
if (b >= 0) {
return l;
}
}
return l | ((in.readByte() & 0xFFL) << 56);
}
DataInput in;

36
lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
View File

@ -155,24 +155,6 @@ public class TestIndexInput extends LuceneTestCase {
0x00,
'n',
'e',
// tests for Exceptions on invalid values
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0x17,
(byte) 0x01, // guard value
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
(byte) 0x01, // guard value
};
static final int COUNT = RANDOM_MULTIPLIER * 65536;
@ -234,24 +216,6 @@ public class TestIndexInput extends LuceneTestCase {
assertEquals("\u0000", is.readString());
assertEquals("Lu\u0000ce\u0000ne", is.readString());
Exception expected =
expectThrows(
expectedEx,
() -> {
is.readVInt();
});
assertTrue(expected.getMessage().startsWith("Invalid vInt"));
assertEquals(1, is.readVInt()); // guard value
expected =
expectThrows(
expectedEx,
() -> {
is.readVLong();
});
assertTrue(expected.getMessage().startsWith("Invalid vLong"));
assertEquals(1L, is.readVLong()); // guard value
}
private void checkRandomReads(DataInput is) throws IOException {