Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5739: Add DataInput.readZ(Int|Long) and DataOutput.writeZ(Int|Long). 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1614870 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2014-07-31 12:49:22 +00:00
parent e6d29d223b
commit cc3501de97
10 changed files with 135 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -24,6 +24,9 @@ New Features
implemented in the spatial module as DateRangePrefixTree used with
NumberRangePrefixTreeStrategy. (David Smiley)
* LUCENE-5739: Added DataInput.readZ(Int|Long) and DataOutput.writeZ(Int|Long)
to read and write small signed integers. (Adrien Grand)
API Changes
* LUCENE-4535: oal.util.FilterIterator is now an internal API.

2
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
View File

@ -54,7 +54,7 @@ public final class ConnectionCosts {
for (int j = 0; j < costs.length; j++) {
final short[] a = costs[j];
for (int i = 0; i < a.length; i++) {
accum += BitUtil.zigZagDecode(in.readVInt());
accum += in.readZInt();
a[i] = (short)accum;
}
}

2
lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
View File

@ -65,7 +65,7 @@ public final class ConnectionCostsWriter {
assert a.length == forwardSize;
for (int i = 0; i < a.length; i++) {
int delta = (int)a[i] - last;
out.writeVInt(BitUtil.zigZagEncode(delta));
out.writeZInt(delta);
last = a[i];
}
}

34
lucene/core/src/java/org/apache/lucene/store/DataInput.java
View File

@ -24,6 +24,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.BitUtil;
/**
* Abstract base class for performing read operations of Lucene's low-level
* data types.
@ -136,6 +138,15 @@ public abstract class DataInput implements Cloneable {
throw new IOException("Invalid vInt detected (too many bits)");
}
/**
* Read a {@link BitUtil#zigZagDecode(int) zig-zag}-encoded
* {@link #readVInt() variable-length} integer.
* @see DataOutput#writeZInt(int)
*/
public int readZInt() throws IOException {
return BitUtil.zigZagDecode(readVInt());
}
/** Reads eight bytes and returns a long.
* @see DataOutput#writeLong(long)
*/
@ -152,6 +163,10 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeVLong(long)
*/
public long readVLong() throws IOException {
return readVLong(false);
}
private long readVLong(boolean allowNegative) throws IOException {
/* This is the original code of this method,
* but a Hotspot bug (see LUCENE-2975) corrupts the for-loop if
* readByte() is inlined. So the loop was unwinded!
@ -190,7 +205,24 @@ public abstract class DataInput implements Cloneable {
b = readByte();
i |= (b & 0x7FL) << 56;
if (b >= 0) return i;
throw new IOException("Invalid vLong detected (negative values disallowed)");
if (allowNegative) {
b = readByte();
i |= (b & 0x7FL) << 63;
if (b == 0 || b == 1) return i;
throw new IOException("Invalid vLong detected (more than 64 bits)");
} else {
throw new IOException("Invalid vLong detected (negative values disallowed)");
}
}
/**
* Read a {@link BitUtil#zigZagDecode(long) zig-zag}-encoded
* {@link #readVLong() variable-length} integer. Reads between one and ten
* bytes.
* @see DataOutput#writeZLong(long)
*/
public long readZLong() throws IOException {
return BitUtil.zigZagDecode(readVLong(true));
}
/** Reads a string.

27
lucene/core/src/java/org/apache/lucene/store/DataOutput.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
@ -195,6 +196,17 @@ public abstract class DataOutput {
writeByte((byte)i);
}
/**
* Write a {@link BitUtil#zigZagEncode(int) zig-zag}-encoded
* {@link #writeVInt(int) variable-length} integer. This is typically useful
* to write small signed ints and is equivalent to calling
* <code>writeVInt(BitUtil.zigZagEncode(i))</code>.
* @see DataInput#readZInt()
*/
public final void writeZInt(int i) throws IOException {
writeVInt(BitUtil.zigZagEncode(i));
}
/** Writes a long as eight bytes.
* <p>
* 64-bit unsigned integer written as eight bytes, high-order bytes first.
@ -215,6 +227,11 @@ public abstract class DataOutput {
*/
public final void writeVLong(long i) throws IOException {
assert i >= 0L;
writeNegativeVLong(i);
}
// write a pontentially negative vLong
private void writeNegativeVLong(long i) throws IOException {
while ((i & ~0x7FL) != 0L) {
writeByte((byte)((i & 0x7FL) | 0x80L));
i >>>= 7;
@ -222,6 +239,16 @@ public abstract class DataOutput {
writeByte((byte)i);
}
/**
* Write a {@link BitUtil#zigZagEncode(long) zig-zag}-encoded
* {@link #writeVLong(long) variable-length} long. Writes between one and ten
* bytes. This is typically useful to write small signed ints.
* @see DataInput#readZLong()
*/
public final void writeZLong(long i) throws IOException {
writeNegativeVLong(BitUtil.zigZagEncode(i));
}
/** Writes a string.
* <p>
* Writes strings as UTF-8 encoded bytes. First the length, in bytes, is

2
lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
View File

@ -72,7 +72,7 @@ public class MonotonicBlockPackedReader extends LongValues implements Accountabl
if (packedIntsVersion < PackedInts.VERSION_MONOTONIC_WITHOUT_ZIGZAG) {
minValues[i] = in.readVLong();
} else {
minValues[i] = zigZagDecode(in.readVLong());
minValues[i] = in.readZLong();
}
averages[i] = Float.intBitsToFloat(in.readInt());
final int bitsPerValue = in.readVInt();

2
lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedWriter.java
View File

@ -90,7 +90,7 @@ public final class MonotonicBlockPackedWriter extends AbstractBlockPackedWriter
maxDelta = Math.max(maxDelta, values[i]);
}
out.writeVLong(zigZagEncode(min));
out.writeZLong(min);
out.writeInt(Float.floatToIntBits(avg));
if (maxDelta == 0) {
out.writeVInt(0);

2
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
View File

@ -58,7 +58,7 @@ final class IDVersionPostingsReader extends PostingsReaderBase {
if (absolute) {
termState.idVersion = in.readVLong();
} else {
termState.idVersion += BitUtil.zigZagDecode(in.readVLong());
termState.idVersion += in.readZLong();
}
}

2
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
View File

@ -158,7 +158,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
out.writeVLong(state.idVersion);
} else {
long delta = state.idVersion - lastEncodedVersion;
out.writeVLong(BitUtil.zigZagEncode(delta));
out.writeZLong(delta);
}
lastEncodedVersion = state.idVersion;
}

66
lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
View File

@ -230,6 +230,72 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
dir.close();
}
public void testZInt() throws Exception {
final int[] ints = new int[random().nextInt(10)];
for (int i = 0; i < ints.length; ++i) {
switch (random().nextInt(3)) {
case 0:
ints[i] = random().nextInt();
break;
case 1:
ints[i] = random().nextBoolean() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
break;
case 2:
ints[i] = (random().nextBoolean() ? -1 : 1) * random().nextInt(1024);
break;
default:
throw new AssertionError();
}
}
Directory dir = getDirectory(createTempDir("testZInt"));
IndexOutput output = dir.createOutput("zint", newIOContext(random()));
for (int i : ints) {
output.writeZInt(i);
}
output.close();
IndexInput input = dir.openInput("zint", newIOContext(random()));
for (int i : ints) {
assertEquals(i, input.readZInt());
}
assertEquals(input.length(), input.getFilePointer());
input.close();
dir.close();
}
public void testZLong() throws Exception {
final long[] longs = new long[random().nextInt(10)];
for (int i = 0; i < longs.length; ++i) {
switch (random().nextInt(3)) {
case 0:
longs[i] = random().nextLong();
break;
case 1:
longs[i] = random().nextBoolean() ? Long.MIN_VALUE : Long.MAX_VALUE;
break;
case 2:
longs[i] = (random().nextBoolean() ? -1 : 1) * random().nextInt(1024);
break;
default:
throw new AssertionError();
}
}
Directory dir = getDirectory(createTempDir("testZLong"));
IndexOutput output = dir.createOutput("zlong", newIOContext(random()));
for (long l : longs) {
output.writeZLong(l);
}
output.close();
IndexInput input = dir.openInput("zlong", newIOContext(random()));
for (long l : longs) {
assertEquals(l, input.readZLong());
}
assertEquals(input.length(), input.getFilePointer());
input.close();
dir.close();
}
public void testStringSet() throws Exception {
Directory dir = getDirectory(createTempDir("testStringSet"));
IndexOutput output = dir.createOutput("stringset", newIOContext(random()));