Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2905: write pointers and skip data more efficiently for fixed and variable intblock 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1069930 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2011-02-11 19:41:55 +00:00
parent 00ad6999ca
commit f7a130e393
4 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java
View File

@ -152,17 +152,17 @@ public abstract class FixedIntBlockIndexInput extends IntIndexInput {
@Override @Override
public void read(final DataInput indexIn, final boolean absolute) throws IOException { public void read(final DataInput indexIn, final boolean absolute) throws IOException {
if (absolute) { if (absolute) {
fp = indexIn.readVLong();
upto = indexIn.readVInt(); upto = indexIn.readVInt();
fp = indexIn.readVLong();
} else { } else {
final long delta = indexIn.readVLong(); final int uptoDelta = indexIn.readVInt();
if (delta == 0) { if ((uptoDelta & 1) == 1) {
// same block // same block
upto += indexIn.readVInt(); upto += uptoDelta >>> 1;
} else { } else {
// new block // new block
fp += delta; upto = uptoDelta >>> 1;
upto = indexIn.readVInt(); fp += indexIn.readVLong();
} }
} }
assert upto < blockSize; assert upto < blockSize;

8
lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java
View File

@ -77,17 +77,17 @@ public abstract class FixedIntBlockIndexOutput extends IntIndexOutput {
@Override @Override
public void write(IndexOutput indexOut, boolean absolute) throws IOException { public void write(IndexOutput indexOut, boolean absolute) throws IOException {
if (absolute) { if (absolute) {
indexOut.writeVLong(fp);
indexOut.writeVInt(upto); indexOut.writeVInt(upto);
indexOut.writeVLong(fp);
} else if (fp == lastFP) { } else if (fp == lastFP) {
// same block // same block
indexOut.writeVLong(0);
assert upto >= lastUpto; assert upto >= lastUpto;
indexOut.writeVInt(upto - lastUpto); int uptoDelta = upto - lastUpto;
indexOut.writeVInt(uptoDelta << 1 | 1);
} else { } else {
// new block // new block
indexOut.writeVInt(upto << 1);
indexOut.writeVLong(fp - lastFP); indexOut.writeVLong(fp - lastFP);
indexOut.writeVInt(upto);
} }
lastUpto = upto; lastUpto = upto;
lastFP = fp; lastFP = fp;

12
lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexInput.java
View File

@ -171,17 +171,17 @@ public abstract class VariableIntBlockIndexInput extends IntIndexInput {
@Override @Override
public void read(final DataInput indexIn, final boolean absolute) throws IOException { public void read(final DataInput indexIn, final boolean absolute) throws IOException {
if (absolute) { if (absolute) {
upto = indexIn.readVInt();
fp = indexIn.readVLong(); fp = indexIn.readVLong();
upto = indexIn.readByte()&0xFF;
} else { } else {
final long delta = indexIn.readVLong(); final int uptoDelta = indexIn.readVInt();
if (delta == 0) { if ((uptoDelta & 1) == 1) {
// same block // same block
upto = indexIn.readByte()&0xFF; upto += uptoDelta >>> 1;
} else { } else {
// new block // new block
fp += delta; upto = uptoDelta >>> 1;
upto = indexIn.readByte()&0xFF; fp += indexIn.readVLong();
} }
} }
// TODO: we can't do this assert because non-causal // TODO: we can't do this assert because non-causal

18
lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java
View File

@ -42,16 +42,14 @@ public abstract class VariableIntBlockIndexOutput extends IntIndexOutput {
private int upto; private int upto;
private static final int MAX_BLOCK_SIZE = 1 << 8; // TODO what Var-Var codecs exist in practice... and what are there blocksizes like?
// if its less than 128 we should set that as max and use byte?
/** NOTE: maxBlockSize plus the max non-causal lookahead /** NOTE: maxBlockSize must be the maximum block size
* of your codec must be less than 256. EG Simple9 * plus the max non-causal lookahead of your codec. EG Simple9
* requires lookahead=1 because on seeing the Nth value * requires lookahead=1 because on seeing the Nth value
* it knows it must now encode the N-1 values before it. */ * it knows it must now encode the N-1 values before it. */
protected VariableIntBlockIndexOutput(IndexOutput out, int maxBlockSize) throws IOException { protected VariableIntBlockIndexOutput(IndexOutput out, int maxBlockSize) throws IOException {
if (maxBlockSize > MAX_BLOCK_SIZE) {
throw new IllegalArgumentException("maxBlockSize must be <= " + MAX_BLOCK_SIZE + "; got " + maxBlockSize);
}
this.out = out; this.out = out;
out.writeInt(maxBlockSize); out.writeInt(maxBlockSize);
} }
@ -88,17 +86,17 @@ public abstract class VariableIntBlockIndexOutput extends IntIndexOutput {
public void write(IndexOutput indexOut, boolean absolute) throws IOException { public void write(IndexOutput indexOut, boolean absolute) throws IOException {
assert upto >= 0; assert upto >= 0;
if (absolute) { if (absolute) {
indexOut.writeVInt(upto);
indexOut.writeVLong(fp); indexOut.writeVLong(fp);
indexOut.writeByte((byte) upto);
} else if (fp == lastFP) { } else if (fp == lastFP) {
// same block // same block
indexOut.writeVLong(0);
assert upto >= lastUpto; assert upto >= lastUpto;
indexOut.writeByte((byte) upto); int uptoDelta = upto - lastUpto;
indexOut.writeVInt(uptoDelta << 1 | 1);
} else { } else {
// new block // new block
indexOut.writeVInt(upto << 1);
indexOut.writeVLong(fp - lastFP); indexOut.writeVLong(fp - lastFP);
indexOut.writeByte((byte) upto);
} }
lastUpto = upto; lastUpto = upto;
lastFP = fp; lastFP = fp;