From f7a130e393ee5dbe18772e5df50ddb89571d6be8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 11 Feb 2011 19:41:55 +0000 Subject: [PATCH] LUCENE-2905: write pointers and skip data more efficiently for fixed and variable intblock git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1069930 13f79535-47bb-0310-9956-ffa450edef68 --- .../intblock/FixedIntBlockIndexInput.java | 12 ++++++------ .../intblock/FixedIntBlockIndexOutput.java | 8 ++++---- .../intblock/VariableIntBlockIndexInput.java | 12 ++++++------ .../intblock/VariableIntBlockIndexOutput.java | 18 ++++++++---------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java b/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java index 652fdea1a70..0d3ef2ec465 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java @@ -152,17 +152,17 @@ public abstract class FixedIntBlockIndexInput extends IntIndexInput { @Override public void read(final DataInput indexIn, final boolean absolute) throws IOException { if (absolute) { - fp = indexIn.readVLong(); upto = indexIn.readVInt(); + fp = indexIn.readVLong(); } else { - final long delta = indexIn.readVLong(); - if (delta == 0) { + final int uptoDelta = indexIn.readVInt(); + if ((uptoDelta & 1) == 1) { // same block - upto += indexIn.readVInt(); + upto += uptoDelta >>> 1; } else { // new block - fp += delta; - upto = indexIn.readVInt(); + upto = uptoDelta >>> 1; + fp += indexIn.readVLong(); } } assert upto < blockSize; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java b/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java index 1c5f757108c..ffbce61c493 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java @@ -77,17 +77,17 @@ public abstract class FixedIntBlockIndexOutput extends IntIndexOutput { @Override public void write(IndexOutput indexOut, boolean absolute) throws IOException { if (absolute) { - indexOut.writeVLong(fp); indexOut.writeVInt(upto); + indexOut.writeVLong(fp); } else if (fp == lastFP) { // same block - indexOut.writeVLong(0); assert upto >= lastUpto; - indexOut.writeVInt(upto - lastUpto); + int uptoDelta = upto - lastUpto; + indexOut.writeVInt(uptoDelta << 1 | 1); } else { // new block + indexOut.writeVInt(upto << 1); indexOut.writeVLong(fp - lastFP); - indexOut.writeVInt(upto); } lastUpto = upto; lastFP = fp; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexInput.java b/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexInput.java index cbdb45271ba..d4b7fcb41ba 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexInput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexInput.java @@ -171,17 +171,17 @@ public abstract class VariableIntBlockIndexInput extends IntIndexInput { @Override public void read(final DataInput indexIn, final boolean absolute) throws IOException { if (absolute) { + upto = indexIn.readVInt(); fp = indexIn.readVLong(); - upto = indexIn.readByte()&0xFF; } else { - final long delta = indexIn.readVLong(); - if (delta == 0) { + final int uptoDelta = indexIn.readVInt(); + if ((uptoDelta & 1) == 1) { // same block - upto = indexIn.readByte()&0xFF; + upto += uptoDelta >>> 1; } else { // new block - fp += delta; - upto = indexIn.readByte()&0xFF; + upto = uptoDelta >>> 1; + fp += indexIn.readVLong(); } } // TODO: we can't do this assert because non-causal diff --git a/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java b/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java index 1dfbf32b3c2..d39db6cd750 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java @@ -42,16 +42,14 @@ public abstract class VariableIntBlockIndexOutput extends IntIndexOutput { private int upto; - private static final int MAX_BLOCK_SIZE = 1 << 8; + // TODO what Var-Var codecs exist in practice... and what are there blocksizes like? + // if its less than 128 we should set that as max and use byte? - /** NOTE: maxBlockSize plus the max non-causal lookahead - * of your codec must be less than 256. EG Simple9 + /** NOTE: maxBlockSize must be the maximum block size + * plus the max non-causal lookahead of your codec. EG Simple9 * requires lookahead=1 because on seeing the Nth value * it knows it must now encode the N-1 values before it. */ protected VariableIntBlockIndexOutput(IndexOutput out, int maxBlockSize) throws IOException { - if (maxBlockSize > MAX_BLOCK_SIZE) { - throw new IllegalArgumentException("maxBlockSize must be <= " + MAX_BLOCK_SIZE + "; got " + maxBlockSize); - } this.out = out; out.writeInt(maxBlockSize); } @@ -88,17 +86,17 @@ public abstract class VariableIntBlockIndexOutput extends IntIndexOutput { public void write(IndexOutput indexOut, boolean absolute) throws IOException { assert upto >= 0; if (absolute) { + indexOut.writeVInt(upto); indexOut.writeVLong(fp); - indexOut.writeByte((byte) upto); } else if (fp == lastFP) { // same block - indexOut.writeVLong(0); assert upto >= lastUpto; - indexOut.writeByte((byte) upto); + int uptoDelta = upto - lastUpto; + indexOut.writeVInt(uptoDelta << 1 | 1); } else { // new block + indexOut.writeVInt(upto << 1); indexOut.writeVLong(fp - lastFP); - indexOut.writeByte((byte) upto); } lastUpto = upto; lastFP = fp;