Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-03-03 06:49:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4225: fix offset delta encoding to be startOffset - lastStartOffset, not startOffset - lastEndOffset

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1368678 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-08-02 19:58:22 +00:00
parent 9130567fa6
commit ccb43109e2
9 changed files with 90 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java
View File

@ -958,7 +958,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
private int payloadByteUpto;
private int payloadLength;
private int lastEndOffset;
private int lastStartOffset;
private int startOffset;
private int endOffset;
@ -1248,7 +1248,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
}
position = 0;
payloadLength = 0;
lastEndOffset = 0;
lastStartOffset = 0;
return doc;
}
@ -1305,7 +1305,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
// Skipper moved
if (DEBUG) {
System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastEndOffset=" + lastEndOffset);
System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset);
}
assert newDocUpto % blockSize == (blockSize-1): "got " + newDocUpto;
@ -1318,7 +1318,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
posPendingFP = skipper.getPosPointer();
payPendingFP = skipper.getPayPointer();
posPendingCount = skipper.getPosBufferUpto();
lastEndOffset = skipper.getEndOffset();
lastStartOffset = skipper.getStartOffset();
payloadByteUpto = skipper.getPayloadByteUpto();
}
}
@ -1359,7 +1359,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@ -1386,11 +1386,11 @@ public final class BlockPostingsReader extends PostingsReaderBase {
if (indexHasOffsets) {
// Must load offset blocks merely to sum
// up into lastEndOffset:
// up into lastStartOffset:
readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaBuffer);
readBlock(payIn, encoded, encodedBuffer, offsetLengthBuffer);
for(int i=0;i<blockSize;i++) {
lastEndOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
lastStartOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
}
}
toSkip -= blockSize;
@ -1403,7 +1403,7 @@ public final class BlockPostingsReader extends PostingsReaderBase {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@ -1414,7 +1414,9 @@ public final class BlockPostingsReader extends PostingsReaderBase {
position = 0;
payloadLength = 0;
lastEndOffset = 0;
// nocommit why carefully sum up lastStartOffset above
// only to set it to 0 now?
lastStartOffset = 0;
}
@Override
@ -1467,9 +1469,9 @@ public final class BlockPostingsReader extends PostingsReaderBase {
}
if (indexHasOffsets) {
startOffset = lastEndOffset + offsetStartDeltaBuffer[posBufferUpto];
startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
lastEndOffset = endOffset;
lastStartOffset = startOffset;
}
posBufferUpto++;

18
lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java
View File

@ -95,11 +95,11 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
private long lastBlockPosFP;
private long lastBlockPayFP;
private int lastBlockPosBufferUpto;
private int lastBlockEndOffset;
private int lastBlockStartOffset;
private int lastBlockPayloadByteUpto;
private int lastDocID;
private int lastPosition;
private int lastEndOffset;
private int lastStartOffset;
private int docCount;
final byte[] encoded;
@ -230,7 +230,7 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
lastBlockPayFP = payOut.getFilePointer();
}
lastBlockPosBufferUpto = posBufferUpto;
lastBlockEndOffset = lastEndOffset;
lastBlockStartOffset = lastStartOffset;
lastBlockPayloadByteUpto = payloadByteUpto;
saveNextPosBlock = false;
if (DEBUG) {
@ -269,7 +269,7 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
if (DEBUG) {
System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-blockSize));
}
skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
lastBlockDocID = docID;
saveNextPosBlock = true;
@ -288,7 +288,7 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
}
lastPosition = 0;
lastEndOffset = 0;
lastStartOffset = 0;
}
/** Add a new position & payload */
@ -313,11 +313,11 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
}
if (fieldHasOffsets) {
assert startOffset >= lastEndOffset;
assert startOffset >= lastStartOffset;
assert endOffset >= startOffset;
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastEndOffset;
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
lastEndOffset = endOffset;
lastStartOffset = startOffset;
}
posBufferUpto++;
@ -388,7 +388,7 @@ public final class BlockPostingsWriter extends PostingsWriterBase {
if (DEBUG) {
System.out.println(" bufferSkip at finishTerm: lastDocID=" + lastBlockDocID + " docCount=" + lastDocCount);
}
skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
if (DEBUG) {

24
lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java
View File

@ -37,12 +37,12 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
private long posPointer[];
private long payPointer[];
private int posBufferUpto[];
private int endOffset[];
private int startOffset[];
private int payloadByteUpto[];
private long lastPosPointer;
private long lastPayPointer;
private int lastEndOffset;
private int lastStartOffset;
private int lastPayloadByteUpto;
private long lastDocPointer;
private int lastPosBufferUpto;
@ -59,9 +59,9 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
payloadByteUpto = null;
}
if (hasOffsets) {
endOffset = new int[maxSkipLevels];
startOffset = new int[maxSkipLevels];
} else {
endOffset = null;
startOffset = null;
}
if (hasOffsets || hasPayloads) {
payPointer = new long[maxSkipLevels];
@ -108,8 +108,8 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
return lastPayPointer;
}
public int getEndOffset() {
return lastEndOffset;
public int getStartOffset() {
return lastStartOffset;
}
public int getPayloadByteUpto() {
@ -126,8 +126,8 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
if (posPointer != null) {
posPointer[level] = lastPosPointer;
posBufferUpto[level] = lastPosBufferUpto;
if (endOffset != null) {
endOffset[level] = lastEndOffset;
if (startOffset != null) {
startOffset[level] = lastStartOffset;
}
if (payloadByteUpto != null) {
payloadByteUpto[level] = lastPayloadByteUpto;
@ -155,8 +155,8 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
if (payPointer != null) {
lastPayPointer = payPointer[level];
}
if (endOffset != null) {
lastEndOffset = endOffset[level];
if (startOffset != null) {
lastStartOffset = startOffset[level];
}
if (payloadByteUpto != null) {
lastPayloadByteUpto = payloadByteUpto[level];
@ -192,8 +192,8 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
payloadByteUpto[level] = skipStream.readVInt();
}
if (endOffset != null) {
endOffset[level] += skipStream.readVInt();
if (startOffset != null) {
startOffset[level] += skipStream.readVInt();
}
if (payPointer != null) {

16
lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java
View File

@ -34,7 +34,7 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
private long[] lastSkipDocPointer;
private long[] lastSkipPosPointer;
private long[] lastSkipPayPointer;
private int[] lastEndOffset;
private int[] lastStartOffset;
private int[] lastPayloadByteUpto;
private final IndexOutput docOut;
@ -46,7 +46,7 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
private long curPosPointer;
private long curPayPointer;
private int curPosBufferUpto;
private int curEndOffset;
private int curStartOffset;
private int curPayloadByteUpto;
private boolean fieldHasPositions;
private boolean fieldHasOffsets;
@ -65,7 +65,7 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
if (payOut != null) {
lastSkipPayPointer = new long[maxSkipLevels];
}
lastEndOffset = new int[maxSkipLevels];
lastStartOffset = new int[maxSkipLevels];
lastPayloadByteUpto = new int[maxSkipLevels];
}
}
@ -84,7 +84,7 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
if (fieldHasPositions) {
Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
if (fieldHasOffsets) {
Arrays.fill(lastEndOffset, 0);
Arrays.fill(lastStartOffset, 0);
}
if (fieldHasPayloads) {
Arrays.fill(lastPayloadByteUpto, 0);
@ -98,14 +98,14 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
/**
* Sets the values for the current skip data.
*/
public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int endOffset, int payloadByteUpto) throws IOException {
public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int startOffset, int payloadByteUpto) throws IOException {
this.curDoc = doc;
this.curDocPointer = docOut.getFilePointer();
this.curPosPointer = posFP;
this.curPayPointer = payFP;
this.curPosBufferUpto = posBufferUpto;
this.curPayloadByteUpto = payloadByteUpto;
this.curEndOffset = endOffset;
this.curStartOffset = startOffset;
bufferSkip(numDocs);
}
@ -134,8 +134,8 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
}
if (fieldHasOffsets) {
skipBuffer.writeVInt(curEndOffset - lastEndOffset[level]);
lastEndOffset[level] = curEndOffset;
skipBuffer.writeVInt(curStartOffset - lastStartOffset[level]);
lastStartOffset[level] = curStartOffset;
}
if (fieldHasOffsets || fieldHasPayloads) {

22
lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java
View File

@ -978,7 +978,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
private int payloadByteUpto;
private int payloadLength;
private int lastEndOffset;
private int lastStartOffset;
private int startOffset;
private int endOffset;
@ -1274,7 +1274,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
}
position = 0;
payloadLength = 0;
lastEndOffset = 0;
lastStartOffset = 0;
return doc;
}
@ -1331,7 +1331,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
// Skipper moved
if (DEBUG) {
System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastEndOffset=" + lastEndOffset);
System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset);
}
assert newDocUpto % blockSize == (blockSize-1): "got " + newDocUpto;
@ -1344,7 +1344,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
posPendingFP = skipper.getPosPointer();
payPendingFP = skipper.getPayPointer();
posPendingCount = skipper.getPosBufferUpto();
lastEndOffset = skipper.getEndOffset();
lastStartOffset = skipper.getStartOffset();
payloadByteUpto = skipper.getPayloadByteUpto();
}
}
@ -1385,7 +1385,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@ -1412,11 +1412,11 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
if (indexHasOffsets) {
// Must load offset blocks merely to sum
// up into lastEndOffset:
// up into lastStartOffset:
readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaLBuffer);
readBlock(payIn, encoded, encodedBuffer, offsetLengthLBuffer);
for(int i=0;i<blockSize;i++) {
lastEndOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
lastStartOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
}
}
toSkip -= blockSize;
@ -1429,7 +1429,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@ -1440,7 +1440,7 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
position = 0;
payloadLength = 0;
lastEndOffset = 0;
lastStartOffset = 0;
}
@Override
@ -1493,9 +1493,9 @@ public final class BlockPackedPostingsReader extends PostingsReaderBase {
}
if (indexHasOffsets) {
startOffset = lastEndOffset + (int)offsetStartDeltaBuffer[posBufferUpto];
startOffset = lastStartOffset + (int)offsetStartDeltaBuffer[posBufferUpto];
endOffset = startOffset + (int)offsetLengthBuffer[posBufferUpto];
lastEndOffset = endOffset;
lastStartOffset = startOffset;
}
posBufferUpto++;

18
lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java
View File

@ -102,11 +102,11 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
private long lastBlockPosFP;
private long lastBlockPayFP;
private int lastBlockPosBufferUpto;
private int lastBlockEndOffset;
private int lastBlockStartOffset;
private int lastBlockPayloadByteUpto;
private int lastDocID;
private int lastPosition;
private int lastEndOffset;
private int lastStartOffset;
private int docCount;
final byte[] encoded;
@ -249,7 +249,7 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
lastBlockPayFP = payOut.getFilePointer();
}
lastBlockPosBufferUpto = posBufferUpto;
lastBlockEndOffset = lastEndOffset;
lastBlockStartOffset = lastStartOffset;
lastBlockPayloadByteUpto = payloadByteUpto;
saveNextPosBlock = false;
if (DEBUG) {
@ -288,7 +288,7 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
if (DEBUG) {
System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-blockSize));
}
skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
lastBlockDocID = docID;
saveNextPosBlock = true;
@ -307,7 +307,7 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
}
lastPosition = 0;
lastEndOffset = 0;
lastStartOffset = 0;
}
/** Add a new position & payload */
@ -332,11 +332,11 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
}
if (fieldHasOffsets) {
assert startOffset >= lastEndOffset;
assert startOffset >= lastStartOffset;
assert endOffset >= startOffset;
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastEndOffset;
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
lastEndOffset = endOffset;
lastStartOffset = startOffset;
}
posBufferUpto++;
@ -407,7 +407,7 @@ public final class BlockPackedPostingsWriter extends PostingsWriterBase {
if (DEBUG) {
System.out.println(" bufferSkip at finishTerm: lastDocID=" + lastBlockDocID + " docCount=" + lastDocCount);
}
skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
if (DEBUG) {

24
lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
View File

@ -37,12 +37,12 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
private long posPointer[];
private long payPointer[];
private int posBufferUpto[];
private int endOffset[];
private int startOffset[];
private int payloadByteUpto[];
private long lastPosPointer;
private long lastPayPointer;
private int lastEndOffset;
private int lastStartOffset;
private int lastPayloadByteUpto;
private long lastDocPointer;
private int lastPosBufferUpto;
@ -59,9 +59,9 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
payloadByteUpto = null;
}
if (hasOffsets) {
endOffset = new int[maxSkipLevels];
startOffset = new int[maxSkipLevels];
} else {
endOffset = null;
startOffset = null;
}
if (hasOffsets || hasPayloads) {
payPointer = new long[maxSkipLevels];
@ -108,8 +108,8 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
return lastPayPointer;
}
public int getEndOffset() {
return lastEndOffset;
public int getStartOffset() {
return lastStartOffset;
}
public int getPayloadByteUpto() {
@ -126,8 +126,8 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
if (posPointer != null) {
posPointer[level] = lastPosPointer;
posBufferUpto[level] = lastPosBufferUpto;
if (endOffset != null) {
endOffset[level] = lastEndOffset;
if (startOffset != null) {
startOffset[level] = lastStartOffset;
}
if (payloadByteUpto != null) {
payloadByteUpto[level] = lastPayloadByteUpto;
@ -155,8 +155,8 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
if (payPointer != null) {
lastPayPointer = payPointer[level];
}
if (endOffset != null) {
lastEndOffset = endOffset[level];
if (startOffset != null) {
lastStartOffset = startOffset[level];
}
if (payloadByteUpto != null) {
lastPayloadByteUpto = payloadByteUpto[level];
@ -192,8 +192,8 @@ final class BlockPackedSkipReader extends MultiLevelSkipListReader {
payloadByteUpto[level] = skipStream.readVInt();
}
if (endOffset != null) {
endOffset[level] += skipStream.readVInt();
if (startOffset != null) {
startOffset[level] += skipStream.readVInt();
}
if (payPointer != null) {

16
lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
View File

@ -34,7 +34,7 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
private long[] lastSkipDocPointer;
private long[] lastSkipPosPointer;
private long[] lastSkipPayPointer;
private int[] lastEndOffset;
private int[] lastStartOffset;
private int[] lastPayloadByteUpto;
private final IndexOutput docOut;
@ -46,7 +46,7 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
private long curPosPointer;
private long curPayPointer;
private int curPosBufferUpto;
private int curEndOffset;
private int curStartOffset;
private int curPayloadByteUpto;
private boolean fieldHasPositions;
private boolean fieldHasOffsets;
@ -65,7 +65,7 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
if (payOut != null) {
lastSkipPayPointer = new long[maxSkipLevels];
}
lastEndOffset = new int[maxSkipLevels];
lastStartOffset = new int[maxSkipLevels];
lastPayloadByteUpto = new int[maxSkipLevels];
}
}
@ -84,7 +84,7 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
if (fieldHasPositions) {
Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
if (fieldHasOffsets) {
Arrays.fill(lastEndOffset, 0);
Arrays.fill(lastStartOffset, 0);
}
if (fieldHasPayloads) {
Arrays.fill(lastPayloadByteUpto, 0);
@ -98,14 +98,14 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
/**
* Sets the values for the current skip data.
*/
public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int endOffset, int payloadByteUpto) throws IOException {
public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int startOffset, int payloadByteUpto) throws IOException {
this.curDoc = doc;
this.curDocPointer = docOut.getFilePointer();
this.curPosPointer = posFP;
this.curPayPointer = payFP;
this.curPosBufferUpto = posBufferUpto;
this.curPayloadByteUpto = payloadByteUpto;
this.curEndOffset = endOffset;
this.curStartOffset = startOffset;
bufferSkip(numDocs);
}
@ -134,8 +134,8 @@ final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
}
if (fieldHasOffsets) {
skipBuffer.writeVInt(curEndOffset - lastEndOffset[level]);
lastEndOffset[level] = curEndOffset;
skipBuffer.writeVInt(curStartOffset - lastStartOffset[level]);
lastStartOffset[level] = curStartOffset;
}
if (fieldHasOffsets || fieldHasPayloads) {

8
lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
View File

@ -439,6 +439,14 @@ public class TestPostingsOffsets extends LuceneTestCase {
}
}
public void testStackedTokens() throws Exception {
checkTokens(new Token[] {
makeToken("foo", 1, 0, 3),
makeToken("foo", 0, 0, 3),
makeToken("foo", 0, 0, 3)
});
}
public void testLegalbutVeryLargeOffsets() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));