numStoredFields will be zero. */
public abstract void startDocument(int numStoredFields) throws IOException;
-
+
+ /** Called when a document and all its fields have been added. */
+ public void finishDocument() throws IOException {}
+
/** Writes a single stored field. */
public abstract void writeField(FieldInfo info, StorableField field) throws IOException;
@@ -116,6 +119,8 @@ public abstract class StoredFieldsWriter implements Closeable {
for (StorableField field : doc) {
writeField(fieldInfos.fieldInfo(field.name()), field);
}
+
+ finishDocument();
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
index f6fcb86712a..c15be770d8c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
@@ -71,18 +71,27 @@ public abstract class TermVectorsWriter implements Closeable {
* has no vector fields, in this case numVectorFields
* will be zero. */
public abstract void startDocument(int numVectorFields) throws IOException;
-
+
+ /** Called after a doc and all its fields have been added. */
+ public void finishDocument() throws IOException {};
+
/** Called before writing the terms of the field.
* {@link #startTerm(BytesRef, int)} will be called numTerms times. */
public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
-
+
+ /** Called after a field and all its terms have been added. */
+ public void finishField() throws IOException {};
+
/** Adds a term and its term frequency freq.
* If this field has positions and/or offsets enabled, then
* {@link #addPosition(int, int, int, BytesRef)} will be called
* freq times respectively.
*/
public abstract void startTerm(BytesRef term, int freq) throws IOException;
-
+
+ /** Called after a term and all its positions have been added. */
+ public void finishTerm() throws IOException {}
+
/** Adds a term position and offsets */
public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
@@ -97,7 +106,7 @@ public abstract class TermVectorsWriter implements Closeable {
* check that this is the case to detect the JRE bug described
* in LUCENE-1282. */
public abstract void finish(FieldInfos fis, int numDocs) throws IOException;
-
+
/**
* Called by IndexWriter when writing new segments.
*
@@ -197,6 +206,7 @@ public abstract class TermVectorsWriter implements Closeable {
protected final void addAllDocVectors(Fields vectors, MergeState mergeState) throws IOException {
if (vectors == null) {
startDocument(0);
+ finishDocument();
return;
}
@@ -275,10 +285,13 @@ public abstract class TermVectorsWriter implements Closeable {
addPosition(pos, startOffset, endOffset, payload);
}
}
+ finishTerm();
}
assert termCount == numTerms;
+ finishField();
}
assert fieldCount == numFields;
+ finishDocument();
}
/** Return the BytesRef Comparator used to sort terms
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
index 797cdb7b6fc..7d06240281a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
@@ -395,8 +395,10 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
* Copy compressed data.
*/
void copyCompressedData(DataOutput out) throws IOException {
- final int chunkSize = chunkSize();
- decompressor.copyCompressedData(fieldsStream, chunkSize, out);
+ final long chunkEnd = docBase + chunkDocs == numDocs
+ ? fieldsStream.length()
+ : indexReader.getStartPointer(docBase + chunkDocs);
+ out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
index d1bf861e9ea..e0498ed2980 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
@@ -136,19 +136,8 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
}
}
- private void endWithPreviousDocument() throws IOException {
- if (numBufferedDocs > 0) {
- endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
- }
- }
-
@Override
public void startDocument(int numStoredFields) throws IOException {
- endWithPreviousDocument();
- if (triggerFlush()) {
- flush();
- }
-
if (numBufferedDocs == this.numStoredFields.length) {
final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
this.numStoredFields = Arrays.copyOf(this.numStoredFields, newLength);
@@ -158,6 +147,14 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
++numBufferedDocs;
}
+ @Override
+ public void finishDocument() throws IOException {
+ endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
+ if (triggerFlush()) {
+ flush();
+ }
+ }
+
private static void saveInts(int[] values, int length, DataOutput out) throws IOException {
assert length > 0;
if (length == 1) {
@@ -295,9 +292,10 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
- endWithPreviousDocument();
if (numBufferedDocs > 0) {
flush();
+ } else {
+ assert bufferedDocs.length == 0;
}
if (docBase != numDocs) {
throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
@@ -351,17 +349,13 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
}
if (compressionMode == matchingFieldsReader.getCompressionMode() // same compression mode
- && (numBufferedDocs == 0 || triggerFlush()) // starting a new chunk
+ && numBufferedDocs == 0 // starting a new chunk
&& startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
&& startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
&& nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
assert docID == it.docBase;
// no need to decompress, just copy data
- endWithPreviousDocument();
- if (triggerFlush()) {
- flush();
- }
indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
writeHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
it.copyCompressedData(fieldsStream);
@@ -380,6 +374,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
final int diff = docID - it.docBase;
startDocument(it.numStoredFields[diff]);
bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ finishDocument();
++docCount;
mergeState.checkAbort.work(300);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
index b38490225ee..700258fdd28 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
@@ -140,14 +140,6 @@ public abstract class CompressionMode {
bytes.length = length;
}
- @Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int copied = LZ4.copyCompressedData(in, originalLength, out);
- if (copied != originalLength) {
- throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
- }
- }
-
@Override
public Decompressor clone() {
return this;
@@ -224,13 +216,6 @@ public abstract class CompressionMode {
bytes.length = length;
}
- @Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int compressedLength = in.readVInt();
- out.writeVInt(compressedLength);
- out.copyBytes(in, compressedLength);
- }
-
@Override
public Decompressor clone() {
return new DeflateDecompressor();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
index a6529994b5d..ef4fcc27789 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
@@ -24,7 +24,10 @@ import org.apache.lucene.store.DataOutput;
/**
* A data compressor.
*/
-abstract class Compressor {
+public abstract class Compressor {
+
+ /** Sole constructor, typically called from sub-classes. */
+ protected Compressor() {}
/**
* Compress bytes into
+ * The sequence is divided into fixed-size blocks and for each block, the
+ * difference between each value and the minimum value of the block is encoded
+ * using as few bits as possible. Memory usage of this class is proportional to
+ * the block size. Each block has an overhead between 1 and 10 bytes to store
+ * the minimum value and the number of bits per value of the block.
+ * @see BlockPackedReader
+ * @lucene.internal
+ */
+public final class BlockPackedWriter {
+
+ static final int MAX_BLOCK_SIZE = 1 << (30 - 3);
+ static final int MIN_VALUE_EQUALS_0 = 1 << 0;
+ static final int BPV_SHIFT = 1;
+
+ static void checkBlockSize(int blockSize) {
+ if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) {
+ throw new IllegalArgumentException("blockSize must be > 0 and < " + MAX_BLOCK_SIZE + ", got " + blockSize);
+ }
+ if (blockSize % 64 != 0) {
+ throw new IllegalArgumentException("blockSize must be a multiple of 64, got " + blockSize);
+ }
+ }
+
+ static long zigZagEncode(long n) {
+ return (n >> 63) ^ (n << 1);
+ }
+
+ // same as DataOutput.writeVLong but accepts negative values
+ static void writeVLong(DataOutput out, long i) throws IOException {
+ int k = 0;
+ while ((i & ~0x7FL) != 0L && k++ < 8) {
+ out.writeByte((byte)((i & 0x7FL) | 0x80L));
+ i >>>= 7;
+ }
+ out.writeByte((byte) i);
+ }
+
+ final DataOutput out;
+ final long[] values;
+ byte[] blocks;
+ int off;
+ long ord;
+ boolean finished;
+
+ /**
+ * Sole constructor.
+ * @param blockSize the number of values of a single block, must be a multiple of 64
+ */
+ public BlockPackedWriter(DataOutput out, int blockSize) {
+ checkBlockSize(blockSize);
+ this.out = out;
+ values = new long[blockSize];
+ off = 0;
+ ord = 0L;
+ finished = false;
+ }
+
+ private void checkNotFinished() {
+ if (finished) {
+ throw new IllegalStateException("Already finished");
+ }
+ }
+
+ /** Append a new long. */
+ public void add(long l) throws IOException {
+ checkNotFinished();
+ if (off == values.length) {
+ flush();
+ }
+ values[off++] = l;
+ ++ord;
+ }
+
+ /** Flush all buffered data to disk. This instance is not usable anymore
+ * after this method has been called. */
+ public void finish() throws IOException {
+ checkNotFinished();
+ if (off > 0) {
+ flush();
+ }
+ finished = true;
+ }
+
+ private void flush() throws IOException {
+ assert off > 0;
+ long min = Long.MAX_VALUE, max = Long.MIN_VALUE;
+ for (int i = 0; i < off; ++i) {
+ min = Math.min(values[i], min);
+ max = Math.max(values[i], max);
+ }
+
+ final long delta = max - min;
+ final int bitsRequired = delta < 0 ? 64 : delta == 0L ? 0 : PackedInts.bitsRequired(delta);
+ if (bitsRequired == 64) {
+ // no need to delta-encode
+ min = 0L;
+ } else if (min > 0L) {
+ // make min as small as possible so that writeVLong requires fewer bytes
+ min = Math.max(0L, max - PackedInts.maxValue(bitsRequired));
+ }
+
+ final int token = (bitsRequired << BPV_SHIFT) | (min == 0 ? MIN_VALUE_EQUALS_0 : 0);
+ out.writeByte((byte) token);
+
+ if (min != 0) {
+ writeVLong(out, zigZagEncode(min) - 1);
+ }
+
+ if (bitsRequired > 0) {
+ if (min != 0) {
+ for (int i = 0; i < off; ++i) {
+ values[i] -= min;
+ }
+ }
+ final PackedInts.Encoder encoder = PackedInts.getEncoder(PackedInts.Format.PACKED, PackedInts.VERSION_CURRENT, bitsRequired);
+ final int iterations = values.length / encoder.valueCount();
+ final int blockSize = encoder.blockCount() * 8 * iterations;
+ if (blocks == null || blocks.length < blockSize) {
+ blocks = new byte[blockSize];
+ }
+ if (off < values.length) {
+ Arrays.fill(values, off, values.length, 0L);
+ }
+ encoder.encode(values, 0, blocks, 0, iterations);
+ final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsRequired);
+ out.writeBytes(blocks, blockCount);
+ }
+
+ off = 0;
+ }
+
+ /** Return the number of values which have been added. */
+ public long ord() {
+ return ord;
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
index 7b25839179f..fd579f673e2 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
@@ -80,16 +80,6 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
}
- static byte[] copyCompressedData(Decompressor decompressor, byte[] compressed, int originalLength) throws IOException {
- GrowableByteArrayDataOutput out = new GrowableByteArrayDataOutput(compressed.length);
- decompressor.copyCompressedData(new ByteArrayDataInput(compressed), originalLength, out);
- return Arrays.copyOf(out.bytes, out.length);
- }
-
- byte[] copyCompressedData(byte[] compressed, int originalLength) throws IOException {
- return copyCompressedData(mode.newDecompressor(), compressed, originalLength);
- }
-
public void testDecompress() throws IOException {
final int iterations = atLeast(10);
for (int i = 0; i < iterations; ++i) {
@@ -117,17 +107,10 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
}
}
- public void testCopyCompressedData() throws IOException {
- final byte[] decompressed = randomArray();
- final byte[] compressed = compress(decompressed);
- assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
- }
-
public byte[] test(byte[] decompressed) throws IOException {
final byte[] compressed = compress(decompressed);
final byte[] restored = decompress(compressed, decompressed.length);
assertEquals(decompressed.length, restored.length);
- assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
return compressed;
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
index 4237a6d0478..1997bcc4296 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
@@ -116,11 +116,28 @@ public class TestCharsRef extends LuceneTestCase {
}
// LUCENE-3590: fix off-by-one in subsequence, and fully obey interface
+ // LUCENE-4671: fix subSequence
public void testCharSequenceSubSequence() {
- CharSequence c = new CharsRef("abc");
+ CharSequence sequences[] = {
+ new CharsRef("abc"),
+ new CharsRef("0abc".toCharArray(), 1, 3),
+ new CharsRef("abc0".toCharArray(), 0, 3),
+ new CharsRef("0abc0".toCharArray(), 1, 3)
+ };
+
+ for (CharSequence c : sequences) {
+ doTestSequence(c);
+ }
+ }
+
+ private void doTestSequence(CharSequence c) {
// slice
assertEquals("a", c.subSequence(0, 1).toString());
+ // mid subsequence
+ assertEquals("b", c.subSequence(1, 2).toString());
+ // end subsequence
+ assertEquals("bc", c.subSequence(1, 3).toString());
// empty subsequence
assertEquals("", c.subSequence(0, 0).toString());
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index 176dbca3770..8fb4b2ab3a5 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -310,7 +310,7 @@ public class TestFSTs extends LuceneTestCase {
final boolean doRewrite = random().nextBoolean();
- Builderout. It it the responsibility of the
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
index 43f30c2c08f..c7fffc7392d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
@@ -20,13 +20,15 @@ package org.apache.lucene.codecs.compressing;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
/**
- * An decompressor.
+ * A decompressor.
*/
-abstract class Decompressor implements Cloneable {
+public abstract class Decompressor implements Cloneable {
+
+ /** Sole constructor, typically called from sub-classes. */
+ protected Decompressor() {}
/**
* Decompress bytes that were stored between offsets offset and
@@ -44,10 +46,6 @@ abstract class Decompressor implements Cloneable {
*/
public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
- /** Copy a compressed stream whose original length is
- * originalLength from in to out. */
- public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
-
@Override
public abstract Decompressor clone();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
index 8807018fb75..7e52339657d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
@@ -506,51 +506,4 @@ class LZ4 {
encodeLastLiterals(src, anchor, srcEnd - anchor, out);
}
- /** Copy bytes from in to out where
- * in is a LZ4-encoded stream. This method copies enough bytes
- * so that out can be used later on to restore the first
- * length bytes of the stream. This method always reads at
- * least one byte from in so make sure not to call this method
- * if in reached the end of the stream, even if
- * length=0. */
- public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
- int n = 0;
- do {
- // literals
- final byte token = in.readByte();
- out.writeByte(token);
- int literalLen = (token & 0xFF) >>> 4;
- if (literalLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- literalLen += 0xFF;
- out.writeByte(len);
- }
- literalLen += len & 0xFF;
- out.writeByte(len);
- }
- out.copyBytes(in, literalLen);
- n += literalLen;
- if (n >= length) {
- break;
- }
-
- // matchs
- out.copyBytes(in, 2); // match dec
- int matchLen = token & 0x0F;
- if (matchLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- matchLen += 0xFF;
- out.writeByte(len);
- }
- matchLen += len & 0xFF;
- out.writeByte(len);
- }
- matchLen += MIN_MATCH;
- n += matchLen;
- } while (n < length);
- return n;
- }
-
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
index 36529b63fff..0386369e336 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
@@ -124,17 +124,16 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter {
if (payloads)
bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
tvf.writeByte(bits);
-
- assert fieldCount <= numVectorFields;
- if (fieldCount == numVectorFields) {
- // last field of the document
- // this is crazy because the file format is crazy!
- for (int i = 1; i < fieldCount; i++) {
- tvd.writeVLong(fps[i] - fps[i-1]);
- }
- }
}
+ @Override
+ public void finishDocument() throws IOException {
+ assert fieldCount == numVectorFields;
+ for (int i = 1; i < fieldCount; i++) {
+ tvd.writeVLong(fps[i] - fps[i-1]);
+ }
+ }
+
private final BytesRef lastTerm = new BytesRef(10);
// NOTE: we override addProx, so we don't need to buffer when indexing.
@@ -222,20 +221,6 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter {
}
bufferedIndex++;
-
- // dump buffer if we are done
- if (bufferedIndex == bufferedFreq) {
- if (payloads) {
- tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
- }
- for (int i = 0; i < bufferedIndex; i++) {
- if (offsets) {
- tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
- tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
- lastOffset = offsetEndBuffer[i];
- }
- }
- }
} else if (positions) {
// write position delta
writePosition(position - lastPosition, payload);
@@ -248,6 +233,25 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter {
}
}
+ @Override
+ public void finishTerm() throws IOException {
+ if (bufferedIndex > 0) {
+ // dump buffer
+ assert positions && (offsets || payloads);
+ assert bufferedIndex == bufferedFreq;
+ if (payloads) {
+ tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
+ }
+ for (int i = 0; i < bufferedIndex; i++) {
+ if (offsets) {
+ tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
+ tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
+ lastOffset = offsetEndBuffer[i];
+ }
+ }
+ }
+ }
+
private void writePosition(int delta, BytesRef payload) throws IOException {
if (payloads) {
int payloadLength = payload == null ? 0 : payload.length;
diff --git a/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java b/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
index b0d7cf7b8ac..eea05b13200 100644
--- a/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
+++ b/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
@@ -108,6 +108,7 @@ final class StoredFieldsProcessor extends StoredFieldsConsumer {
while(lastDocID < docID) {
fieldsWriter.startDocument(0);
lastDocID++;
+ fieldsWriter.finishDocument();
}
}
@@ -123,6 +124,7 @@ final class StoredFieldsProcessor extends StoredFieldsConsumer {
for (int i = 0; i < numStoredFields; i++) {
fieldsWriter.writeField(fieldInfos[i], storedFields[i]);
}
+ fieldsWriter.finishDocument();
lastDocID++;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
index 40756e1fa25..bafba22ca80 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
@@ -78,6 +78,7 @@ final class TermVectorsConsumer extends TermsHashConsumer {
void fill(int docID) throws IOException {
while(lastDocID < docID) {
writer.startDocument(0);
+ writer.finishDocument();
lastDocID++;
}
}
@@ -108,6 +109,7 @@ final class TermVectorsConsumer extends TermsHashConsumer {
for (int i = 0; i < numVectorFields; i++) {
perFields[i].finishDocument();
}
+ writer.finishDocument();
assert lastDocID == docState.docID: "lastDocID=" + lastDocID + " docState.docID=" + docState.docID;
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
index 0e5a366f327..c3cb2c8d4cc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
@@ -182,7 +182,9 @@ final class TermVectorsConsumerPerField extends TermsHashConsumerPerField {
}
tv.addProx(freq, posReader, offReader);
}
+ tv.finishTerm();
}
+ tv.finishField();
termsHashPerField.reset();
diff --git a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
index 60537bfdeb9..30e95439c76 100644
--- a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
+++ b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
@@ -218,7 +218,7 @@ public final class CharsRef implements Comparablecount values. */
+ public void skip(long count) throws IOException {
+ assert count >= 0;
+ if (ord + count > valueCount || ord + count < 0) {
+ throw new EOFException();
+ }
+
+ // 1. skip buffered values
+ final int skipBuffer = (int) Math.min(count, blockSize - off);
+ off += skipBuffer;
+ ord += skipBuffer;
+ count -= skipBuffer;
+ if (count == 0L) {
+ return;
+ }
+
+ // 2. skip as many blocks as necessary
+ assert off == blockSize;
+ while (count >= blockSize) {
+ final int token = in.readByte() & 0xFF;
+ final int bitsPerValue = token >>> BPV_SHIFT;
+ if (bitsPerValue > 64) {
+ throw new IOException("Corrupted");
+ }
+ if ((token & MIN_VALUE_EQUALS_0) == 0) {
+ readVLong(in);
+ }
+ final long blockBytes = PackedInts.Format.PACKED.byteCount(packedIntsVersion, blockSize, bitsPerValue);
+ skipBytes(blockBytes);
+ ord += blockSize;
+ count -= blockSize;
+ }
+ if (count == 0L) {
+ return;
+ }
+
+ // 3. skip last values
+ assert count < blockSize;
+ refill();
+ ord += count;
+ off += count;
+ }
+
+ private void skipBytes(long count) throws IOException {
+ if (in instanceof IndexInput) {
+ final IndexInput iin = (IndexInput) in;
+ iin.seek(iin.getFilePointer() + count);
+ } else {
+ if (blocks == null) {
+ blocks = new byte[blockSize];
+ }
+ long skipped = 0;
+ while (skipped < count) {
+ final int toSkip = (int) Math.min(blocks.length, count - skipped);
+ in.readBytes(blocks, 0, toSkip);
+ skipped += toSkip;
+ }
+ }
+ }
+
+ /** Read the next value. */
+ public long next() throws IOException {
+ next(1);
+ assert values.length == 1;
+ return values.longs[values.offset];
+ }
+
+ /** Read between 1 and count values. */
+ public LongsRef next(int count) throws IOException {
+ assert count > 0;
+ if (ord == valueCount) {
+ throw new EOFException();
+ }
+ if (off == blockSize) {
+ refill();
+ }
+
+ count = Math.min(count, blockSize - off);
+ count = (int) Math.min(count, valueCount - ord);
+
+ values.offset = off;
+ values.length = count;
+ off += count;
+ ord += count;
+ return values;
+ }
+
+ private void refill() throws IOException {
+ final int token = in.readByte() & 0xFF;
+ final boolean minEquals0 = (token & MIN_VALUE_EQUALS_0) != 0;
+ final int bitsPerValue = token >>> BPV_SHIFT;
+ if (bitsPerValue > 64) {
+ throw new IOException("Corrupted");
+ }
+ final long minValue = minEquals0 ? 0L : zigZagDecode(1L + readVLong(in));
+ assert minEquals0 || minValue != 0;
+
+ if (bitsPerValue == 0) {
+ Arrays.fill(values.longs, minValue);
+ } else {
+ final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue);
+ final int iterations = blockSize / decoder.valueCount();
+ final int blocksSize = iterations * 8 * decoder.blockCount();
+ if (blocks == null || blocks.length < blocksSize) {
+ blocks = new byte[blocksSize];
+ }
+
+ final int valueCount = (int) Math.min(this.valueCount - ord, blockSize);
+ final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue);
+ in.readBytes(blocks, 0, blocksCount);
+
+ decoder.decode(blocks, 0, values.longs, 0, iterations);
+
+ if (minValue != 0) {
+ for (int i = 0; i < valueCount; ++i) {
+ values.longs[i] += minValue;
+ }
+ }
+ }
+ off = 0;
+ }
+
+ /** Return the offset of the next value to read. */
+ public long ord() {
+ return ord;
+ }
+
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java b/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
new file mode 100644
index 00000000000..43670cc1a5f
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
@@ -0,0 +1,164 @@
+package org.apache.lucene.util.packed;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * A writer for large sequences of longs.
+ *
+ * NOTE: associations list do not encode the counting list data. You
+ * should use {@link CountingListBuilder} to build that information and then
+ * merge the results of both {@link #build(IntsRef, Iterable)}.
+ */
+public class AssociationsListBuilder implements CategoryListBuilder {
+
+ private final CategoryAssociationsContainer associations;
+ private final ByteArrayDataOutput output = new ByteArrayDataOutput();
+
+ public AssociationsListBuilder(CategoryAssociationsContainer associations) {
+ this.associations = associations;
+ }
+
+ @Override
+ public Map
+ * NOTE: you are not expected to modify the returned map.
*/
- public float getAssociation(int ordinal) {
- if (ordinalAssociations.containsKey(ordinal)) {
- return ordinalAssociations.get(ordinal);
- }
-
- return Float.NaN;
+ public IntToFloatMap getAssociations(int docID) throws IOException {
+ ordinalAssociations.clear();
+ return setNextDoc(docID) ? ordinalAssociations : null;
}
-
+
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java
index ad983114084..e3bed4f51e5 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java
@@ -31,12 +31,6 @@ public class IntAssociationsPayloadIterator extends AssociationsPayloadIterator<
private final IntToIntMap ordinalAssociations = new IntToIntMap();
- /**
- * The long-special-value returned for ordinals which have no associated int
- * value. It is not in the int range of values making it a valid mark.
- */
- public final static long NO_ASSOCIATION = Integer.MAX_VALUE + 1;
-
public IntAssociationsPayloadIterator(IndexReader reader, String field, CategoryIntAssociation association)
throws IOException {
super(reader, field, association);
@@ -47,22 +41,16 @@ public class IntAssociationsPayloadIterator extends AssociationsPayloadIterator<
ordinalAssociations.put(ordinal, association.getValue());
}
- @Override
- public boolean setNextDoc(int docId) throws IOException {
- ordinalAssociations.clear();
- return super.setNextDoc(docId);
- }
-
/**
- * Get the integer association value for the given ordinal, or
- * {@link #NO_ASSOCIATION} in case the ordinal has no association value.
+ * Returns the integer association values of the categories that are
+ * associated with the given document, or {@code null} if the document has no
+ * associations.
+ *
+ * NOTE: you are not expected to modify the returned map.
*/
- public long getAssociation(int ordinal) {
- if (ordinalAssociations.containsKey(ordinal)) {
- return ordinalAssociations.get(ordinal);
- }
-
- return NO_ASSOCIATION;
+ public IntToIntMap getAssociations(int docID) throws IOException {
+ ordinalAssociations.clear();
+ return setNextDoc(docID) ? ordinalAssociations : null;
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java
index 509c0c61684..0bea43235b4 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java
@@ -1,19 +1,11 @@
package org.apache.lucene.facet.index;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map.Entry;
+import java.util.Map;
-import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.util.PartitionsUtils;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.UnsafeByteArrayOutputStream;
-import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -33,149 +25,14 @@ import org.apache.lucene.util.encoding.IntEncoder;
*/
/**
- * Builds a category list by encoding the category ordinals into one or more
- * {@link BytesRef}. Each {@link BytesRef} corresponds to a set of ordinals that
- * belong to the same partition. When partitions are not enabled (i.e.
- * {@link FacetIndexingParams#getPartitionSize()} returns
- * {@link Integer#MAX_VALUE}), only one {@link BytesRef} is returned by this
- * class.
+ * Builds a category list data by encoding the appropriate information for every
+ * category and ordinal given to {@link #build(IntsRef, Iterable)}.
+ *
+ * @lucene.experimental
*/
-public class CategoryListBuilder {
-
- /** Specializes encoding ordinals when partitions are enabled/disabled. */
- private static abstract class OrdinalsEncoder {
- OrdinalsEncoder() {}
- public abstract void encode(int ordinal);
- public abstract HashMap
+ * Counting lists are used usually for computing the weight of categories by
+ * summing their number of occurrences (hence counting) in a result set.
+ */
+public class CountingListBuilder implements CategoryListBuilder {
+
+ /** Specializes encoding ordinals when partitions are enabled/disabled. */
+ private static abstract class OrdinalsEncoder {
+ OrdinalsEncoder() {}
+ public abstract Map
+ * NOTE: the {@code ordinals} array is modified by adding parent
+ * ordinals to it. Also, some encoders may sort the array and remove duplicate
+ * ordinals. Therefore you may want to invoke this method after you finished
+ * processing the array for other purposes.
+ */
+ @Override
+ public Map
- * By default, all categories are written to the same category list, but
- * applications which know in advance that in some situations only parts of
- * the category hierarchy needs to be counted can divide the categories into
- * two or more different category lists.
- *
- * If {@code null} is returned for a category, it means that this category
- * should not appear in any category list, and thus weights for it cannot be
- * aggregated. This category can still be used for drill-down, even though the
- * its weight is unknown.
+ * Returns the {@link CategoryListParams} for this {@link CategoryPath}. The
+ * default implementation returns the same {@link CategoryListParams} for all
+ * categories (even if {@code category} is {@code null}).
*
* @see PerDimensionIndexingParams
*/
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java
index 1ccb58d309f..c4b971a5ca9 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java
@@ -78,7 +78,9 @@ public class PerDimensionIndexingParams extends FacetIndexingParams {
/**
* Returns the {@link CategoryListParams} for the corresponding dimension
- * which is returned by {@code category.getComponent(0)}.
+ * which is returned by {@code category.getComponent(0)}. If {@code category}
+ * is {@code null}, or was not specified in the map given to the constructor,
+ * returns the default {@link CategoryListParams}.
*/
@Override
public CategoryListParams getCategoryListParams(CategoryPath category) {
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java
index 79f89909030..5132e930264 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java
@@ -2,6 +2,8 @@ package org.apache.lucene.facet.search;
import java.io.IOException;
+import org.apache.lucene.util.IntsRef;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -20,20 +22,10 @@ import java.io.IOException;
*/
/**
- * An interface for iterating over a "category list", i.e., the list of
- * categories per document.
+ * An interface for obtaining the category ordinals of documents.
*
- * NOTE:
- *
+ * NOTE: if the requested document does not category ordinals
+ * associated with it, {@link IntsRef#length} is set to zero.
*/
- public boolean skipTo(int docId) throws IOException;
-
- /**
- * Returns the next category for the current document that is set through
- * {@link #skipTo(int)}, or a number higher than {@link Integer#MAX_VALUE}.
- * No assumptions can be made on the order of the categories.
- */
- public long nextCategory() throws IOException;
-
+ public void getOrdinals(int docID, IntsRef ints) throws IOException;
+
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
similarity index 52%
rename from lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java
rename to lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
index db4803e1019..3deba112f3e 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
@@ -5,7 +5,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnsafeByteArrayInputStream;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.encoding.IntDecoder;
/*
@@ -26,44 +26,21 @@ import org.apache.lucene.util.encoding.IntDecoder;
*/
/**
- * A payload deserializer comes with its own working space (buffer). One need to
- * define the {@link IndexReader} and {@link Term} in which the payload resides.
- * The iterator then consumes the payload information of each document and
- * decodes it into categories. A typical use case of this class is:
- *
- *
- * Users should call this method with increasing docIds, and implementations
- * can assume that this is the case.
+ * Returns the {@link BytesRef payload} of the given document, or {@code null}
+ * if the document does not exist, there are no more documents in the posting
+ * list, or the document exists but has not payload. You should call
+ * {@link #init()} before the first call to this method.
*/
- public boolean setdoc(int docId) throws IOException {
+ public BytesRef getPayload(int docID) throws IOException {
if (!hasMore) {
- return false;
+ return null;
}
// re-basing docId->localDocID is done fewer times than currentDoc->globalDoc
- int localDocID = docId - curDocBase;
+ int localDocID = docID - curDocBase;
if (curDocID > localDocID) {
// document does not exist
- return false;
+ return null;
}
if (curDocID < localDocID) {
// look for the document either in that segment, or others
while (hasMore && (curDocID = currentDPE.advance(localDocID)) == DocIdSetIterator.NO_MORE_DOCS) {
nextSegment(); // also updates curDocID
- localDocID = docId - curDocBase;
+ localDocID = docID - curDocBase;
// nextSegment advances to nextDoc, so check if we still need to advance
if (curDocID >= localDocID) {
break;
@@ -127,7 +126,7 @@ public class PayloadIterator {
// 1. we iterated over all segments (hasMore=false)
// 2. current segment advanced to a doc, either requested or higher
if (!hasMore || curDocID != localDocID) {
- return false;
+ return null;
}
}
@@ -135,12 +134,7 @@ public class PayloadIterator {
assert currentDPE.freq() == 1 : "expecting freq=1 (got " + currentDPE.freq() + ") term=" + term + " doc=" + (curDocID + curDocBase);
int pos = currentDPE.nextPosition();
assert pos != -1 : "no positions for term=" + term + " doc=" + (curDocID + curDocBase);
- data = currentDPE.getPayload();
- return data != null;
+ return currentDPE.getPayload();
}
- public BytesRef getPayload() {
- return data;
- }
-
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
index 78b362ea063..32466269d60 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
@@ -10,6 +10,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -231,9 +232,9 @@ public class StandardFacetsAccumulator extends FacetsAccumulator {
facetArrays.free(); // to get a cleared array for this partition
}
- HashMap
- * Compatibility:
- * Extensions of this class need to implement {@link #encode(int)} in order to
- * build the proper indicator (flags). When enough values were accumulated
- * (typically the batch size), extensions can call {@link #encodeChunk()} to
- * flush the indicator and the rest of the values.
+ * Extensions of this class need to implement {@link #encode(IntsRef, BytesRef)}
+ * in order to build the proper indicator (flags). When enough values were
+ * accumulated (typically the batch size), extensions can call
+ * {@link #encodeChunk(BytesRef)} to flush the indicator and the rest of the
+ * values.
*
* NOTE: flags encoders do not accept values ≤ 0 (zero) in their
- * {@link #encode(int)}. For performance reasons they do not check that
- * condition, however if such value is passed the result stream may be corrupt
- * or an exception will be thrown. Also, these encoders perform the best when
- * there are many consecutive small values (depends on the encoder
+ * {@link #encode(IntsRef, BytesRef)}. For performance reasons they do not check
+ * that condition, however if such value is passed the result stream may be
+ * corrupt or an exception will be thrown. Also, these encoders perform the best
+ * when there are many consecutive small values (depends on the encoder
* implementation). If that is not the case, the encoder will occupy 1 more byte
* for every batch number of integers, over whatever
* {@link VInt8IntEncoder} would have occupied. Therefore make sure to check
* whether your data fits into the conditions of the specific encoder.
*
* For the reasons mentioned above, these encoders are usually chained with
- * {@link UniqueValuesIntEncoder} and {@link DGapIntEncoder} in the following
- * manner:
* NOTE: this encoder assumes the values are given to
- * {@link #encode(int)} in an ascending sorted manner, which ensures only
+ * {@link #encode(IntsRef, BytesRef)} in an ascending sorted manner, which ensures only
* positive values are encoded and thus yields better performance. If you are
* not sure whether the values are sorted or not, it is possible to chain this
* encoder with {@link SortingIntEncoder} to ensure the values will be
@@ -37,17 +37,20 @@ import java.io.OutputStream;
*/
public class DGapIntEncoder extends IntEncoderFilter {
- private int prev = 0;
-
/** Initializes with the given encoder. */
public DGapIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void encode(int value) throws IOException {
- encoder.encode(value - prev);
- prev = value;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int prev = 0;
+ for (int i = values.offset; i < upto; i++) {
+ int tmp = values.ints[i];
+ values.ints[i] -= prev;
+ prev = tmp;
+ }
+ encoder.doEncode(values, buf, upto);
}
@Override
@@ -55,12 +58,6 @@ public class DGapIntEncoder extends IntEncoderFilter {
return new DGapIntDecoder(encoder.createMatchingDecoder());
}
- @Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- prev = 0;
- }
-
@Override
public String toString() {
return "DGap (" + encoder.toString() + ")";
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
index fe2b9c6d67b..270fcffca52 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,20 +21,17 @@ import java.io.InputStream;
*/
/**
- * Decodes data which was encoded by {@link EightFlagsIntEncoder}. Scans
- * the
+ * NOTE: it is ok to use the buffer's offset as the current position in
+ * the buffer (and modify it), it will be reset by
+ * {@link #decode(BytesRef, IntsRef)}.
+ */
+ protected abstract void doDecode(BytesRef buf, IntsRef values, int upto);
+
+ /**
+ * Called before {@link #doDecode(BytesRef, IntsRef, int)} so that decoders
+ * can reset their state.
+ */
+ protected void reset() {
+ // do nothing by default
}
/**
- * Decodes data received from the input stream, and returns one decoded
- * integer. If end of stream is reached, {@link #EOS} is returned.
- *
- * @return one decoded integer as long or {@link #EOS} if end-of-stream
- * reached.
- * @throws IOException if an I/O error occurs
+ * Decodes the values from the buffer into the given {@link IntsRef}. Note
+ * that {@code values.offset} and {@code values.length} are set to 0.
*/
- public abstract long decode() throws IOException;
+ public final void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0; // must do that because we cannot grow() them otherwise
+
+ // some decoders may use the buffer's offset as a position index, so save
+ // current offset.
+ int bufOffset = buf.offset;
+
+ reset();
+ doDecode(buf, values, buf.offset + buf.length);
+ assert values.offset == 0 : "offset should not have been modified by the decoder.";
+
+ // fix offset
+ buf.offset = bufOffset;
+ }
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
index 4055d750810..0a3197d6c6b 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
@@ -1,8 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,94 +21,47 @@ import java.io.OutputStream;
*/
/**
- * Encodes integers to a set {@link OutputStream}. Extending classes need to
- * override {@link #encode(int)} to encode the value using their encoding
- * algorithm. The default implementation of {@link #close()} closes the set
- * {@link OutputStream}.
- *
- * The default {@link #IntEncoder() constructor} is provided for convenience
- * only. One must call {@link #reInit(OutputStream)} before calling
- * {@link #encode(int)} or {@link #close()}.
- *
- * For convenience, each encoder implements {@link #createMatchingDecoder()} for
- * easy access to the matching decoder.
- *
- * NOTE: some implementations may buffer the encoded values in memory
- * (such as {@link IntEncoderFilter} implementations) and encoding will happen
- * only upon calling {@link #close()}. Therefore it is important to always call
- * {@link #close()} on the encoder at hand.
- *
- * NOTE: encoders are usually not thread safe, unless specifically
- * documented otherwise by an implementation.
+ * Encodes integers to a set {@link BytesRef}. For convenience, each encoder
+ * implements {@link #createMatchingDecoder()} for easy access to the matching
+ * decoder.
*
* @lucene.experimental
*/
-public abstract class IntEncoder implements Closeable {
+public abstract class IntEncoder {
- protected OutputStream out = null;
+ public IntEncoder() {}
/**
- * Default constructor, provided here for robustness: if in the future a
- * constructor with parameters will be added, this might break custom
- * implementations of this class which call this implicit constructor. So we
- * make it explicit to avoid any such issue in the future.
+ * Performs the actual encoding. Values should be read from
+ * {@link IntsRef#offset} up to {@code upto}. Also, it is guaranteed that
+ * {@code buf's} offset and length are set to 0 and the encoder is expected to
+ * update {@link BytesRef#length}, but not {@link BytesRef#offset}.
*/
- public IntEncoder() {
+ protected abstract void doEncode(IntsRef values, BytesRef buf, int upto);
+
+ /**
+ * Called before {@link #doEncode(IntsRef, BytesRef, int)} so that encoders
+ * can reset their state.
+ */
+ protected void reset() {
+ // do nothing by default
}
/**
- * Instructs the encoder to finish the encoding process. This method closes
- * the output stream which was specified by {@link #reInit(OutputStream)
- * reInit}. An implementation may do here additional cleanup required to
- * complete the encoding, such as flushing internal buffers, etc.
- * NOTE: overriding classes should make sure they either call
- *
- * NOTE: this method should create a new instance of the matching
- * decoder and leave the instance sharing to the caller. Returning the same
- * instance over and over is risky because encoders and decoders are not
- * thread safe.
+ * Returns an {@link IntDecoder} which can decode the values that were encoded
+ * with this encoder.
*/
public abstract IntDecoder createMatchingDecoder();
- /**
- * Reinitializes the encoder with the give {@link OutputStream}. For
- * re-usability it can be changed without the need to reconstruct a new
- * object.
- *
- * NOTE: after calling {@link #close()}, one must call
- * this method even if the output stream itself hasn't changed. An example
- * case is that the output stream wraps a byte[], and the output stream itself
- * is reset, but its instance hasn't changed. Some implementations of
- * {@link IntEncoder} may write some metadata about themselves to the output
- * stream, and therefore it is imperative that one calls this method before
- * encoding any data.
- */
- public void reInit(OutputStream out) {
- this.out = out;
- }
-
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
index 6c7a40340d0..ee2e5db7e9e 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
@@ -1,7 +1,5 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,17 +19,7 @@ import java.io.OutputStream;
*/
/**
- * An abstract implementation of {@link IntEncoder} which is served as a filter
- * on the values to encode. An encoder filter wraps another {@link IntEncoder}
- * which does the actual encoding. This allows for chaining filters and
- * encoders, such as:
- * The default implementation implements {@link #close()} by closing the wrapped
- * encoder and {@link #reInit(OutputStream)} by re-initializing the wrapped
- * encoder.
+ * An abstract implementation of {@link IntEncoder} which wraps another encoder.
*
* @lucene.experimental
*/
@@ -44,15 +32,8 @@ public abstract class IntEncoderFilter extends IntEncoder {
}
@Override
- public void close() throws IOException {
- // There is no need to call super.close(), since we don't pass the output
- // stream to super.
- encoder.close();
- }
-
- @Override
- public void reInit(OutputStream out) {
- encoder.reInit(out);
+ public void reset() {
+ encoder.reset();
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
index 6d00e049080..1cf33857280 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,54 +21,65 @@ import java.io.InputStream;
*/
/**
- * Decodes data which was encoded by {@link NOnesIntEncoder}. Uses a
- * {@link FourFlagsIntDecoder} to perform the actual encoding and translates the
- * values back as described in {@link NOnesIntEncoder}.
+ * Decodes values encoded encoded with {@link NOnesIntEncoder}.
*
- * @see NOnesIntEncoder
* @lucene.experimental
*/
public class NOnesIntDecoder extends FourFlagsIntDecoder {
- /** Number of consecutive '1's to generate upon decoding a '2'. */
- private int n;
-
- private int onesCounter;
-
+ // Number of consecutive '1's to generate upon decoding a '2'
+ private final int n;
+ private final IntsRef internalBuffer;
+
/**
* Constructs a decoder with a given N (Number of consecutive '1's which are
* translated into a single target value '2'.
*/
public NOnesIntDecoder(int n) {
this.n = n;
+ // initial size (room for 100 integers)
+ internalBuffer = new IntsRef(100);
}
@Override
- public long decode() throws IOException {
- // If we read '2', we should return n '1's.
- if (onesCounter > 0) {
- --onesCounter;
- return 1;
- }
-
- long decode = super.decode();
- if (decode == 1) {
- return 1;
- }
- if (decode == 2) {
- onesCounter = n - 1;
- return 1;
- }
- if (decode == 3) {
- return 2;
- }
- return decode == EOS ? EOS : decode - 1;
+ protected void reset() {
+ internalBuffer.length = 0;
+ super.reset();
}
-
+
@Override
- public void reInit(InputStream in) {
- super.reInit(in);
- onesCounter = 0;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ super.doDecode(buf, internalBuffer, upto);
+ if (values.ints.length < internalBuffer.length) {
+ // need space for internalBuffer.length to internalBuffer.length*N,
+ // grow mildly at first
+ values.grow(internalBuffer.length * n/2);
+ }
+
+ for (int i = 0; i < internalBuffer.length; i++) {
+ int decode = internalBuffer.ints[i];
+ if (decode == 1) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
+ }
+ // 1 is 1
+ values.ints[values.length++] = 1;
+ } else if (decode == 2) {
+ if (values.length + n >= values.ints.length) {
+ values.grow(values.length + n); // grow by few items, however not too many
+ }
+ // '2' means N 1's
+ for (int j = 0; j < n; j++) {
+ values.ints[values.length++] = 1;
+ }
+ } else {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
+ }
+ // any other value is val-1
+ values.ints[values.length++] = decode - 1;
+ }
+ }
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
index badfe1cd31d..956eea253a3 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -50,11 +50,10 @@ import java.io.OutputStream;
*/
public class NOnesIntEncoder extends FourFlagsIntEncoder {
+ private final IntsRef internalBuffer;
+
/** Number of consecutive '1's to be translated into single target value '2'. */
- private int n;
-
- /** Counts the number of consecutive ones seen. */
- private int onesCounter = 0;
+ private final int n;
/**
* Constructs an encoder with a given value of N (N: Number of consecutive
@@ -62,38 +61,48 @@ public class NOnesIntEncoder extends FourFlagsIntEncoder {
*/
public NOnesIntEncoder(int n) {
this.n = n;
+ internalBuffer = new IntsRef(n);
}
@Override
- public void close() throws IOException {
- // We might have ones in our buffer, encode them as neccesary.
- while (onesCounter-- > 0) {
- super.encode(1);
+ protected void reset() {
+ internalBuffer.length = 0;
+ super.reset();
+ }
+
+ @Override
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ // make sure the internal buffer is large enough
+ if (values.length > internalBuffer.ints.length) {
+ internalBuffer.grow(values.length);
}
-
- super.close();
- }
-
- @Override
- public void encode(int value) throws IOException {
- if (value == 1) {
- // Increment the number of consecutive ones seen so far
- if (++onesCounter == n) {
- super.encode(2);
- onesCounter = 0;
+
+ int onesCounter = 0;
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value == 1) {
+ // every N 1's should be encoded as '2'
+ if (++onesCounter == n) {
+ internalBuffer.ints[internalBuffer.length++] = 2;
+ onesCounter = 0;
+ }
+ } else {
+ // there might have been 1's that we need to encode
+ while (onesCounter > 0) {
+ --onesCounter;
+ internalBuffer.ints[internalBuffer.length++] = 1;
+ }
+
+ // encode value as value+1
+ internalBuffer.ints[internalBuffer.length++] = value + 1;
}
- return;
}
-
- // If it's not one - there might have been ones we had to encode prior to
- // this value
+ // there might have been 1's that we need to encode
while (onesCounter > 0) {
--onesCounter;
- super.encode(1);
+ internalBuffer.ints[internalBuffer.length++] = 1;
}
-
- // encode value + 1 --> the translation.
- super.encode(value + 1);
+ super.doEncode(internalBuffer, buf, internalBuffer.length);
}
@Override
@@ -101,12 +110,6 @@ public class NOnesIntEncoder extends FourFlagsIntEncoder {
return new NOnesIntDecoder(n);
}
- @Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- onesCounter = 0;
- }
-
@Override
public String toString() {
return "NOnes (" + n + ") (" + super.toString() + ")";
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
index 414af6effa7..af6fce26af7 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.StreamCorruptedException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,41 +21,24 @@ import java.io.StreamCorruptedException;
*/
/**
- * A simple stream decoder which can decode values encoded with
- * {@link SimpleIntEncoder}.
+ * Decodes values encoded with {@link SimpleIntEncoder}.
*
* @lucene.experimental
*/
public class SimpleIntDecoder extends IntDecoder {
- /**
- * reusable buffer - allocated only once as this is not a thread-safe object
- */
- private byte[] buffer = new byte[4];
-
@Override
- public long decode() throws IOException {
-
- // we need exactly 4 bytes to decode an int in this decoder impl, otherwise, throw an exception
- int offset = 0;
- while (offset < 4) {
- int nRead = in.read(buffer, offset, 4 - offset);
- if (nRead == -1) {
- if (offset > 0) {
- throw new StreamCorruptedException(
- "Need 4 bytes for decoding an int, got only " + offset);
- }
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
}
- offset += nRead;
+ values.ints[values.length++] =
+ ((buf.bytes[buf.offset++] & 0xFF) << 24) |
+ ((buf.bytes[buf.offset++] & 0xFF) << 16) |
+ ((buf.bytes[buf.offset++] & 0xFF) << 8) |
+ (buf.bytes[buf.offset++] & 0xFF);
}
-
- int v = buffer[3] & 0xff;
- v |= (buffer[2] << 8) & 0xff00;
- v |= (buffer[1] << 16) & 0xff0000;
- v |= (buffer[0] << 24) & 0xff000000;
-
- return v;
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
index 2e17ef29640..fd6a0206117 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,22 +27,21 @@ import java.io.IOException;
*/
public class SimpleIntEncoder extends IntEncoder {
- /**
- * This method makes sure the value wasn't previously encoded by checking
- * against the Set. If the value wasn't encoded, it's added to the Set, and
- * encoded with {#link Vint8#encode}
- *
- * @param value
- * an integer to be encoded
- * @throws IOException
- * possibly thrown by the OutputStream
- */
@Override
- public void encode(int value) throws IOException {
- out.write(value >>> 24);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write(value & 0xFF);
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ // ensure there's enough room in the buffer
+ int bytesNeeded = values.length * 4;
+ if (buf.bytes.length < bytesNeeded) {
+ buf.grow(bytesNeeded);
+ }
+
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ buf.bytes[buf.length++] = (byte) (value >>> 24);
+ buf.bytes[buf.length++] = (byte) ((value >> 16) & 0xFF);
+ buf.bytes[buf.length++] = (byte) ((value >> 8) & 0xFF);
+ buf.bytes[buf.length++] = (byte) (value & 0xFF);
+ }
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
index 9f6cee635ce..0ebb06efa85 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
@@ -1,9 +1,10 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.Arrays;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -23,47 +24,21 @@ import java.util.Arrays;
/**
* An {@link IntEncoderFilter} which sorts the values to encode in ascending
- * order before encoding them. Encoding therefore happens upon calling
- * {@link #close()}. Since this encoder is usually chained with another encoder
- * that relies on sorted values, it does not offer a default constructor.
+ * order before encoding them.
*
* @lucene.experimental
*/
public class SortingIntEncoder extends IntEncoderFilter {
- private float grow = 2.0f;
- private int index = 0;
- private int[] set = new int[1024];
-
/** Initializes with the given encoder. */
public SortingIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void close() throws IOException {
- if (index == 0) {
- return;
- }
-
- Arrays.sort(set, 0, index);
- for (int i = 0; i < index; i++) {
- encoder.encode(set[i]);
- }
- encoder.close();
- index = 0;
-
- super.close();
- }
-
- @Override
- public void encode(int value) throws IOException {
- if (index == set.length) {
- int[] newSet = new int[(int) (set.length * grow)];
- System.arraycopy(set, 0, newSet, 0, set.length);
- set = newSet;
- }
- set[index++] = value;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ Arrays.sort(values.ints, values.offset, upto);
+ encoder.doEncode(values, buf, upto);
}
@Override
@@ -71,12 +46,6 @@ public class SortingIntEncoder extends IntEncoderFilter {
return encoder.createMatchingDecoder();
}
- @Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- index = 0;
- }
-
@Override
public String toString() {
return "Sorting (" + encoder.toString() + ")";
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
index 3583402295f..c9a6be5c848 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +22,7 @@ import java.io.OutputStream;
/**
* An {@link IntEncoderFilter} which ensures only unique values are encoded. The
- * implementation assumes the values given to {@link #encode(int)} are sorted.
+ * implementation assumes the values given to {@link #encode(IntsRef, BytesRef)} are sorted.
* If this is not the case, you can chain this encoder with
* {@link SortingIntEncoder}.
*
@@ -30,26 +30,23 @@ import java.io.OutputStream;
*/
public final class UniqueValuesIntEncoder extends IntEncoderFilter {
- /**
- * Denotes an illegal value which we can use to init 'prev' to. Since all
- * encoded values are integers, this value is init to MAX_INT+1 and is of type
- * long. Therefore we are guaranteed not to get this value in encode.
- */
- private static final long ILLEGAL_VALUE = Integer.MAX_VALUE + 1;
-
- private long prev = ILLEGAL_VALUE;
-
/** Constructs a new instance with the given encoder. */
public UniqueValuesIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void encode(int value) throws IOException {
- if (prev != value) {
- encoder.encode(value);
- prev = value;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int prev = values.ints[values.offset];
+ int idx = values.offset + 1;
+ for (int i = idx; i < upto; i++) {
+ if (values.ints[i] != prev) {
+ values.ints[idx++] = values.ints[i];
+ prev = values.ints[i];
+ }
}
+ values.length = idx - values.offset;
+ encoder.doEncode(values, buf, idx);
}
@Override
@@ -57,12 +54,6 @@ public final class UniqueValuesIntEncoder extends IntEncoderFilter {
return encoder.createMatchingDecoder();
}
- @Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- prev = ILLEGAL_VALUE;
- }
-
@Override
public String toString() {
return "Unique (" + encoder.toString() + ")";
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
new file mode 100644
index 00000000000..267d52bae96
--- /dev/null
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
@@ -0,0 +1,138 @@
+package org.apache.lucene.util.encoding;
+
+import org.apache.lucene.util.BytesRef;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Variable-length encoding of 32-bit integers, into 8-bit bytes. A number is
+ * encoded as follows:
+ * An encoder encodes the integers that are passed to {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} into a
-set output stream (see {@link
-org.apache.lucene.util.encoding.IntEncoder#reInit(OutputStream)
-reInit}). One should always call {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close} when all
-integers have been encoded, to ensure proper finish by the encoder. Some
-encoders buffer values in-memory and encode in batches in order to
-optimize the encoding, and not closing them may result in loss of
-information or corrupt stream.
- A proper and typical usage of an encoder looks like this:
- Some encoders don't perform any encoding at all, or do not include an
+
+Some encoders don't perform any encoding at all, or do not include an
encoding logic. Those are called {@link
org.apache.lucene.util.encoding.IntEncoderFilter}s. A filter is an
encoder which delegates the encoding task to a given encoder, however
@@ -76,91 +35,6 @@ example is {@link org.apache.lucene.util.encoding.DGapIntEncoder}
which encodes the gaps between values rather than the values themselves.
Another example is {@link
org.apache.lucene.util.encoding.SortingIntEncoder} which sorts all the
-values in ascending order before they are sent for encoding. This
-encoder aggregates the values in its {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} implementation
-and decoding only happens upon calling {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close}.
-
- *
+ * NOTE: this class operates as a key to a map, and therefore you should
+ * implement {@code equals()} and {@code hashCode()} for proper behavior.
*
* @lucene.experimental
*/
@@ -41,29 +33,20 @@ public interface CategoryListIterator {
/**
* Initializes the iterator. This method must be called before any calls to
- * {@link #skipTo(int)}, and its return value indicates whether there are
- * any relevant documents for this iterator. If it returns false, any call
- * to {@link #skipTo(int)} will return false as well.hashCode() and equals() must be provided.
- *
- * NOTE: calling this method twice may result in skipping over
- * documents for some implementations. Also, calling it again after all
- * documents were consumed may yield unexpected behavior.
+ * {@link #getOrdinals(int, IntsRef)}, and its return value indicates whether there are
+ * any relevant documents for this iterator.
*/
public boolean init() throws IOException;
/**
- * Skips forward to document docId. Returns true iff this document exists
- * and has any categories. This method must be called before calling
- * {@link #nextCategory()} for a particular document.
- * NOTE: Users should call this method with increasing docIds, and
- * implementations can assume that this is the case.
+ * Stores the category ordinals of the given document ID in the given
+ * {@link IntsRef}, starting at position 0 upto {@link IntsRef#length}. Grows
+ * the {@link IntsRef} if it is not large enough.
+ *
+ *
- * IndexReader reader = [open your reader];
- * Term t = new Term("field", "where-payload-exists");
- * CategoryListIterator cli = new PayloadIntDecodingIterator(reader, t);
- * if (!cli.init()) {
- * // it means there are no payloads / documents associated with that term.
- * // Usually a sanity check. However, init() must be called.
- * }
- * DocIdSetIterator disi = [you usually iterate on something else, such as a Scorer];
- * int doc;
- * while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- * cli.setdoc(doc);
- * long category;
- * while ((category = cli.nextCategory()) < Integer.MAX_VALUE) {
- * }
- * }
- *
+ * A {@link CategoryListIterator} which reads the category ordinals from a
+ * payload.
*
* @lucene.experimental
*/
-public class PayloadIntDecodingIterator implements CategoryListIterator {
+public class PayloadCategoryListIteraor implements CategoryListIterator {
- private final UnsafeByteArrayInputStream ubais;
private final IntDecoder decoder;
-
private final IndexReader indexReader;
private final Term term;
private final PayloadIterator pi;
private final int hashCode;
- public PayloadIntDecodingIterator(IndexReader indexReader, Term term, IntDecoder decoder) throws IOException {
+ public PayloadCategoryListIteraor(IndexReader indexReader, Term term, IntDecoder decoder) throws IOException {
pi = new PayloadIterator(indexReader, term);
- ubais = new UnsafeByteArrayInputStream();
this.decoder = decoder;
hashCode = indexReader.hashCode() ^ term.hashCode();
this.term = term;
@@ -72,10 +49,10 @@ public class PayloadIntDecodingIterator implements CategoryListIterator {
@Override
public boolean equals(Object other) {
- if (!(other instanceof PayloadIntDecodingIterator)) {
+ if (!(other instanceof PayloadCategoryListIteraor)) {
return false;
}
- PayloadIntDecodingIterator that = (PayloadIntDecodingIterator) other;
+ PayloadCategoryListIteraor that = (PayloadCategoryListIteraor) other;
if (hashCode != that.hashCode) {
return false;
}
@@ -95,21 +72,12 @@ public class PayloadIntDecodingIterator implements CategoryListIterator {
}
@Override
- public long nextCategory() throws IOException {
- return decoder.decode();
- }
-
- @Override
- public boolean skipTo(int docId) throws IOException {
- if (!pi.setdoc(docId)) {
- return false;
+ public void getOrdinals(int docID, IntsRef ints) throws IOException {
+ ints.length = 0;
+ BytesRef payload = pi.getPayload(docID);
+ if (payload != null) {
+ decoder.decode(payload, ints);
}
-
- // Initializing the decoding mechanism with the new payload data
- BytesRef data = pi.getPayload();
- ubais.reInit(data.bytes, data.offset, data.length + data.offset);
- decoder.reInit(ubais);
- return true;
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
index a12f2cd91f6..7cc7527280d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
@@ -34,9 +34,9 @@ import org.apache.lucene.util.BytesRef;
* A utility class for iterating through a posting list of a given term and
* retrieving the payload of the first position in every document. For
* efficiency, this class does not check if documents passed to
- * {@link #setdoc(int)} are deleted, since it is usually used to iterate on
+ * {@link #getPayload(int)} are deleted, since it is usually used to iterate on
* payloads of documents that matched a query. If you need to skip over deleted
- * documents, you should do so before calling {@link #setdoc(int)}.
+ * documents, you should do so before calling {@link #getPayload(int)}.
*
* @lucene.experimental
*/
@@ -84,8 +84,8 @@ public class PayloadIterator {
/**
* Initialize the iterator. Should be done before the first call to
- * {@link #setdoc(int)}. Returns {@code false} if no category list is found,
- * or the category list has no documents.
+ * {@link #getPayload(int)}. Returns {@code false} if no category list is
+ * found, or the category list has no documents.
*/
public boolean init() throws IOException {
nextSegment();
@@ -93,30 +93,29 @@ public class PayloadIterator {
}
/**
- * Skip forward to document docId. Return true if this document exists and
- * has any payload.
- *
- *
- * Examples:
- *
- *
- * This encoder/decoder will correctly handle any 32-bit integer, but for negative numbers,
- * and positive numbers with more than 28 significant bits, encoding requires 5 bytes; this
- * is not an efficient encoding scheme for large
- * positive numbers or any negative number.
- *
- * This class has been used in products that have shipped to customers, and is needed to
- * decode legacy data. Do not modify this class in ways that will break compatibility.
- *
- * @lucene.experimental
- */
-public class Vint8 {
-
- /**
- * Because Java lacks call-by-reference, this class boxes the decoding position, which
- * is initially set by the caller, and returned after decoding, incremented by the number
- * of bytes processed.
- */
- public static class Position {
- /**
- * Creates a position value set to zero.
- */
- public Position() {
- // The initial position is zero by default.
- }
- /**
- * Creates a position set to {@code initialPosition}.
- * @param initialPosition The starting decoding position in the source buffer.
- */
- public Position(int initialPosition) {
- this.pos = initialPosition;
- }
- /**
- * The value passed by reference.
- */
- public int pos;
- }
-
- /**
- * Returns the number of bytes needed to encode {@code number}.
- * @param number The number whose encoded length is needed.
- * @return The number of bytes needed to encode {@code number}.
- */
- public static int bytesNeeded(int number) {
- if ((number & ~0x7F) == 0) {
- return 1;
- } else if ((number & ~0x3FFF) == 0) {
- return 2;
- } else if ((number & ~0x1FFFFF) == 0) {
- return 3;
- } else if ((number & ~0xFFFFFFF) == 0) {
- return 4;
- } else {
- return 5;
- }
- }
-
- /**
- * The maximum number of bytes needed to encode a number using {@code Vint8}.
- */
- public static final int MAXIMUM_BYTES_NEEDED = 5;
-
- /**
- * Encodes {@code number} to {@code out}.
- * @param number The value to be written in encoded form, to {@code out}.
- * @param out The output stream receiving the encoded bytes.
- * @exception IOException If there is a problem writing to {@code out}.
- */
- public static void encode(int number, OutputStream out) throws IOException {
- if ((number & ~0x7F) == 0) {
- out.write(number);
- } else if ((number & ~0x3FFF) == 0) {
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else if ((number & ~0x1FFFFF) == 0) {
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else if ((number & ~0xFFFFFFF) == 0) {
- out.write(0x80 | (number >> 21));
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else {
- out.write(0x80 | (number >> 28));
- out.write(0x80 | (number >> 21));
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- }
- }
-
- /**
- * Encodes {@code number} into {@code dest}, starting at offset {@code start} from
- * the beginning of the array. This method assumes {@code dest} is large enough to
- * hold the required number of bytes.
- * @param number The number to be encoded.
- * @param dest The destination array.
- * @param start The starting offset in the array.
- * @return The number of bytes used in the array.
- */
- public static int encode(int number, byte[] dest, int start) {
- if ((number & ~0x7F) == 0) {
- dest[start] = (byte) number;
- return 1;
- } else if ((number & ~0x3FFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 1] = (byte) (number & 0x7F);
- return 2;
- } else if ((number & ~0x1FFFFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 1] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 2] = (byte) (number & 0x7F);
- return 3;
- } else if ((number & ~0xFFFFFFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0xFE00000) >> 21));
- dest[start + 1] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 2] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 3] = (byte) (number & 0x7F);
- return 4;
- } else {
- dest[start] = (byte) (0x80 | ((number & 0xF0000000) >> 28));
- dest[start + 1] = (byte) (0x80 | ((number & 0xFE00000) >> 21));
- dest[start + 2] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 3] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 4] = (byte) (number & 0x7F);
- return 5;
- }
- }
-
- /**
- * Decodes a 32-bit integer from {@code bytes}, beginning at offset {@code pos.pos}.
- * The decoded value is returned, and {@code pos.pos} is incremented by the number of
- * bytes processed.
- * @param bytes The byte array containing an encoded value.
- * @param pos On entry, the starting position in the array; on return, one greater
- * than the position of the last byte decoded in the call.
- * @return The decoded value.
- */
- public static int decode(byte[] bytes, Position pos) {
- int value = 0;
- while (true) {
- byte first = bytes[pos.pos];
- ++pos.pos;
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- return value;
- }
- value <<= 7;
- }
- }
-
- /**
- * Decodes a 32-bit integer from bytes read from {@code in}. Bytes are read,
- * one at a time, from {@code in}, and it is assumed they represent a 32-bit
- * integer encoded using this class's encoding scheme. The decoded value is
- * returned.
- * @param in The input stream containing the encoded bytes.
- * @return The decoded value.
- * @exception EOFException If the stream ends before a value has been decoded.
- */
- public static int decode(InputStream in) throws IOException {
- int value = 0;
- while (true) {
- int first = in.read();
- if (first < 0) {
- throw new EOFException();
- }
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- return value;
- }
- value <<= 7;
- }
- }
-
- /**
- * The default ctor is made private because all methods of this class are static.
- */
- private Vint8() {
- // Just making it impossible to instantiate.
- }
-
-}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
index c640c53b6cc..a19550c29e5 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,38 +27,31 @@ import java.io.OutputStream;
* read more on the two implementations {@link FourFlagsIntEncoder} and
* {@link EightFlagsIntEncoder}.
*
- * IntEncoder fourFlags =
- * new SortingEncoderFilter(new UniqueValuesIntEncoder(new DGapIntEncoder(new FlagsIntEncoderImpl())));
- *
indicator, one flag (1-bits) at a time, and decodes extra
- * data using {@link VInt8IntDecoder}.
+ * Decodes values encoded with {@link EightFlagsIntEncoder}.
*
- * @see EightFlagsIntEncoder
* @lucene.experimental
*/
public class EightFlagsIntDecoder extends IntDecoder {
- /**
+ /*
* Holds all combinations of indicator for fast decoding (saves time
* on real-time bit manipulation)
*/
- private static final byte[][] decodeTable = new byte[256][8];
+ private static final byte[][] DECODE_TABLE = new byte[256][8];
/** Generating all combinations of indicator into separate flags. */
static {
@@ -42,45 +39,36 @@ public class EightFlagsIntDecoder extends IntDecoder {
--i;
for (int j = 8; j != 0;) {
--j;
- decodeTable[i][j] = (byte) ((i >>> j) & 0x1);
+ DECODE_TABLE[i][j] = (byte) ((i >>> j) & 0x1);
}
}
}
- private final IntDecoder decoder = new VInt8IntDecoder();
-
- /** The indicator for decoding a chunk of 8 integers. */
- private int indicator;
-
- /** Used as an ordinal of 0 - 7, as the decoder decodes chunks of 8 integers. */
- private int ordinal = 0;
-
@Override
- public long decode() throws IOException {
- // If we've decoded 8 integers, read the next indicator.
- if ((ordinal & 0x7) == 0) {
- indicator = in.read();
- if (indicator < 0) {
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ // read indicator
+ int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int ordinal = 0;
+
+ int capacityNeeded = values.length + 8;
+ if (values.ints.length < capacityNeeded) {
+ values.grow(capacityNeeded);
+ }
+
+ // process indicator, until we read 8 values, or end-of-buffer
+ while (ordinal != 8) {
+ if (DECODE_TABLE[indicator][ordinal++] == 0) {
+ if (buf.offset == upto) { // end of buffer
+ return;
+ }
+ // decode the value from the stream.
+ values.ints[values.length++] = VInt8.decode(buf) + 2;
+ } else {
+ values.ints[values.length++] = 1;
+ }
}
- ordinal = 0;
}
-
- if (decodeTable[indicator][ordinal++] == 0) {
- // decode the value from the stream.
- long decode = decoder.decode();
- return decode == EOS ? EOS : decode + 2;
- }
-
- return 1;
- }
-
- @Override
- public void reInit(InputStream in) {
- super.reInit(in);
- decoder.reInit(in);
- ordinal = 0;
- indicator = 0;
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
index ac427725674..143660a4742 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,14 +21,15 @@ import java.io.IOException;
*/
/**
- * A {@link ChunksIntEncoder} which encodes data in chunks of 8. Every group starts with a single
- * byte (called indicator) which represents 8 - 1 bit flags, where the value:
+ * A {@link ChunksIntEncoder} which encodes data in chunks of 8. Every group
+ * starts with a single byte (called indicator) which represents 8 - 1 bit
+ * flags, where the value:
*
*
* Encoding example:
*
- * Since value 0 is illegal, and 1 is encoded in the indicator, the actual
- * value that is encoded is value-2, which saves some more bits.
+ * Since value 0 is illegal, and 1 is encoded in the indicator, the actual value
+ * that is encoded is value-2, which saves some more bits.
*
@@ -46,28 +48,36 @@ import java.io.IOException;
*/
public class EightFlagsIntEncoder extends ChunksIntEncoder {
- /**
+ /*
* Holds all combinations of indicator flags for fast encoding (saves
* time on bit manipulation at encode time)
*/
- private static byte[] encodeTable = new byte[] { 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, (byte) 0x80 };
+ private static final byte[] ENCODE_TABLE = new byte[] { 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, (byte) 0x80 };
public EightFlagsIntEncoder() {
super(8);
}
@Override
- public void encode(int data) throws IOException {
- if (data == 1) {
- indicator |= encodeTable[ordinal];
- } else {
- encodeQueue[encodeQueueSize++] = data - 2;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value == 1) {
+ indicator |= ENCODE_TABLE[ordinal];
+ } else {
+ encodeQueue.ints[encodeQueue.length++] = value - 2;
+ }
+ ++ordinal;
+
+ // encode the chunk and the indicator
+ if (ordinal == 8) {
+ encodeChunk(buf);
+ }
}
- ++ordinal;
-
- // If 8 values were encoded thus far, 'flush' them including the indicator.
- if ((ordinal & 0x7) == 0) {
- encodeChunk();
+
+ // encode remaining values
+ if (ordinal != 0) {
+ encodeChunk(buf);
}
}
@@ -78,7 +88,7 @@ public class EightFlagsIntEncoder extends ChunksIntEncoder {
@Override
public String toString() {
- return "EightFlags (" + encoder.toString() + ")";
+ return "EightFlags (VInt)";
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
index a597f9e2a38..ebc161ac9cd 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,11 +21,8 @@ import java.io.InputStream;
*/
/**
- * Decodes data which was encoded by {@link FourFlagsIntEncoder}. Scans
- * the
indicator, one flag (1-bits) at a time, and decodes extra
- * data using {@link VInt8IntDecoder}.
+ * Decodes values encoded with {@link FourFlagsIntEncoder}.
*
- * @see FourFlagsIntEncoder
* @lucene.experimental
*/
public class FourFlagsIntDecoder extends IntDecoder {
@@ -34,7 +31,7 @@ public class FourFlagsIntDecoder extends IntDecoder {
* Holds all combinations of indicator for fast decoding (saves time
* on real-time bit manipulation)
*/
- private final static byte[][] decodeTable = new byte[256][4];
+ private final static byte[][] DECODE_TABLE = new byte[256][4];
/** Generating all combinations of indicator into separate flags. */
static {
@@ -42,46 +39,36 @@ public class FourFlagsIntDecoder extends IntDecoder {
--i;
for (int j = 4; j != 0;) {
--j;
- decodeTable[i][j] = (byte) ((i >>> (j << 1)) & 0x3);
+ DECODE_TABLE[i][j] = (byte) ((i >>> (j << 1)) & 0x3);
}
}
}
- private final IntDecoder decoder = new VInt8IntDecoder();
-
- /** The indicator for decoding a chunk of 4 integers. */
- private int indicator;
-
- /** Used as an ordinal of 0 - 3, as the decoder decodes chunks of 4 integers. */
- private int ordinal = 0;
-
@Override
- public long decode() throws IOException {
- // If we've decoded 8 integers, read the next indicator.
- if ((ordinal & 0x3) == 0) {
- indicator = in.read();
- if (indicator < 0) {
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ // read indicator
+ int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int ordinal = 0;
+
+ int capacityNeeded = values.length + 4;
+ if (values.ints.length < capacityNeeded) {
+ values.grow(capacityNeeded);
+ }
+
+ while (ordinal != 4) {
+ byte decodeVal = DECODE_TABLE[indicator][ordinal++];
+ if (decodeVal == 0) {
+ if (buf.offset == upto) { // end of buffer
+ return;
+ }
+ // decode the value from the stream.
+ values.ints[values.length++] = VInt8.decode(buf) + 4;
+ } else {
+ values.ints[values.length++] = decodeVal;
+ }
}
- ordinal = 0;
}
-
- byte decodeVal = decodeTable[indicator][ordinal++];
- if (decodeVal == 0) {
- // decode the value from the stream.
- long decode = decoder.decode();
- return decode == EOS ? EOS : decode + 4;
- }
-
- return decodeVal;
- }
-
- @Override
- public void reInit(InputStream in) {
- super.reInit(in);
- decoder.reInit(in);
- ordinal = 0;
- indicator = 0;
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
index a1e227d5cdf..535a90fb60c 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -48,11 +49,11 @@ import java.io.IOException;
*/
public class FourFlagsIntEncoder extends ChunksIntEncoder {
- /**
+ /*
* Holds all combinations of indicator flags for fast encoding (saves
* time on bit manipulation @ encode time)
*/
- private static byte[][] encodeTable = new byte[][] {
+ private static final byte[][] ENCODE_TABLE = new byte[][] {
new byte[] { 0x00, 0x00, 0x00, 0x00 },
new byte[] { 0x01, 0x04, 0x10, 0x40 },
new byte[] { 0x02, 0x08, 0x20, (byte) 0x80 },
@@ -63,26 +64,26 @@ public class FourFlagsIntEncoder extends ChunksIntEncoder {
super(4);
}
- /**
- * Small values (<=3) are stored in the indicator while larger
- * values are saved for later encoding in the {@link #encodeQueue}. Since
- * Vint8 will only encode values larger or equal to 4, the values saves for
- * encoded are transformed to (value - 4).
- * When a chunk is ready (got 4 values), the {@link #encodeChunk()}
- * takes control.
- */
@Override
- public void encode(int data) throws IOException {
- if (data <= 3) {
- indicator |= encodeTable[data][ordinal];
- } else {
- encodeQueue[encodeQueueSize++] = data - 4;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value <= 3) {
+ indicator |= ENCODE_TABLE[value][ordinal];
+ } else {
+ encodeQueue.ints[encodeQueue.length++] = value - 4;
+ }
+ ++ordinal;
+
+ // encode the chunk and the indicator
+ if (ordinal == 4) {
+ encodeChunk(buf);
+ }
}
- ++ordinal;
-
- // If 4 values were encoded thus far, 'flush' them including the indicator.
- if ((ordinal & 0x3) == 0) {
- encodeChunk();
+
+ // encode remaining values
+ if (ordinal != 0) {
+ encodeChunk(buf);
}
}
@@ -93,7 +94,7 @@ public class FourFlagsIntEncoder extends ChunksIntEncoder {
@Override
public String toString() {
- return "FourFlags (" + encoder.toString() + ")";
+ return "FourFlags (VInt)";
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
index faabad4c8ed..acf83cc6158 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,33 +21,50 @@ import java.io.InputStream;
*/
/**
- * Decodes integers from a set {@link InputStream}. For re-usability, the
- * decoder's input stream can be set by ({@link #reInit(InputStream)}).
- * By design, Decoders are NOT thread-safe.
+ * Decodes integers from a set {@link BytesRef}.
*
* @lucene.experimental
*/
public abstract class IntDecoder {
- /** A special long value which is used to indicate end-of-stream has reached. */
- public static final long EOS = 0x100000000L;
-
- /** Input stream from which the encoded bytes are read */
- protected InputStream in;
-
- /** Sets the input stream from which the encoded data is read. */
- public void reInit(InputStream in) {
- this.in = in;
+ /**
+ * Performs the actual decoding. Values should be read from
+ * {@link BytesRef#offset} up to {@code upto}. Also, {@code values} offset and
+ * length are set to 0 and the encoder is expected to update
+ * {@link IntsRef#length}, but not {@link IntsRef#offset}.
+ *
+ *
- * Once this method was called, no further calls to {@link #encode(int)
- * encode} should be made before first calling {@link #reInit(OutputStream)
- * reInit}.
- * super.close() or close the output stream themselves.
+ * Encodes the values to the given buffer. Note that the buffer's offset and
+ * length are set to 0.
*/
- @Override
- public void close() throws IOException {
- if (out != null) {
- out.close();
- }
+ public final void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ reset();
+ doEncode(values, buf, values.offset + values.length);
+ assert buf.offset == 0 : "offset should not have been modified by the encoder.";
}
/**
- * Encodes an integer to the output stream given in
- * {@link #reInit(OutputStream) reInit}
- */
- public abstract void encode(int value) throws IOException;
-
- /**
- * Returns an {@link IntDecoder} which matches this encoder. Every encoder
- * must return an {@link IntDecoder} and null is not a valid
- * value. If an encoder is just a filter, it should at least return its
- * wrapped encoder's matching decoder.
- *
- * new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEnoder()));
- * {@link UniqueValuesIntEncoder} followed by {@link DGapIntEncoder}
-
+ *
+ * Examples:
+ *
+ *
+ * {@link #encode(int, BytesRef)} and {@link #decode(BytesRef)} will correctly
+ * handle any 32-bit integer, but for negative numbers, and positive numbers
+ * with more than 28 significant bits, encoding requires 5 bytes; this is not an
+ * efficient encoding scheme for large positive numbers or any negative number.
+ *
+ * @lucene.experimental
+ */
+public class VInt8 {
+
+ /** The maximum number of bytes needed to encode an integer. */
+ public static final int MAXIMUM_BYTES_NEEDED = 5;
+
+ /**
+ * Decodes an int from the given bytes, starting at {@link BytesRef#offset}.
+ * Returns the decoded bytes and updates {@link BytesRef#offset}.
+ */
+ public static int decode(BytesRef bytes) {
+ /*
+ This is the original code of this method, but a Hotspot bug
+ corrupted the for-loop of DataInput.readVInt() (see LUCENE-2975)
+ so the loop was unwounded here too, to be on the safe side
+ int value = 0;
+ while (true) {
+ byte first = bytes.bytes[bytes.offset++];
+ value |= first & 0x7F;
+ if ((first & 0x80) == 0) {
+ return value;
+ }
+ value <<= 7;
+ }
+ */
+
+ // byte 1
+ byte b = bytes.bytes[bytes.offset++];
+ if (b >= 0) return b;
+
+ // byte 2
+ int value = b & 0x7F;
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 3
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 4
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 5
+ b = bytes.bytes[bytes.offset++];
+ return (value << 7) | b & 0x7F;
+ }
+
+ /**
+ * Encodes the given number into bytes, starting at {@link BytesRef#length}.
+ * Assumes that the array is large enough.
+ */
+ public static void encode(int value, BytesRef bytes) {
+ if ((value & ~0x7F) == 0) {
+ bytes.bytes[bytes.length] = (byte) value;
+ bytes.length++;
+ } else if ((value & ~0x3FFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 1] = (byte) (value & 0x7F);
+ bytes.length += 2;
+ } else if ((value & ~0x1FFFFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 2] = (byte) (value & 0x7F);
+ bytes.length += 3;
+ } else if ((value & ~0xFFFFFFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 3] = (byte) (value & 0x7F);
+ bytes.length += 4;
+ } else {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ bytes.bytes[bytes.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 4] = (byte) (value & 0x7F);
+ bytes.length += 5;
+ }
+ }
+
+ private VInt8() {
+ // Just making it impossible to instantiate.
+ }
+
+}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
index 2beaf773d78..e9fe5600c9a 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,33 +21,19 @@ import java.io.IOException;
*/
/**
- * An {@link IntDecoder} which can decode values encoded by
- * {@link VInt8IntEncoder}.
+ * Decodes values encoded by {@link VInt8IntEncoder}.
*
* @lucene.experimental
*/
public class VInt8IntDecoder extends IntDecoder {
- private boolean legalEOS = true;
-
@Override
- public long decode() throws IOException {
- int value = 0;
- while (true) {
- int first = in.read();
- if (first < 0) {
- if (!legalEOS) {
- throw new IOException("Unexpected End-Of-Stream");
- }
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
}
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- legalEOS = true;
- return value;
- }
- legalEOS = false;
- value <<= 7;
+ values.ints[values.length++] = VInt8.decode(buf);
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
index 8fe6fbc9e09..7c62bf3e035 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -49,27 +50,14 @@ import java.io.IOException;
public class VInt8IntEncoder extends IntEncoder {
@Override
- public void encode(int value) throws IOException {
- if ((value & ~0x7F) == 0) {
- out.write(value);
- } else if ((value & ~0x3FFF) == 0) {
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0x1FFFFF) == 0) {
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0xFFFFFFF) == 0) {
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else {
- out.write(0x80 | (value >> 28));
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
+ if (buf.bytes.length < maxBytesNeeded) {
+ buf.grow(maxBytesNeeded);
+ }
+
+ for (int i = values.offset; i < upto; i++) {
+ VInt8.encode(values.ints[i], buf);
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html b/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
index 2e50e8f38eb..8a81b258e34 100644
--- a/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
+++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
@@ -25,49 +25,8 @@ mechanisms to create new ones. The super class for all encoders is
encoders there is a matching {@link
org.apache.lucene.util.encoding.IntDecoder} implementation (not all
encoders need a decoder).
-
-Each encoder also implements {@link
-org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} which returns the matching decoder for this encoder.
-As mentioned above, not all encoders have a matching decoder (like some
-encoder filters which are explained next), however every encoder should
-return a decoder following a call to that method. To complete the
-example above, one can easily iterate over the decoded values like this:
-
-int[] data = <the values to encode>
-IntEncoder encoder = new VInt8IntEncoder();
-OutputStream out = new ByteArrayOutputStream();
-encoder.reInit(out);
-for (int val : data) {
- encoder.encode(val);
-}
-encoder.close();
-
-// Print the bytes in binary
-byte[] bytes = out.toByteArray();
-for (byte b : bytes) {
- System.out.println(Integer.toBinaryString(b));
-}
-
-
-IntDecoder d = e.createMatchingDecoder();
-d.reInit(new ByteArrayInputStream(bytes));
-long val;
-while ((val = d.decode()) != IntDecoder.EOS) {
- System.out.println(val);
-}
-Extending IntEncoder
-Extending {@link org.apache.lucene.util.encoding.IntEncoder} is a very
-easy task. One only needs to implement {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} and
-{@link org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} as the base implementation takes care of
-re-initializing the output stream and closing it. The following example
-illustrates how can one write an encoder (and a matching decoder) which
-'tags' the stream with type/ID of the encoder. Such tagging is important
-in scenarios where an application uses different encoders for different
-streams, and wants to manage some sort of mapping between an encoder ID
-to an IntEncoder/Decoder implementation, so a proper decoder will be
-initialized on the fly:
-
-And the matching decoder:
-
-public class TaggingIntEncoder extends IntEncoderFilter {
-
- public TaggingIntEncoder(IntEncoder encoder) {
- super(encoder);
- }
-
- @Override
- public void encode(int value) throws IOException {
- encoder.encode(value);
- }
-
- @Override
- public IntDecoder createMatchingDecoder() {
- return new TaggingIntDecoder();
- }
-
- @Override
- public void reInit(OutputStream out) {
- super.reInit(os);
- // Assumes the application has a static EncodersMap class which is able to
- // return a unique ID for a given encoder.
- int encoderID = EncodersMap.getID(encoder);
- this.out.write(encoderID);
- }
-
- @Override
- public String toString() {
- return "Tagging (" + encoder.toString() + ")";
- }
-
-}
-
-The example implements
-public class TaggingIntDecoder extends IntDecoder {
-
- // Will be initialized upon calling reInit.
- private IntDecoder decoder;
-
- @Override
- public void reInit(InputStream in) {
- super.reInit(in);
-
- // Read the ID of the encoder that tagged this stream.
- int encoderID = in.read();
-
- // Assumes EncodersMap can return the proper IntEncoder given the ID.
- decoder = EncodersMap.getEncoder(encoderID).createMatchingDecoder();
- }
-
- @Override
- public long decode() throws IOException {
- return decoder.decode();
- }
-
- @Override
- public String toString() {
- return "Tagging (" + decoder == null ? "none" : decoder.toString() + ")";
- }
-
-}
-TaggingIntEncoder as a filter over another
-encoder. Even though it does not do any filtering on the actual values, it feels
-right to present it as a filter. Anyway, this is just an example code and one
-can choose to implement it however it makes sense to the application. For
-simplicity, error checking was omitted from the sample code.
+values in ascending order before they are sent for encoding.