LUCENE-5670: add skip/FinalOutput to FST Outputs

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1596369 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2014-05-20 19:07:47 +00:00
parent 1e524542e1
commit 86e7b08381
12 changed files with 97 additions and 11 deletions

View File

@ -182,6 +182,9 @@ Optimizations
to 8 (for int/float) and 16 (for long/double), for faster indexing
time and smaller indices. (Robert Muir, Uwe Schindler, Mike McCandless)
* LUCENE-5670: Add skip/FinalOutput to FST Outputs. (Christian
Ziech via Mike McCandless).
Bug fixes
* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws

View File

@ -300,6 +300,33 @@ class FSTTermOutputs extends Outputs<FSTTermOutputs.TermData> {
return new TermData(longs, bytes, docFreq, totalTermFreq);
}
@Override
public void skipOutput(DataInput in) throws IOException {
int bits = in.readByte() & 0xff;
int bit0 = bits & 1;
int bit1 = bits & 2;
int bit2 = bits & 4;
int bytesSize = (bits >>> 3);
if (bit1 > 0 && bytesSize == 0) { // determine extra length
bytesSize = in.readVInt();
}
if (bit0 > 0) { // not all-zero case
for (int pos = 0; pos < longsSize; pos++) {
in.readVLong();
}
}
if (bit1 > 0) { // bytes exists
in.skipBytes(bytesSize);
}
if (bit2 > 0) { // stats exist
int code = in.readVInt();
if (hasPos && (code & 1) == 0) {
in.readVLong();
}
}
}
@Override
public TermData getNoOutput() {
return NO_OUTPUT;

View File

@ -128,6 +128,14 @@ public final class ByteSequenceOutputs extends Outputs<BytesRef> {
}
}
@Override
public void skipOutput(DataInput in) throws IOException {
final int len = in.readVInt();
if (len != 0) {
in.skipBytes(len);
}
}
@Override
public BytesRef getNoOutput() {
return NO_OUTPUT;

View File

@ -362,7 +362,7 @@ class BytesStore extends DataOutput {
}
@Override
public void skipBytes(int count) {
public void skipBytes(long count) {
setPosition(getPosition() + count);
}
@ -430,7 +430,7 @@ class BytesStore extends DataOutput {
}
@Override
public void skipBytes(int count) {
public void skipBytes(long count) {
setPosition(getPosition() - count);
}

View File

@ -133,6 +133,14 @@ public final class CharSequenceOutputs extends Outputs<CharsRef> {
}
}
@Override
public void skipOutput(DataInput in) throws IOException {
final int len = in.readVInt();
for(int idx=0;idx<len;idx++) {
in.readVInt();
}
}
@Override
public CharsRef getNoOutput() {
return NO_OUTPUT;

View File

@ -893,10 +893,10 @@ public final class FST<T> {
// skip this arc:
readLabel(in);
if (arc.flag(BIT_ARC_HAS_OUTPUT)) {
outputs.read(in);
outputs.skipOutput(in);
}
if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) {
outputs.readFinalOutput(in);
outputs.skipFinalOutput(in);
}
if (arc.flag(BIT_STOP_NODE)) {
} else if (arc.flag(BIT_TARGET_NEXT)) {
@ -1252,11 +1252,11 @@ public final class FST<T> {
readLabel(in);
if (flag(flags, BIT_ARC_HAS_OUTPUT)) {
outputs.read(in);
outputs.skipOutput(in);
}
if (flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) {
outputs.readFinalOutput(in);
outputs.skipFinalOutput(in);
}
if (!flag(flags, BIT_STOP_NODE) && !flag(flags, BIT_TARGET_NEXT)) {
@ -1330,9 +1330,6 @@ public final class FST<T> {
/** Returns true if this reader uses reversed bytes
* under-the-hood. */
public abstract boolean reversed();
/** Skips bytes. */
public abstract void skipBytes(int count);
}
private static class ArcAndState<T> {

View File

@ -41,7 +41,7 @@ final class ForwardBytesReader extends FST.BytesReader {
}
@Override
public void skipBytes(int count) {
public void skipBytes(long count) {
pos += count;
}

View File

@ -132,6 +132,17 @@ public final class IntSequenceOutputs extends Outputs<IntsRef> {
}
}
@Override
public void skipOutput(DataInput in) throws IOException {
final int len = in.readVInt();
if (len == 0) {
return;
}
for(int idx=0;idx<len;idx++) {
in.readVInt();
}
}
@Override
public IntsRef getNoOutput() {
return NO_OUTPUT;

View File

@ -63,6 +63,12 @@ public abstract class Outputs<T> {
* #write(Object, DataOutput)}. */
public abstract T read(DataInput in) throws IOException;
/** Skip the output; defaults to just calling {@link #read}
* and discarding the result. */
public void skipOutput(DataInput in) throws IOException {
read(in);
}
/** Decode an output value previously written with {@link
* #writeFinalOutput(Object, DataOutput)}. By default this
* just calls {@link #read(DataInput)}. */
@ -70,6 +76,13 @@ public abstract class Outputs<T> {
return read(in);
}
/** Skip the output previously written with {@link #writeFinalOutput};
* defaults to just calling {@link #readFinalOutput} and discarding
* the result. */
public void skipFinalOutput(DataInput in) throws IOException {
skipOutput(in);
}
/** NOTE: this output is compared with == so you must
* ensure that all methods return the single object if
* it's really no output */

View File

@ -149,6 +149,12 @@ public class PairOutputs<A,B> extends Outputs<PairOutputs.Pair<A,B>> {
return newPair(output1, output2);
}
@Override
public void skipOutput(DataInput in) throws IOException {
outputs1.skipOutput(in);
outputs2.skipOutput(in);
}
@Override
public Pair<A,B> getNoOutput() {
return NO_OUTPUT;

View File

@ -39,7 +39,7 @@ final class ReverseBytesReader extends FST.BytesReader {
}
@Override
public void skipBytes(int count) {
public void skipBytes(long count) {
pos -= count;
}

View File

@ -123,6 +123,11 @@ public final class ListOfOutputs<T> extends Outputs<Object> {
return outputs.read(in);
}
@Override
public void skipOutput(DataInput in) throws IOException {
outputs.skipOutput(in);
}
@Override
public Object readFinalOutput(DataInput in) throws IOException {
int count = in.readVInt();
@ -137,6 +142,14 @@ public final class ListOfOutputs<T> extends Outputs<Object> {
}
}
@Override
public void skipFinalOutput(DataInput in) throws IOException {
int count = in.readVInt();
for(int i=0;i<count;i++) {
outputs.skipOutput(in);
}
}
@Override
public Object getNoOutput() {
return outputs.getNoOutput();