Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8719: Traverse all paths at the end of a TokenStream in FixedShingleFilter

This commit is contained in:
Alan Woodward 2019-03-14 11:10:02 +00:00
parent 84785e8e45
commit 9096b846af
3 changed files with 69 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -28,6 +28,9 @@ Bug fixes
* LUCENE-8726: ValueSource.asDoubleValuesSource() could leak a reference to
IndexSearcher (Alan Woodward, Yury Pakhomov)
* LUCENE-8719: FixedShingleFilter can miss shingles at the end of a token stream if
there are multiple paths with different lengths. (Alan Woodward)
Improvements
* LUCENE-8673: Use radix partitioning when merging dimensional points instead

94
lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
View File

@ -92,63 +92,67 @@ public final class FixedShingleFilter extends GraphTokenFilter {
@Override
public boolean incrementToken() throws IOException {
int shinglePosInc;
if (incrementGraph() == false) {
if (incrementBaseToken() == false) {
return false;
}
// starting a shingle at a new base position, use base position increment
shinglePosInc = incAtt.getPositionIncrement();
}
else {
// starting a new shingle at the same base with a different graph, use a 0
// position increment
shinglePosInc = 0;
}
int shinglePosInc, startOffset, endOffset;
final int startOffset = offsetAtt.startOffset();
int endOffset = offsetAtt.endOffset();
this.buffer.setEmpty();
this.buffer.append(termAtt);
// build the shingle by iterating over the current graph, adding
// filler tokens if we encounter gaps
for (int i = 1; i < shingleSize; i++) {
if (incrementGraphToken() == false) {
// we've reached the end of the token stream, check for trailing
// positions and add fillers if necessary
int trailingPositions = getTrailingPositions();
if (i + trailingPositions < shingleSize) {
// not enough trailing positions to make a full shingle
outer: while (true) {
if (incrementGraph() == false) {
if (incrementBaseToken() == false) {
return false;
}
while (i < shingleSize) {
this.buffer.append(tokenSeparator).append(fillerToken);
i++;
}
break;
// starting a shingle at a new base position, use base position increment
shinglePosInc = incAtt.getPositionIncrement();
} else {
// starting a new shingle at the same base with a different graph, use a 0
// position increment
shinglePosInc = 0;
}
int posInc = incAtt.getPositionIncrement();
if (posInc > 1) {
// if we have a posInc > 1, we need to fill in the gaps
if (i + posInc > shingleSize) {
// if the posInc is greater than the shingle size, we need to add fillers
// up to the shingle size but no further
startOffset = offsetAtt.startOffset();
endOffset = offsetAtt.endOffset();
this.buffer.setEmpty();
this.buffer.append(termAtt);
// build the shingle by iterating over the current graph, adding
// filler tokens if we encounter gaps
for (int i = 1; i < shingleSize; i++) {
if (incrementGraphToken() == false) {
// we've reached the end of the token stream, check for trailing
// positions and add fillers if necessary
int trailingPositions = getTrailingPositions();
if (i + trailingPositions < shingleSize) {
// not enough trailing positions to make a full shingle
// start again at a different graph
continue outer;
}
while (i < shingleSize) {
this.buffer.append(tokenSeparator).append(fillerToken);
i++;
}
break;
}
// otherwise just add them in as far as we need
while (posInc > 1) {
this.buffer.append(tokenSeparator).append(fillerToken);
posInc--;
i++;
int posInc = incAtt.getPositionIncrement();
if (posInc > 1) {
// if we have a posInc > 1, we need to fill in the gaps
if (i + posInc > shingleSize) {
// if the posInc is greater than the shingle size, we need to add fillers
// up to the shingle size but no further
while (i < shingleSize) {
this.buffer.append(tokenSeparator).append(fillerToken);
i++;
}
break;
}
// otherwise just add them in as far as we need
while (posInc > 1) {
this.buffer.append(tokenSeparator).append(fillerToken);
posInc--;
i++;
}
}
this.buffer.append(tokenSeparator).append(termAtt);
endOffset = offsetAtt.endOffset();
}
this.buffer.append(tokenSeparator).append(termAtt);
endOffset = offsetAtt.endOffset();
break;
}
clearAttributes();
this.offsetAtt.setOffset(startOffset, endOffset);

17
lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/FixedShingleFilterTest.java
View File

@ -199,6 +199,23 @@ public class FixedShingleFilterTest extends BaseTokenStreamTestCase {
new int[] { 1, 0, 0, 0, 1, 0, });
}
public void testTrailingGraphsOfDifferingLengths() throws IOException {
// a b:3/c d e f
TokenStream ts = new CannedTokenStream(
new Token("a", 0, 1),
new Token("b", 1, 2, 3, 3),
new Token("c", 0, 2, 3),
new Token("d", 2, 3),
new Token("e", 2, 3),
new Token("f", 4, 5)
);
assertTokenStreamContents(new FixedShingleFilter(ts, 3),
new String[]{ "a b f", "a c d", "c d e", "d e f"});
}
public void testParameterLimits() {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
new FixedShingleFilter(new CannedTokenStream(), 1);