Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-19 08:27:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8719: Traverse all paths at the end of a TokenStream in FixedShingleFilter

Browse Source
This commit is contained in:
Alan Woodward 2019-03-14 11:10:02 +00:00
parent 003edafdc8
commit 6571d06c25
3 changed files with 69 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -16,6 +16,9 @@ Bug fixes
* LUCENE-8726: ValueSource.asDoubleValuesSource() could leak a reference to * LUCENE-8726: ValueSource.asDoubleValuesSource() could leak a reference to
IndexSearcher (Alan Woodward, Yury Pakhomov) IndexSearcher (Alan Woodward, Yury Pakhomov)
* LUCENE-8719: FixedShingleFilter can miss shingles at the end of a token stream if
there are multiple paths with different lengths. (Alan Woodward)
Improvements Improvements
* LUCENE-8673: Use radix partitioning when merging dimensional points instead * LUCENE-8673: Use radix partitioning when merging dimensional points instead

16
lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
View File

@ -92,22 +92,23 @@ public final class FixedShingleFilter extends GraphTokenFilter {
@Override @Override
public boolean incrementToken() throws IOException { public boolean incrementToken() throws IOException {
int shinglePosInc; int shinglePosInc, startOffset, endOffset;
outer: while (true) {
if (incrementGraph() == false) { if (incrementGraph() == false) {
if (incrementBaseToken() == false) { if (incrementBaseToken() == false) {
return false; return false;
} }
// starting a shingle at a new base position, use base position increment // starting a shingle at a new base position, use base position increment
shinglePosInc = incAtt.getPositionIncrement(); shinglePosInc = incAtt.getPositionIncrement();
} } else {
else {
// starting a new shingle at the same base with a different graph, use a 0 // starting a new shingle at the same base with a different graph, use a 0
// position increment // position increment
shinglePosInc = 0; shinglePosInc = 0;
} }
final int startOffset = offsetAtt.startOffset(); startOffset = offsetAtt.startOffset();
int endOffset = offsetAtt.endOffset(); endOffset = offsetAtt.endOffset();
this.buffer.setEmpty(); this.buffer.setEmpty();
this.buffer.append(termAtt); this.buffer.append(termAtt);
@ -120,7 +121,8 @@ public final class FixedShingleFilter extends GraphTokenFilter {
int trailingPositions = getTrailingPositions(); int trailingPositions = getTrailingPositions();
if (i + trailingPositions < shingleSize) { if (i + trailingPositions < shingleSize) {
// not enough trailing positions to make a full shingle // not enough trailing positions to make a full shingle
return false; // start again at a different graph
continue outer;
} }
while (i < shingleSize) { while (i < shingleSize) {
this.buffer.append(tokenSeparator).append(fillerToken); this.buffer.append(tokenSeparator).append(fillerToken);
@ -150,6 +152,8 @@ public final class FixedShingleFilter extends GraphTokenFilter {
this.buffer.append(tokenSeparator).append(termAtt); this.buffer.append(tokenSeparator).append(termAtt);
endOffset = offsetAtt.endOffset(); endOffset = offsetAtt.endOffset();
} }
break;
}
clearAttributes(); clearAttributes();
this.offsetAtt.setOffset(startOffset, endOffset); this.offsetAtt.setOffset(startOffset, endOffset);
this.incAtt.setPositionIncrement(shinglePosInc); this.incAtt.setPositionIncrement(shinglePosInc);

17
lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/FixedShingleFilterTest.java
View File

@ -199,6 +199,23 @@ public class FixedShingleFilterTest extends BaseTokenStreamTestCase {
new int[] { 1, 0, 0, 0, 1, 0, }); new int[] { 1, 0, 0, 0, 1, 0, });
} }
public void testTrailingGraphsOfDifferingLengths() throws IOException {
// a b:3/c d e f
TokenStream ts = new CannedTokenStream(
new Token("a", 0, 1),
new Token("b", 1, 2, 3, 3),
new Token("c", 0, 2, 3),
new Token("d", 2, 3),
new Token("e", 2, 3),
new Token("f", 4, 5)
);
assertTokenStreamContents(new FixedShingleFilter(ts, 3),
new String[]{ "a b f", "a c d", "c d e", "d e f"});
}
public void testParameterLimits() { public void testParameterLimits() {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
new FixedShingleFilter(new CannedTokenStream(), 1); new FixedShingleFilter(new CannedTokenStream(), 1);