diff --git a/CHANGES.txt b/CHANGES.txt index fe0b9fb2e75..5fa26b3865a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,12 +7,6 @@ $Id$ 1. Added catch of BooleanQuery$TooManyClauses in QueryParser to throw ParseException instead. (Erik Hatcher) - 2. Modified StopFilter to increment positions to account for - stop words removed. This prevents exact phrase queries from - matching erroneously (use slop factor to account for missing - stop words). StopFilter is used by StopAnalyzer, StandardAnalyzer - and some others. (Erik Hatcher) - 1.3 RC3 1. Added minMergeDocs in IndexWriter. This can be raised to speed diff --git a/src/java/org/apache/lucene/analysis/StopFilter.java b/src/java/org/apache/lucene/analysis/StopFilter.java index 6cf767321ae..e23e686b4d9 100644 --- a/src/java/org/apache/lucene/analysis/StopFilter.java +++ b/src/java/org/apache/lucene/analysis/StopFilter.java @@ -57,12 +57,8 @@ package org.apache.lucene.analysis; import java.io.IOException; import java.util.Hashtable; -/** - * Removes stop words from a token stream. Position increments - * on tokens emitted are adjusted to account for words - * removed. Exact phrase queries will not match across holes left - * by stop word removal, but sloppy phrase queries may match. - */ +/** Removes stop words from a token stream. */ + public final class StopFilter extends TokenFilter { private Hashtable table; @@ -93,17 +89,10 @@ public final class StopFilter extends TokenFilter { /** Returns the next input Token whose termText() is not a stop word. */ public final Token next() throws IOException { - int position = 1; - // return the first non-stop word found - for (Token token = input.next(); token != null; token = input.next()) { - if (table.get(token.termText) == null) { - token.setPositionIncrement(position); + for (Token token = input.next(); token != null; token = input.next()) + if (table.get(token.termText) == null) return token; - } - - position++; - } // reached EOS -- return null return null; }