StopFilter ignore case: LUCENE-248

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@354253 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -207,6 +207,9 @@ New features
     successive position.
     (Erik Hatcher, with advice from Yonik)
 
+32. StopFilter can now ignore case when checking for stop words.
+    (Grant Ingersoll via Yonik, LUCENE-248)
+
 API Changes
 
  1. Several methods and fields have been deprecated. The API documentation

src/java/org/apache/lucene/analysis/StopFilter.java

@@ -27,15 +27,25 @@ import java.util.Set;
 
 public final class StopFilter extends TokenFilter {
 
-  private Set stopWords;
+  private final Set stopWords;
+  private final boolean ignoreCase;
+
+    /**
+     * Construct a token stream filtering the given input.
+     */
+    public StopFilter(TokenStream input, String [] stopWords)
+    {
+        this(input, stopWords, false);
+    }
 
   /**
    * Constructs a filter which removes words from the input
    * TokenStream that are named in the array of words.
    */
-  public StopFilter(TokenStream in, String[] stopWords) {
+  public StopFilter(TokenStream in, String[] stopWords, boolean ignoreCase) {
     super(in);
-    this.stopWords = makeStopSet(stopWords);
+    this.ignoreCase = ignoreCase;
+    this.stopWords = makeStopSet(stopWords, ignoreCase);
   }
 
   /**
@@ -43,11 +53,33 @@ public final class StopFilter extends TokenFilter {
    * TokenStream that are named in the Hashtable.
    *
    * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead
-   */
+   */ 
   public StopFilter(TokenStream in, Hashtable stopTable) {
-    super(in);
-    stopWords = new HashSet(stopTable.keySet());
+    this(in, stopTable, false);
   }
+  /**
+   * Constructs a filter which removes words from the input
+   * TokenStream that are named in the Hashtable.
+   * If ignoreCase is true, all keys in the stopTable should already
+   * be lowercased.
+   * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead
+   */
+  public StopFilter(TokenStream in, Hashtable stopTable, boolean ignoreCase) {
+    this(in, stopTable.keySet(), ignoreCase);
+  }
+
+    /**
+     * Construct a token stream filtering the given input.
+     * @param input
+     * @param stopWords The set of Stop Words, as Strings.  If ignoreCase is true, all strings should be lower cased
+     * @param ignoreCase -Ignore case when stopping.  The stopWords set must be setup to contain only lower case words 
+     */
+    public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase)
+    {
+        super(input);
+        this.ignoreCase = ignoreCase;
+        this.stopWords = stopWords;
+    }
 
   /**
    * Constructs a filter which removes words from the input
@@ -58,10 +90,8 @@ public final class StopFilter extends TokenFilter {
    * @see #makeStopSet(java.lang.String[])
    */
   public StopFilter(TokenStream in, Set stopWords) {
-    super(in);
-    this.stopWords = stopWords;
+    this(in, stopWords, false);
   }
-
   /**
    * Builds a Hashtable from an array of stop words,
    * appropriate for passing into the StopFilter constructor.
@@ -71,9 +101,23 @@ public final class StopFilter extends TokenFilter {
    * @deprecated Use {@link #makeStopSet(String[])} instead.
    */
   public static final Hashtable makeStopTable(String[] stopWords) {
-    Hashtable stopTable = new Hashtable(stopWords.length);
-    for (int i = 0; i < stopWords.length; i++)
-      stopTable.put(stopWords[i], stopWords[i]);
+      return makeStopTable(stopWords, false);
+  }
+    
+    /**
+    * Builds a Hashtable from an array of stop words,
+    * appropriate for passing into the StopFilter constructor.
+    * This permits this table construction to be cached once when
+    * an Analyzer is constructed.
+  * @deprecated Use {@link #makeStopSet(java.lang.String[], boolean)}  instead.
+ */ 
+  public static final Hashtable makeStopTable(String [] stopWords, boolean ignoreCase) {
+      Hashtable stopTable = new Hashtable(stopWords.length);                                
+      for (int i = 0; i < stopWords.length; i++)
+      {
+          String stopWord = ignoreCase ? stopWords[i].toLowerCase() : stopWords[i];
+          stopTable.put(stopWord, stopWord);
+      }
     return stopTable;
   }
 
@@ -82,13 +126,25 @@ public final class StopFilter extends TokenFilter {
    * appropriate for passing into the StopFilter constructor.
    * This permits this stopWords construction to be cached once when
    * an Analyzer is constructed.
+   * 
+   * @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
    */
   public static final Set makeStopSet(String[] stopWords) {
+    return makeStopSet(stopWords, false);
+  }
+    
+  /**
+   * 
+    * @param stopWords
+   * @param ignoreCase If true, all words are lower cased first.  
+   * @return a Set containing the words
+   */    
+  public static final Set makeStopSet(String[] stopWords, boolean ignoreCase) {
     HashSet stopTable = new HashSet(stopWords.length);
     for (int i = 0; i < stopWords.length; i++)
-      stopTable.add(stopWords[i]);
+      stopTable.add(ignoreCase ? stopWords[i].toLowerCase() : stopWords[i]);
     return stopTable;
-  }
+  }    
 
   /**
    * Returns the next input Token whose termText() is not a stop word.
@@ -96,8 +152,11 @@ public final class StopFilter extends TokenFilter {
   public final Token next() throws IOException {
     // return the first non-stop word found
     for (Token token = input.next(); token != null; token = input.next())
-      if (!stopWords.contains(token.termText))
-        return token;
+    {
+        String termText = ignoreCase ? token.termText.toLowerCase() : token.termText;
+        if (!stopWords.contains(termText))
+          return token;
+    }
     // reached EOS -- return null
     return null;
   }

src/test/org/apache/lucene/analysis/TestStopFilter.java

@@ -0,0 +1,50 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+
+import java.io.StringReader;
+import java.io.IOException;
+import java.util.Set;
+import java.util.HashSet;
+
+/**
+ * @author yonik
+ */
+public class TestStopFilter extends TestCase {
+
+  // other StopFilter functionality is already tested by TestStopAnalyzer
+
+  public void testExactCase() throws IOException {
+    StringReader reader = new StringReader("Now is The Time");
+    String[] stopWords = new String[] { "is", "the", "Time" };
+    TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords);
+    assertEquals("Now", stream.next().termText());
+    assertEquals("The", stream.next().termText());
+    assertEquals(null, stream.next());
+  }
+
+  public void testIgnoreCase() throws IOException {
+    StringReader reader = new StringReader("Now is The Time");
+    String[] stopWords = new String[] { "is", "the", "Time" };
+    TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords, true);
+    assertEquals("Now", stream.next().termText());
+    assertEquals(null,stream.next());
+  }
+
+}