LUCENE-3748: EnglishPossessiveFilter did not work with a proper right quotation mark

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1242740 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/CHANGES.txt

@@ -178,6 +178,9 @@ Changes in runtime behavior
    the Snowball German2 algorithm to ae/oe/ue and case-folds ß. Add 
    GalicianMinimalStemFilter for plural removal only. (Robert Muir)
 
+ * LUCENE-3748: EnglishPossessiveFilter did not work with Unicode right 
+   single quotation mark (U+2019).  (David Croley via Robert Muir)
+
 Optimizations
 
 * SOLR-2888: FSTSuggester refactoring: internal storage is now UTF-8, 

modules/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java

@@ -105,7 +105,7 @@ public final class EnglishAnalyzer extends StopwordAnalyzerBase {
     TokenStream result = new StandardFilter(matchVersion, source);
     // prior to this we get the classic behavior, standardfilter does it for us.
     if (matchVersion.onOrAfter(Version.LUCENE_31))
-      result = new EnglishPossessiveFilter(result);
+      result = new EnglishPossessiveFilter(matchVersion, result);
     result = new LowerCaseFilter(matchVersion, result);
     result = new StopFilter(matchVersion, result, stopwords);
     if(!stemExclusionSet.isEmpty())

modules/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishPossessiveFilter.java

@@ -22,15 +22,34 @@ import java.io.IOException;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
 
 /**
  * TokenFilter that removes possessives (trailing 's) from words.
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating EnglishPossessiveFilter:
+ * <ul>
+ *    <li> As of 3.6, U+2019 RIGHT SINGLE QUOTATION MARK and 
+ *         U+FF07 FULLWIDTH APOSTROPHE are also treated as
+ *         quotation marks.
+ * </ul>
  */
 public final class EnglishPossessiveFilter extends TokenFilter {
   private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private Version matchVersion;
 
+  /**
+   * @deprecated Use {@link #EnglishPossessiveFilter(Version, TokenStream)} instead.
+   */
+  @Deprecated
   public EnglishPossessiveFilter(TokenStream input) {
+    this(Version.LUCENE_35, input);
+  }
+
+  public EnglishPossessiveFilter(Version version, TokenStream input) {
     super(input);
+    this.matchVersion = version;
   }
 
   @Override
@@ -43,9 +62,11 @@ public final class EnglishPossessiveFilter extends TokenFilter {
     final int bufferLength = termAtt.length();
     
     if (bufferLength >= 2 && 
-        buffer[bufferLength-2] == '\'' &&
-        (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S'))
+        (buffer[bufferLength-2] == '\'' || 
+         (matchVersion.onOrAfter(Version.LUCENE_36) && (buffer[bufferLength-2] == '\u2019' || buffer[bufferLength-2] == '\uFF07'))) &&
+        (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
       termAtt.setLength(bufferLength - 2); // Strip last 2 characters off
+    }
 
     return true;
   }

modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java

@@ -40,6 +40,8 @@ public class TestEnglishAnalyzer extends BaseTokenStreamTestCase {
     assertAnalyzesTo(a, "the", new String[] {});
     // possessive removal
     checkOneTermReuse(a, "steven's", "steven");
+    checkOneTermReuse(a, "steven\u2019s", "steven");
+    checkOneTermReuse(a, "steven\uFF07s", "steven");
   }
   
   /** test use of exclusion set */

solr/core/src/java/org/apache/solr/analysis/EnglishPossessiveFilterFactory.java

@@ -17,6 +17,8 @@ package org.apache.solr.analysis;
  * limitations under the License.
  */
 
+import java.util.Map;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
 
@@ -33,7 +35,14 @@ import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
  *
  */
 public class EnglishPossessiveFilterFactory extends BaseTokenFilterFactory {
+  
+  @Override
+  public void init(Map<String,String> args) {
+    super.init(args);
+    assureMatchVersion();
+  }
+  
   public TokenStream create(TokenStream input) {
-    return new EnglishPossessiveFilter(input);
+    return new EnglishPossessiveFilter(luceneMatchVersion, input);
   }
 }