LUCENE-9749: Hunspell: apply output conversion (OCONV) to the suggestions (#2329)

lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java

@@ -22,12 +22,14 @@ import static org.apache.lucene.analysis.hunspell.WordContext.COMPOUND_END;
 import static org.apache.lucene.analysis.hunspell.WordContext.COMPOUND_MIDDLE;
 import static org.apache.lucene.analysis.hunspell.WordContext.SIMPLE_WORD;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.stream.Collectors;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.IntsRef;
 
@@ -438,7 +440,7 @@ public class SpellChecker {
         result.add(candidate);
       }
     }
-    return new ArrayList<>(result);
+    return result.stream().map(this::cleanOutput).collect(Collectors.toList());
   }
 
   private String adjustSuggestionCase(String candidate, WordCase original) {
@@ -480,4 +482,16 @@ public class SpellChecker {
     }
     return result;
   }
+
+  private String cleanOutput(String s) {
+    if (!dictionary.needsOutputCleaning) return s;
+
+    try {
+      StringBuilder sb = new StringBuilder(s);
+      Dictionary.applyMappings(dictionary.oconv, sb);
+      return sb.toString();
+    } catch (IOException bogus) {
+      throw new RuntimeException(bogus);
+    }
+  }
 }

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java

@@ -172,6 +172,10 @@ public class SpellCheckerTest extends StemmerTestBase {
     doTest("germancompounding");
   }
 
+  public void testApplyOconvToSuggestions() throws Exception {
+    doTest("oconv");
+  }
+
   public void testModifyingSuggestions() throws Exception {
     doTest("sug");
   }

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/oconv.aff

@@ -0,0 +1,20 @@
+# output conversion 
+SET UTF-8
+
+# Testing also whitespace and comments.
+OCONV 7 # space, space
+OCONV	a A # tab, space, space
+OCONV	á	Á # tab, tab, space
+OCONV	b	B	# tab, tab, tab
+OCONV  c  C		# 2xspace, 2xspace, 2xtab
+OCONV	 d 	D # tab+space, space+tab, space
+OCONV e E #
+OCONV é É 	
+# Only comment. Note that line above ends with space+tab.
+
+ # space
+  # 2xspace
+	# tab
+		# 2xtab
+ 	# space+tab
+	 # tab+space

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/oconv.dic

@@ -0,0 +1,4 @@
+3
+bébé
+dádá
+aábcdeé

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/oconv.good

@@ -0,0 +1,2 @@
+bébé
+dádá

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/oconv.sug

@@ -0,0 +1,3 @@
+BÉBÉ
+DÁDÁ
+AÁBCDEÉ

lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/oconv.wrong

@@ -0,0 +1,3 @@
+béb
+dád
+aábcde