LUCENE-8891: Add snowball stemmer and analyzer for Estonian language.

Signed-off-by: Tomoko Uchida <tomoko@apache.org>

lucene/CHANGES.txt

@@ -52,6 +52,9 @@ API Changes
 
 New Features
 
+* LUCENE-8891: Snowball stemmer/analyzer for the Estonian language.
+ (Gert Morten Paimla via Tomoko Uchida)
+
 * LUCENE-8815: Provide a DoubleValues implementation for retrieving the value of features without
   requiring a separate numeric field. Note that as feature values are stored with only 8 bits of 
   mantissa the values returned may have a delta from the original values indexed.

lucene/analysis/common/src/java/org/apache/lucene/analysis/et/EstonianAnalyzer.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.et;
+
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.snowball.SnowballFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.tartarus.snowball.ext.EstonianStemmer;
+
+/**
+ * {@link Analyzer} for Estonian.
+ */
+public final class EstonianAnalyzer extends StopwordAnalyzerBase {
+    private final CharArraySet stemExclusionSet;
+
+    /** File containing default Estonian stopwords. */
+    public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
+    /**
+     * Returns an unmodifiable instance of the default stop words set.
+     * @return default stop words set.
+     */
+    public static CharArraySet getDefaultStopSet(){
+        return DefaultSetHolder.DEFAULT_STOP_SET;
+    }
+
+    /**
+     * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
+     * accesses the static final set the first time.;
+     */
+    private static class DefaultSetHolder {
+        static final CharArraySet DEFAULT_STOP_SET;
+
+        static {
+            try {
+                DEFAULT_STOP_SET = loadStopwordSet(false,
+                        EstonianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
+            } catch (IOException ex) {
+                // default set should always be present as it is part of the
+                // distribution (JAR)
+                throw new RuntimeException("Unable to load default stopword set");
+            }
+        }
+    }
+
+    /**
+     * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
+     */
+    public EstonianAnalyzer() {
+        this(DefaultSetHolder.DEFAULT_STOP_SET);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words.
+     *
+     * @param stopwords a stopword set
+     */
+    public EstonianAnalyzer(CharArraySet stopwords) {
+        this(stopwords, CharArraySet.EMPTY_SET);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is
+     * provided this analyzer will add a {@link SetKeywordMarkerFilter} before
+     * stemming.
+     *
+     * @param stopwords a stopword set
+     * @param stemExclusionSet a set of terms not to be stemmed
+     */
+    public EstonianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
+        super(stopwords);
+        this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
+    }
+
+    /**
+     * Creates a
+     * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
+     * which tokenizes all the text in the provided {@link Reader}.
+     *
+     * @return A
+     *         {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
+     *         built from an {@link StandardTokenizer} filtered with
+     *         {@link LowerCaseFilter}, {@link StopFilter}
+     *         , {@link SetKeywordMarkerFilter} if a stem exclusion set is
+     *         provided and {@link SnowballFilter}.
+     */
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+        final Tokenizer source = new StandardTokenizer();
+        TokenStream result = new LowerCaseFilter(source);
+        result = new StopFilter(result, stopwords);
+        if(!stemExclusionSet.isEmpty())
+            result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+        result = new SnowballFilter(result, new EstonianStemmer());
+        return new TokenStreamComponents(source, result);
+    }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+        return new LowerCaseFilter(in);
+    }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/et/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Analyzer for Estonian.
+ */
+package org.apache.lucene.analysis.et;

lucene/analysis/common/src/test/org/apache/lucene/analysis/et/TestEstonianAnalyzer.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.et;
+
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+
+public class TestEstonianAnalyzer extends BaseTokenStreamTestCase {
+
+    /** This test fails with NPE when the
+     * stopwords file is missing in classpath */
+    public void testResourcesAvailable() {
+        new EstonianAnalyzer().close();
+    }
+
+    /** test stopwords and stemming */
+    public void testBasics() throws IOException {
+        Analyzer a = new EstonianAnalyzer();
+        // stemming
+        checkOneTerm(a, "teadaolevalt", "teadaole");
+        checkOneTerm(a, "teadaolevaid", "teadaole");
+        checkOneTerm(a, "teadaolevatest", "teadaole");
+        checkOneTerm(a, "teadaolevail", "teadaole");
+        checkOneTerm(a, "teadaolevatele", "teadaole");
+        checkOneTerm(a, "teadaolevatel", "teadaole");
+        checkOneTerm(a, "teadaolevateks", "teadaole");
+        checkOneTerm(a, "teadaolevate", "teadaole");
+        checkOneTerm(a, "teadaolevaks", "teadaole");
+        checkOneTerm(a, "teadaoleval", "teadaole");
+        checkOneTerm(a, "teadaolevates", "teadaole");
+        checkOneTerm(a, "teadaolevat", "teadaole");
+        checkOneTerm(a, "teadaolevast", "teadaole");
+        checkOneTerm(a, "teadaoleva", "teadaole");
+        checkOneTerm(a, "teadaolevais", "teadaole");
+        checkOneTerm(a, "teadaolevas", "teadaole");
+        checkOneTerm(a, "teadaolevad", "teadaole");
+        checkOneTerm(a, "teadaolevale", "teadaole");
+        checkOneTerm(a, "teadaolevatesse", "teadaole");
+        // stopword
+        assertAnalyzesTo(a, "alla", new String[] { });
+        a.close();
+    }
+
+
+}