mirror of https://github.com/apache/lucene.git
LUCENE-6742: Lovins & Finnish implementation of SnowballFilter was fixed to behave exactly as specified. A bug in the snowball compiler caused differences in output of the filter in comparison to the original test data. In addition, the performance of those filters was improved significantly
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1696147 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler
parent
0a54365ffa
commit
e5455614da
|
|
@ -73,6 +73,12 @@ Bug Fixes
|
|||
* LUCENE-6730: Hyper-parameter c is ignored in term frequency NormalizationH1.
|
||||
(Ahmet Arslan via Robert Muir)
|
||||
|
||||
* LUCENE-6742: Lovins & Finnish implementation of SnowballFilter was
|
||||
fixed to behave exactly as specified. A bug in the snowball compiler
|
||||
caused differences in output of the filter in comparison to the original
|
||||
test data. In addition, the performance of those filters was improved
|
||||
significantly. (Uwe Schindler, Robert Muir)
|
||||
|
||||
Other
|
||||
|
||||
* LUCENE-6174: Improve "ant eclipse" to select right JRE for building.
|
||||
|
|
@ -90,6 +96,13 @@ Build
|
|||
detect javadoc-style license headers. Use Groovy to implement the
|
||||
checks instead of plain Ant. (Uwe Schindler)
|
||||
|
||||
Changes in Backwards Compatibility Policy
|
||||
|
||||
* LUCENE-6742: The Lovins & Finnish implementation of SnowballFilter
|
||||
were fixed to now behave exactly like the original Snowball stemmer.
|
||||
If you have indexed text using those stemmers you may need to reindex.
|
||||
(Uwe Schindler, Robert Muir)
|
||||
|
||||
======================= Lucene 5.3.0 =======================
|
||||
|
||||
New Features
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ Lucene Analyzers README file
|
|||
|
||||
This project provides pre-compiled version of the Snowball stemmers
|
||||
based on revision 502 of the Tartarus Snowball repository,
|
||||
now located at https://github.com/snowballstem/snowball/tree/e103b5c257383ee94a96e7fc58cab3c567bf079b (GitHub),
|
||||
together with classes integrating them with the Lucene search engine.
|
||||
|
||||
A few changes has been made to the static Snowball code and compiled stemmers:
|
||||
|
|
@ -9,7 +10,11 @@ A few changes has been made to the static Snowball code and compiled stemmers:
|
|||
* Class SnowballProgram is made abstract and contains new abstract method stem() to avoid reflection in Lucene filter class SnowballFilter.
|
||||
* All use of StringBuffers has been refactored to StringBuilder for speed.
|
||||
* Snowball BSD license header has been added to the Java classes to avoid having RAT adding new ASL headers.
|
||||
* Uses Java 7 MethodHandles and fixes method visibility bug: http://article.gmane.org/gmane.comp.search.snowball/1139
|
||||
|
||||
If you want to add new stemmers, use the exact revision / Git commit above to generate the Java class, place it
|
||||
in src/java/org/tartarus/snowball/ext, and finally execute "ant patch-snowball". The latter will change the APIs
|
||||
of the generated class to make it compatible. Already patched classes are not modified.
|
||||
|
||||
IMPORTANT NOTICE ON BACKWARDS COMPATIBILITY!
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@
|
|||
|
||||
<import file="../analysis-module-build.xml"/>
|
||||
|
||||
<property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>
|
||||
|
||||
<target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
|
||||
-jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
|
||||
|
||||
|
|
@ -140,4 +142,15 @@
|
|||
<target name="javadocs" depends="module-build.javadocs"/>
|
||||
|
||||
<target name="regenerate" depends="jflex"/>
|
||||
|
||||
<target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
|
||||
<fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
|
||||
<replaceregexp match="^public class \w+Stemmer\b" replace="@SuppressWarnings("unused") \0" flags="m" encoding="UTF-8">
|
||||
<fileset refid="snowball.programs"/>
|
||||
</replaceregexp>
|
||||
<replaceregexp match="private final static \w+Stemmer methodObject\b.*$" replace="/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();" flags="m" encoding="UTF-8">
|
||||
<fileset refid="snowball.programs"/>
|
||||
</replaceregexp>
|
||||
<fixcrlf srcdir="${snowball.programs.dir}" includes="*Stemmer.java" tab="remove" tablength="2" encoding="UTF-8" javafiles="yes" fixlast="yes"/>
|
||||
</target>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -31,41 +31,53 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||
|
||||
package org.tartarus.snowball;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.invoke.MethodHandle;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.lang.invoke.MethodType;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* This is the rev 502 of the Snowball SVN trunk,
|
||||
* now located at <a target="_blank" href="https://github.com/snowballstem/snowball/tree/e103b5c257383ee94a96e7fc58cab3c567bf079b">GitHub</a>,
|
||||
* but modified:
|
||||
* made abstract and introduced abstract method stem to avoid expensive reflection in filter class.
|
||||
* refactored StringBuffers to StringBuilder
|
||||
* uses char[] as buffer instead of StringBuffer/StringBuilder
|
||||
* eq_s,eq_s_b,insert,replace_s take CharSequence like eq_v and eq_v_b
|
||||
* <ul>
|
||||
* <li>made abstract and introduced abstract method stem to avoid expensive reflection in filter class.
|
||||
* <li>refactored StringBuffers to StringBuilder
|
||||
* <li>uses char[] as buffer instead of StringBuffer/StringBuilder
|
||||
* <li>eq_s,eq_s_b,insert,replace_s take CharSequence like eq_v and eq_v_b
|
||||
* <li>use MethodHandles and fix <a target="_blank" href="http://article.gmane.org/gmane.comp.search.snowball/1139">method visibility bug</a>.
|
||||
* </ul>
|
||||
*/
|
||||
public class Among {
|
||||
|
||||
public final class Among {
|
||||
|
||||
public Among(String s, int substring_i, int result,
|
||||
String methodname, SnowballProgram methodobject) {
|
||||
String methodname, MethodHandles.Lookup methodobject) {
|
||||
this.s_size = s.length();
|
||||
this.s = s.toCharArray();
|
||||
this.substring_i = substring_i;
|
||||
this.result = result;
|
||||
this.methodobject = methodobject;
|
||||
if (methodname.length() == 0) {
|
||||
if (methodname.isEmpty()) {
|
||||
this.method = null;
|
||||
} else {
|
||||
final Class<? extends SnowballProgram> clazz = methodobject.lookupClass().asSubclass(SnowballProgram.class);
|
||||
try {
|
||||
this.method = methodobject.getClass().getDeclaredMethod(methodname);
|
||||
} catch (NoSuchMethodException e) {
|
||||
throw new RuntimeException(e);
|
||||
this.method = methodobject.findVirtual(clazz, methodname, MethodType.methodType(boolean.class))
|
||||
.asType(MethodType.methodType(boolean.class, SnowballProgram.class));
|
||||
} catch (NoSuchMethodException | IllegalAccessException e) {
|
||||
throw new RuntimeException(String.format(Locale.ENGLISH,
|
||||
"Snowball program '%s' is broken, cannot access method: boolean %s()",
|
||||
clazz.getSimpleName(), methodname
|
||||
), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public final int s_size; /* search string */
|
||||
public final char[] s; /* search string */
|
||||
public final int substring_i; /* index to longest matching substring */
|
||||
public final int result; /* result of the lookup */
|
||||
public final Method method; /* method to use if substring matches */
|
||||
public final SnowballProgram methodobject; /* object to invoke method on */
|
||||
|
||||
};
|
||||
final int s_size; /* search string */
|
||||
final char[] s; /* search string */
|
||||
final int substring_i; /* index to longest matching substring */
|
||||
final int result; /* result of the lookup */
|
||||
|
||||
// Make sure this is not accessible outside package for Java security reasons!
|
||||
final MethodHandle method; /* method to use if substring matches */
|
||||
|
||||
}
|
||||
|
|
@ -32,18 +32,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||
|
||||
package org.tartarus.snowball;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
|
||||
/**
|
||||
* This is the rev 502 of the Snowball SVN trunk,
|
||||
* now located at <a target="_blank" href="https://github.com/snowballstem/snowball/tree/e103b5c257383ee94a96e7fc58cab3c567bf079b">GitHub</a>,
|
||||
* but modified:
|
||||
* made abstract and introduced abstract method stem to avoid expensive reflection in filter class.
|
||||
* refactored StringBuffers to StringBuilder
|
||||
* uses char[] as buffer instead of StringBuffer/StringBuilder
|
||||
* eq_s,eq_s_b,insert,replace_s take CharSequence like eq_v and eq_v_b
|
||||
* <ul>
|
||||
* <li>made abstract and introduced abstract method stem to avoid expensive reflection in filter class.
|
||||
* <li>refactored StringBuffers to StringBuilder
|
||||
* <li>uses char[] as buffer instead of StringBuffer/StringBuilder
|
||||
* <li>eq_s,eq_s_b,insert,replace_s take CharSequence like eq_v and eq_v_b
|
||||
* <li>use MethodHandles and fix <a target="_blank" href="http://article.gmane.org/gmane.comp.search.snowball/1139">method visibility bug</a>.
|
||||
* </ul>
|
||||
*/
|
||||
public abstract class SnowballProgram {
|
||||
|
||||
|
|
@ -310,16 +312,11 @@ public abstract class SnowballProgram {
|
|||
if (common_i >= w.s_size) {
|
||||
cursor = c + w.s_size;
|
||||
if (w.method == null) return w.result;
|
||||
boolean res;
|
||||
boolean res = false;
|
||||
try {
|
||||
Object resobj = w.method.invoke(w.methodobject);
|
||||
res = resobj.toString().equals("true");
|
||||
} catch (InvocationTargetException e) {
|
||||
res = false;
|
||||
// FIXME - debug message
|
||||
} catch (IllegalAccessException e) {
|
||||
res = false;
|
||||
// FIXME - debug message
|
||||
res = (boolean) w.method.invokeExact(this);
|
||||
} catch (Throwable e) {
|
||||
rethrow(e);
|
||||
}
|
||||
cursor = c + w.s_size;
|
||||
if (res) return w.result;
|
||||
|
|
@ -378,16 +375,11 @@ public abstract class SnowballProgram {
|
|||
cursor = c - w.s_size;
|
||||
if (w.method == null) return w.result;
|
||||
|
||||
boolean res;
|
||||
boolean res = false;
|
||||
try {
|
||||
Object resobj = w.method.invoke(w.methodobject);
|
||||
res = resobj.toString().equals("true");
|
||||
} catch (InvocationTargetException e) {
|
||||
res = false;
|
||||
// FIXME - debug message
|
||||
} catch (IllegalAccessException e) {
|
||||
res = false;
|
||||
// FIXME - debug message
|
||||
res = (boolean) w.method.invokeExact(this);
|
||||
} catch (Throwable e) {
|
||||
rethrow(e);
|
||||
}
|
||||
cursor = c - w.s_size;
|
||||
if (res) return w.result;
|
||||
|
|
@ -496,5 +488,14 @@ extern void debug(struct SN_env * z, int number, int line_count)
|
|||
}
|
||||
*/
|
||||
|
||||
// Hack to rethrow unknown Exceptions from {@link MethodHandle#invoke}:
|
||||
private static void rethrow(Throwable t) {
|
||||
SnowballProgram.<Error>rethrow0(t);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T extends Throwable> void rethrow0(Throwable t) throws T {
|
||||
throw (T) t;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class ArmenianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class ArmenianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static ArmenianStemmer methodObject = new ArmenianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "\u0580\u0578\u0580\u0564", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class BasqueStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class BasqueStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static BasqueStemmer methodObject = new BasqueStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "idea", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class CatalanStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class CatalanStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static CatalanStemmer methodObject = new CatalanStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 13, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class DanishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class DanishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static DanishStemmer methodObject = new DanishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "hed", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class DutchStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class DutchStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static DutchStemmer methodObject = new DutchStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 6, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class EnglishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class EnglishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static EnglishStemmer methodObject = new EnglishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "arsen", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class FinnishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class FinnishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static FinnishStemmer methodObject = new FinnishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "pa", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class FrenchStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class FrenchStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static FrenchStemmer methodObject = new FrenchStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "col", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class German2Stemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class German2Stemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static German2Stemmer methodObject = new German2Stemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 6, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class GermanStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class GermanStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static GermanStemmer methodObject = new GermanStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 6, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class HungarianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class HungarianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static HungarianStemmer methodObject = new HungarianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "cs", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class IrishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class IrishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static IrishStemmer methodObject = new IrishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "b'", -1, 4, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class ItalianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class ItalianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static ItalianStemmer methodObject = new ItalianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 7, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class KpStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class KpStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static KpStemmer methodObject = new KpStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "nde", -1, 7, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -9,11 +9,11 @@ import org.tartarus.snowball.Among;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class LithuanianStemmer extends org.tartarus.snowball.SnowballProgram {
|
||||
@SuppressWarnings("unused") public class LithuanianStemmer extends org.tartarus.snowball.SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static LithuanianStemmer methodObject = new LithuanianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "a", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class LovinsStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class LovinsStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static LovinsStemmer methodObject = new LovinsStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "d", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class NorwegianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class NorwegianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static NorwegianStemmer methodObject = new NorwegianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "a", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class PorterStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class PorterStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static PorterStemmer methodObject = new PorterStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "s", -1, 3, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class PortugueseStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class PortugueseStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static PortugueseStemmer methodObject = new PortugueseStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 3, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class RomanianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class RomanianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static RomanianStemmer methodObject = new RomanianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 3, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class RussianStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class RussianStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static RussianStemmer methodObject = new RussianStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "\u0432", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class SpanishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class SpanishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static SpanishStemmer methodObject = new SpanishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "", -1, 6, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class SwedishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class SwedishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static SwedishStemmer methodObject = new SwedishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "a", -1, 1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import org.tartarus.snowball.SnowballProgram;
|
|||
* It implements the stemming algorithm defined by a snowball script.
|
||||
*/
|
||||
|
||||
public class TurkishStemmer extends SnowballProgram {
|
||||
@SuppressWarnings("unused") public class TurkishStemmer extends SnowballProgram {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final static TurkishStemmer methodObject = new TurkishStemmer ();
|
||||
/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
|
||||
|
||||
private final static Among a_0[] = {
|
||||
new Among ( "m", -1, -1, "", methodObject ),
|
||||
|
|
|
|||
|
|
@ -39,18 +39,14 @@ public class TestSnowballVocab extends LuceneTestCase {
|
|||
assertCorrectOutput("Danish", "danish");
|
||||
assertCorrectOutput("Dutch", "dutch");
|
||||
assertCorrectOutput("English", "english");
|
||||
// disabled due to snowball java code generation bug:
|
||||
// see http://article.gmane.org/gmane.comp.search.snowball/1139
|
||||
// assertCorrectOutput("Finnish", "finnish");
|
||||
assertCorrectOutput("Finnish", "finnish");
|
||||
assertCorrectOutput("French", "french");
|
||||
assertCorrectOutput("German", "german");
|
||||
assertCorrectOutput("German2", "german2");
|
||||
assertCorrectOutput("Hungarian", "hungarian");
|
||||
assertCorrectOutput("Italian", "italian");
|
||||
assertCorrectOutput("Kp", "kraaij_pohlmann");
|
||||
// disabled due to snowball java code generation bug:
|
||||
// see http://article.gmane.org/gmane.comp.search.snowball/1139
|
||||
// assertCorrectOutput("Lovins", "lovins");
|
||||
assertCorrectOutput("Lovins", "lovins");
|
||||
assertCorrectOutput("Norwegian", "norwegian");
|
||||
assertCorrectOutput("Porter", "porter");
|
||||
assertCorrectOutput("Portuguese", "portuguese");
|
||||
|
|
|
|||
Loading…
Reference in New Issue