From 048f8b5701aea87e3a086d0315ffe9006412e615 Mon Sep 17 00:00:00 2001 From: Dariusz Gertych Date: Tue, 19 Jun 2012 14:46:20 +0200 Subject: [PATCH] Support for polish_stem token filter Added missing token filter support for polish analyzer. Now it's possible to combine polish_stem token filter with the other token filters available in ES to create custom analyzer. Closes #3. --- README.md | 2 +- .../pl/PolishAnalysisBinderProcessor.java | 4 ++ .../pl/PolishStemTokenFilterFactory.java | 56 +++++++++++++++++++ .../analysis/pl/PolishIndicesAnalysis.java | 25 +++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java diff --git a/README.md b/README.md index 13ad8b5018d..1f690ec336c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | 1.0.0 | 0.18 | ---------------------------------------------------- -The plugin includes the `polish` analyzer. +The plugin includes the `polish` analyzer and `polish_stem` token filter. License ------- diff --git a/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java b/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java index 0d3495c8e13..16b1d199271 100644 --- a/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java +++ b/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java @@ -29,4 +29,8 @@ public class PolishAnalysisBinderProcessor extends AnalysisModule.AnalysisBinder public void processAnalyzers(AnalyzersBindings analyzersBindings) { analyzersBindings.processAnalyzer("polish", PolishAnalyzerProvider.class); } + @Override + public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { + tokenFiltersBindings.processTokenFilter("polish_stem", PolishStemTokenFilterFactory.class); + } } diff --git a/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java new file mode 100644 index 00000000000..bb73492e6e5 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis.pl; + +import org.apache.lucene.analysis.pl.PolishAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.stempel.StempelFilter; +import org.apache.lucene.analysis.stempel.StempelStemmer; +import org.egothor.stemmer.Trie; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.assistedinject.Assisted; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; + +import java.io.IOException; + + + +public class PolishStemTokenFilterFactory extends AbstractTokenFilterFactory { + + private final StempelStemmer stemmer; + + @Inject public PolishStemTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { + super(index, indexSettings, name, settings); + Trie tire; + try { + tire = StempelStemmer.load(PolishAnalyzer.class.getResourceAsStream(PolishAnalyzer.DEFAULT_STEMMER_FILE)); + } catch (IOException ex) { + throw new RuntimeException("Unable to load default stemming tables", ex); + } + stemmer = new StempelStemmer(tire); + } + + @Override public TokenStream create(TokenStream tokenStream) { + return new StempelFilter(tokenStream, stemmer); + } +} diff --git a/src/main/java/org/elasticsearch/indices/analysis/pl/PolishIndicesAnalysis.java b/src/main/java/org/elasticsearch/indices/analysis/pl/PolishIndicesAnalysis.java index c2f6db774f8..7dea59938e4 100644 --- a/src/main/java/org/elasticsearch/indices/analysis/pl/PolishIndicesAnalysis.java +++ b/src/main/java/org/elasticsearch/indices/analysis/pl/PolishIndicesAnalysis.java @@ -26,7 +26,16 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory; +import org.elasticsearch.index.analysis.PreBuiltTokenFilterFactoryFactory; import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.analysis.TokenFilterFactory; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.stempel.StempelFilter; +import org.apache.lucene.analysis.stempel.StempelStemmer; +import org.egothor.stemmer.Trie; + +import java.io.IOException; /** * Registers indices level analysis components so, if not explicitly configured, will be shared @@ -38,5 +47,21 @@ public class PolishIndicesAnalysis extends AbstractComponent { public PolishIndicesAnalysis(Settings settings, IndicesAnalysisService indicesAnalysisService) { super(settings); indicesAnalysisService.analyzerProviderFactories().put("default", new PreBuiltAnalyzerProviderFactory("default", AnalyzerScope.INDICES, new PolishAnalyzer(Lucene.ANALYZER_VERSION))); + + indicesAnalysisService.tokenFilterFactories().put("polish_stem", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { + @Override public String name() { + return "polish_stem"; + } + + @Override public TokenStream create(TokenStream tokenStream) { + Trie tire; + try { + tire = StempelStemmer.load(PolishAnalyzer.class.getResourceAsStream(PolishAnalyzer.DEFAULT_STEMMER_FILE)); + } catch (IOException ex) { + throw new RuntimeException("Unable to load default stemming tables", ex); + } + return new StempelFilter(tokenStream, new StempelStemmer(tire)); + } + })); } }