From 71b95fb63c31ad40ad18e1730b277d3d5b007b95 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 24 Jun 2016 16:37:23 -0400 Subject: [PATCH] Switch analysis from push to pull Instead of plugins calling `registerTokenizer` to extend the analyzer they now instead have to implement `AnalysisPlugin` and override `getTokenizer`. This lines up extending plugins in with extending scripts. This allows `AnalysisModule` to construct the `AnalysisRegistry` immediately as part of its constructor which makes testing anslysis much simpler. This also moves the default analysis configuration into `AnalysisModule` which is how search is setup. Like `ScriptModule`, `AnalysisModule` no longer extends `AbstractModule`. Instead it is only responsible for building `AnslysisRegistry`. We still bind `AnalysisRegistry` but we only do so in `Node`. This is means it is available at module construction time so we slowly remove the need to bind it in guice. --- .../resources/checkstyle_suppressions.xml | 7 - .../index/analysis/AnalysisRegistry.java | 203 ++------ .../index/analysis/AnalysisService.java | 6 +- .../PreBuiltAnalyzerProviderFactory.java | 7 +- .../indices/analysis/AnalysisModule.java | 455 +++++++++++++----- .../indices/analysis/HunspellService.java | 8 +- .../java/org/elasticsearch/node/Node.java | 5 +- .../elasticsearch/plugins/AnalysisPlugin.java | 84 ++++ .../org/elasticsearch/plugins/Plugin.java | 9 + .../elasticsearch/plugins/PluginsService.java | 5 +- .../elasticsearch/plugins/ScriptPlugin.java | 3 +- .../indices/TransportAnalyzeActionTests.java | 5 +- .../elasticsearch/index/IndexModuleTests.java | 50 +- .../index/analysis/AnalysisServiceTests.java | 57 ++- .../index/analysis/AnalysisTestsHelper.java | 9 +- .../index/analysis/CharFilterTests.java | 8 +- .../index/analysis/CompoundAnalysisTests.java | 28 +- .../PatternCaptureTokenFilterTests.java | 3 +- .../index/analysis/StopAnalyzerTests.java | 4 +- .../synonyms/SynonymsAnalysisTests.java | 6 +- .../elasticsearch/index/codec/CodecTests.java | 4 +- .../analysis/AnalysisModuleTests.java | 59 ++- .../indices/analysis/DummyAnalysisPlugin.java | 35 +- ...rviceIT.java => HunspellServiceTests.java} | 68 +-- .../migration/migrate_5_0/plugins.asciidoc | 5 + .../messy/tests/TemplateQueryParserTests.java | 4 +- .../analysis/icu/AnalysisICUPlugin.java | 40 +- .../analysis/IcuTokenizerFactoryTests.java | 2 +- .../analysis/SimpleIcuAnalysisTests.java | 3 +- .../SimpleIcuCollationTokenFilterTests.java | 18 +- .../SimpleIcuNormalizerCharFilterTests.java | 4 +- .../kuromoji/AnalysisKuromojiPlugin.java | 52 +- .../index/analysis/KuromojiAnalysisTests.java | 2 +- .../analysis/AnalysisPhoneticPlugin.java | 14 +- .../analysis/SimplePhoneticAnalysisTests.java | 3 +- .../smartcn/AnalysisSmartChinesePlugin.java | 38 +- .../SimpleSmartChineseAnalysisTests.java | 5 +- .../stempel/AnalysisStempelPlugin.java | 22 +- .../index/analysis/PolishAnalysisTests.java | 3 +- .../SimplePolishTokenFilterTests.java | 5 +- .../elasticsearch/index/MapperTestUtils.java | 4 +- .../test/AbstractQueryTestCase.java | 6 +- .../org/elasticsearch/test/ESTestCase.java | 39 +- 43 files changed, 844 insertions(+), 553 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java rename core/src/test/java/org/elasticsearch/{index => indices}/analysis/AnalysisModuleTests.java (87%) rename core/src/test/java/org/elasticsearch/indices/analyze/{HunspellServiceIT.java => HunspellServiceTests.java} (58%) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 02aec49b1c7..745c8374063 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -477,8 +477,6 @@ - - @@ -879,9 +877,6 @@ - - - @@ -1186,8 +1181,6 @@ - - diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 2d73df76f07..548bc91b0a5 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -23,14 +23,11 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; -import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.indices.analysis.HunspellService; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.indices.analysis.PreBuiltCharFilters; import org.elasticsearch.indices.analysis.PreBuiltTokenFilters; @@ -45,48 +42,32 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import static java.util.Collections.unmodifiableMap; + /** * An internal registry for tokenizer, token filter, char filter and analyzer. * This class exists per node and allows to create per-index {@link AnalysisService} via {@link #build(IndexSettings)} */ public final class AnalysisRegistry implements Closeable { - private final Map> charFilters; - private final Map> tokenFilters; - private final Map> tokenizers; - private final Map> analyzers; + private final PrebuiltAnalysis prebuiltAnalysis = new PrebuiltAnalysis(); private final Map cachedAnalyzer = new ConcurrentHashMap<>(); - private final PrebuiltAnalysis prebuiltAnalysis; - private final HunspellService hunspellService; + private final Environment environment; + private final Map> charFilters; + private final Map> tokenFilters; + private final Map> tokenizers; + private final Map>> analyzers; - public AnalysisRegistry(HunspellService hunspellService, Environment environment) { - this(hunspellService, environment, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); - } - - public AnalysisRegistry(HunspellService hunspellService, Environment environment, - Map> charFilters, - Map> tokenFilters, - Map> tokenizers, - Map> analyzers) { - prebuiltAnalysis = new PrebuiltAnalysis(); - this.hunspellService = hunspellService; + public AnalysisRegistry(Environment environment, + Map> charFilters, + Map> tokenFilters, + Map> tokenizers, + Map>> analyzers) { this.environment = environment; - final Map> charFilterBuilder = new HashMap<>(charFilters); - final Map> tokenFilterBuilder = new HashMap<>(tokenFilters); - final Map> tokenizerBuilder = new HashMap<>(tokenizers); - final Map> analyzerBuilder= new HashMap<>(analyzers); - registerBuiltInAnalyzer(analyzerBuilder); - registerBuiltInCharFilter(charFilterBuilder); - registerBuiltInTokenizer(tokenizerBuilder); - registerBuiltInTokenFilters(tokenFilterBuilder); - this.tokenFilters = Collections.unmodifiableMap(tokenFilterBuilder); - this.tokenizers = Collections.unmodifiableMap(tokenizerBuilder); - this.charFilters = Collections.unmodifiableMap(charFilterBuilder); - this.analyzers = Collections.unmodifiableMap(analyzerBuilder); - } - - public HunspellService getHunspellService() { - return hunspellService; + this.charFilters = unmodifiableMap(charFilters); + this.tokenFilters = unmodifiableMap(tokenFilters); + this.tokenizers = unmodifiableMap(tokenizers); + this.analyzers = unmodifiableMap(analyzers); } /** @@ -114,9 +95,9 @@ public final class AnalysisRegistry implements Closeable { * Returns a registered {@link Analyzer} provider by name or null if the analyzer was not registered */ public Analyzer getAnalyzer(String analyzer) throws IOException { - AnalysisModule.AnalysisProvider analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer); + AnalysisModule.AnalysisProvider> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer); if (analyzerProvider == null) { - AnalysisModule.AnalysisProvider provider = analyzers.get(analyzer); + AnalysisModule.AnalysisProvider> provider = analyzers.get(analyzer); return provider == null ? null : cachedAnalyzer.computeIfAbsent(analyzer, (key) -> { try { return provider.get(environment, key).get(); @@ -157,7 +138,8 @@ public final class AnalysisRegistry implements Closeable { */ tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, tokenizerFactories, name, settings))); final Map tokenFilterFactories = buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories); - final Map analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories); + final Map> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings, + analyzers, prebuiltAnalysis.analyzerProviderFactories); return new AnalysisService(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); } @@ -175,140 +157,9 @@ public final class AnalysisRegistry implements Closeable { }; } - private void registerBuiltInCharFilter(Map> charFilters) { - charFilters.put("html_strip", HtmlStripCharFilterFactory::new); - charFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new)); - charFilters.put("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new)); - } - - private void registerBuiltInTokenizer(Map> tokenizers) { - tokenizers.put("standard", StandardTokenizerFactory::new); - tokenizers.put("uax_url_email", UAX29URLEmailTokenizerFactory::new); - tokenizers.put("path_hierarchy", PathHierarchyTokenizerFactory::new); - tokenizers.put("PathHierarchy", PathHierarchyTokenizerFactory::new); - tokenizers.put("keyword", KeywordTokenizerFactory::new); - tokenizers.put("letter", LetterTokenizerFactory::new); - tokenizers.put("lowercase", LowerCaseTokenizerFactory::new); - tokenizers.put("whitespace", WhitespaceTokenizerFactory::new); - tokenizers.put("nGram", NGramTokenizerFactory::new); - tokenizers.put("ngram", NGramTokenizerFactory::new); - tokenizers.put("edgeNGram", EdgeNGramTokenizerFactory::new); - tokenizers.put("edge_ngram", EdgeNGramTokenizerFactory::new); - tokenizers.put("pattern", PatternTokenizerFactory::new); - tokenizers.put("classic", ClassicTokenizerFactory::new); - tokenizers.put("thai", ThaiTokenizerFactory::new); - } - - private void registerBuiltInTokenFilters(Map> tokenFilters) { - tokenFilters.put("stop", StopTokenFilterFactory::new); - tokenFilters.put("reverse", ReverseTokenFilterFactory::new); - tokenFilters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new); - tokenFilters.put("length", LengthTokenFilterFactory::new); - tokenFilters.put("lowercase", LowerCaseTokenFilterFactory::new); - tokenFilters.put("uppercase", UpperCaseTokenFilterFactory::new); - tokenFilters.put("porter_stem", PorterStemTokenFilterFactory::new); - tokenFilters.put("kstem", KStemTokenFilterFactory::new); - tokenFilters.put("standard", StandardTokenFilterFactory::new); - tokenFilters.put("nGram", NGramTokenFilterFactory::new); - tokenFilters.put("ngram", NGramTokenFilterFactory::new); - tokenFilters.put("edgeNGram", EdgeNGramTokenFilterFactory::new); - tokenFilters.put("edge_ngram", EdgeNGramTokenFilterFactory::new); - tokenFilters.put("shingle", ShingleTokenFilterFactory::new); - tokenFilters.put("unique", UniqueTokenFilterFactory::new); - tokenFilters.put("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new)); - tokenFilters.put("trim", TrimTokenFilterFactory::new); - tokenFilters.put("limit", LimitTokenCountFilterFactory::new); - tokenFilters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new)); - tokenFilters.put("snowball", SnowballTokenFilterFactory::new); - tokenFilters.put("stemmer", StemmerTokenFilterFactory::new); - tokenFilters.put("word_delimiter", WordDelimiterTokenFilterFactory::new); - tokenFilters.put("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new); - tokenFilters.put("elision", ElisionTokenFilterFactory::new); - tokenFilters.put("keep", requriesAnalysisSettings(KeepWordFilterFactory::new)); - tokenFilters.put("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new)); - tokenFilters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new)); - tokenFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new)); - tokenFilters.put("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new)); - tokenFilters.put("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new)); - tokenFilters.put("arabic_stem", ArabicStemTokenFilterFactory::new); - tokenFilters.put("brazilian_stem", BrazilianStemTokenFilterFactory::new); - tokenFilters.put("czech_stem", CzechStemTokenFilterFactory::new); - tokenFilters.put("dutch_stem", DutchStemTokenFilterFactory::new); - tokenFilters.put("french_stem", FrenchStemTokenFilterFactory::new); - tokenFilters.put("german_stem", GermanStemTokenFilterFactory::new); - tokenFilters.put("russian_stem", RussianStemTokenFilterFactory::new); - tokenFilters.put("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new)); - tokenFilters.put("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); - tokenFilters.put("arabic_normalization", ArabicNormalizationFilterFactory::new); - tokenFilters.put("german_normalization", GermanNormalizationFilterFactory::new); - tokenFilters.put("hindi_normalization", HindiNormalizationFilterFactory::new); - tokenFilters.put("indic_normalization", IndicNormalizationFilterFactory::new); - tokenFilters.put("sorani_normalization", SoraniNormalizationFilterFactory::new); - tokenFilters.put("persian_normalization", PersianNormalizationFilterFactory::new); - tokenFilters.put("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new); - tokenFilters.put("scandinavian_folding", ScandinavianFoldingFilterFactory::new); - tokenFilters.put("serbian_normalization", SerbianNormalizationFilterFactory::new); - - if (hunspellService != null) { - tokenFilters.put("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService))); - } - tokenFilters.put("cjk_bigram", CJKBigramFilterFactory::new); - tokenFilters.put("cjk_width", CJKWidthFilterFactory::new); - - tokenFilters.put("apostrophe", ApostropheFilterFactory::new); - tokenFilters.put("classic", ClassicFilterFactory::new); - tokenFilters.put("decimal_digit", DecimalDigitFilterFactory::new); - tokenFilters.put("fingerprint", FingerprintTokenFilterFactory::new); - } - - private void registerBuiltInAnalyzer(Map> analyzers) { - analyzers.put("default", StandardAnalyzerProvider::new); - analyzers.put("standard", StandardAnalyzerProvider::new); - analyzers.put("standard_html_strip", StandardHtmlStripAnalyzerProvider::new); - analyzers.put("simple", SimpleAnalyzerProvider::new); - analyzers.put("stop", StopAnalyzerProvider::new); - analyzers.put("whitespace", WhitespaceAnalyzerProvider::new); - analyzers.put("keyword", KeywordAnalyzerProvider::new); - analyzers.put("pattern", PatternAnalyzerProvider::new); - analyzers.put("snowball", SnowballAnalyzerProvider::new); - analyzers.put("arabic", ArabicAnalyzerProvider::new); - analyzers.put("armenian", ArmenianAnalyzerProvider::new); - analyzers.put("basque", BasqueAnalyzerProvider::new); - analyzers.put("brazilian", BrazilianAnalyzerProvider::new); - analyzers.put("bulgarian", BulgarianAnalyzerProvider::new); - analyzers.put("catalan", CatalanAnalyzerProvider::new); - analyzers.put("chinese", ChineseAnalyzerProvider::new); - analyzers.put("cjk", CjkAnalyzerProvider::new); - analyzers.put("czech", CzechAnalyzerProvider::new); - analyzers.put("danish", DanishAnalyzerProvider::new); - analyzers.put("dutch", DutchAnalyzerProvider::new); - analyzers.put("english", EnglishAnalyzerProvider::new); - analyzers.put("finnish", FinnishAnalyzerProvider::new); - analyzers.put("french", FrenchAnalyzerProvider::new); - analyzers.put("galician", GalicianAnalyzerProvider::new); - analyzers.put("german", GermanAnalyzerProvider::new); - analyzers.put("greek", GreekAnalyzerProvider::new); - analyzers.put("hindi", HindiAnalyzerProvider::new); - analyzers.put("hungarian", HungarianAnalyzerProvider::new); - analyzers.put("indonesian", IndonesianAnalyzerProvider::new); - analyzers.put("irish", IrishAnalyzerProvider::new); - analyzers.put("italian", ItalianAnalyzerProvider::new); - analyzers.put("latvian", LatvianAnalyzerProvider::new); - analyzers.put("lithuanian", LithuanianAnalyzerProvider::new); - analyzers.put("norwegian", NorwegianAnalyzerProvider::new); - analyzers.put("persian", PersianAnalyzerProvider::new); - analyzers.put("portuguese", PortugueseAnalyzerProvider::new); - analyzers.put("romanian", RomanianAnalyzerProvider::new); - analyzers.put("russian", RussianAnalyzerProvider::new); - analyzers.put("sorani", SoraniAnalyzerProvider::new); - analyzers.put("spanish", SpanishAnalyzerProvider::new); - analyzers.put("swedish", SwedishAnalyzerProvider::new); - analyzers.put("turkish", TurkishAnalyzerProvider::new); - analyzers.put("thai", ThaiAnalyzerProvider::new); - analyzers.put("fingerprint", FingerprintAnalyzerProvider::new); - } - - private Map buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map settingsMap, Map> providerMap, Map> defaultInstance) throws IOException { + private Map buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map settingsMap, + Map> providerMap, Map> defaultInstance) + throws IOException { Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build(); Map factories = new HashMap<>(); for (Map.Entry entry : settingsMap.entrySet()) { @@ -383,7 +234,7 @@ public final class AnalysisRegistry implements Closeable { private static class PrebuiltAnalysis implements Closeable { - final Map> analyzerProviderFactories; + final Map>> analyzerProviderFactories; final Map> tokenizerFactories; final Map> tokenFilterFactories; final Map> charFilterFactories; @@ -446,7 +297,7 @@ public final class AnalysisRegistry implements Closeable { return tokenizerFactories.get(name); } - public AnalysisModule.AnalysisProvider getAnalyzerProvider(String name) { + public AnalysisModule.AnalysisProvider> getAnalyzerProvider(String name) { return analyzerProviderFactories.get(name); } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java index b9146df8c96..0009fc95409 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java @@ -48,7 +48,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable private final NamedAnalyzer defaultSearchQuoteAnalyzer; public AnalysisService(IndexSettings indexSettings, - Map analyzerProviders, + Map> analyzerProviders, Map tokenizerFactoryFactories, Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { @@ -69,8 +69,8 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable } Map analyzers = new HashMap<>(); - for (Map.Entry entry : analyzerProviders.entrySet()) { - AnalyzerProvider analyzerFactory = entry.getValue(); + for (Map.Entry> entry : analyzerProviders.entrySet()) { + AnalyzerProvider analyzerFactory = entry.getValue(); String name = entry.getKey(); /* * Lucene defaults positionIncrementGap to 0 in all analyzers but diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index 00724dccfa1..786e24a0844 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -32,7 +32,7 @@ import java.io.IOException; /** * */ -public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider { +public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider> { private final PreBuiltAnalyzerProvider analyzerProvider; @@ -40,7 +40,7 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer); } - public AnalyzerProvider create(String name, Settings settings) { + public AnalyzerProvider create(String name, Settings settings) { Version indexVersion = Version.indexCreated(settings); if (!Version.CURRENT.equals(indexVersion)) { PreBuiltAnalyzers preBuiltAnalyzers = PreBuiltAnalyzers.getOrDefault(name, null); @@ -54,7 +54,8 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP } @Override - public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { + public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) + throws IOException { return create(name, settings); } diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index b5b480fc909..12d0b8bbb6e 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -19,49 +19,141 @@ package org.elasticsearch.indices.analysis; -import org.apache.lucene.analysis.hunspell.Dictionary; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.ApostropheFilterFactory; +import org.elasticsearch.index.analysis.ArabicAnalyzerProvider; +import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory; +import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory; +import org.elasticsearch.index.analysis.ArmenianAnalyzerProvider; +import org.elasticsearch.index.analysis.BasqueAnalyzerProvider; +import org.elasticsearch.index.analysis.BrazilianAnalyzerProvider; +import org.elasticsearch.index.analysis.BrazilianStemTokenFilterFactory; +import org.elasticsearch.index.analysis.BulgarianAnalyzerProvider; +import org.elasticsearch.index.analysis.CJKBigramFilterFactory; +import org.elasticsearch.index.analysis.CJKWidthFilterFactory; +import org.elasticsearch.index.analysis.CatalanAnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.ChineseAnalyzerProvider; +import org.elasticsearch.index.analysis.CjkAnalyzerProvider; +import org.elasticsearch.index.analysis.ClassicFilterFactory; +import org.elasticsearch.index.analysis.ClassicTokenizerFactory; +import org.elasticsearch.index.analysis.CommonGramsTokenFilterFactory; +import org.elasticsearch.index.analysis.CzechAnalyzerProvider; +import org.elasticsearch.index.analysis.CzechStemTokenFilterFactory; +import org.elasticsearch.index.analysis.DanishAnalyzerProvider; +import org.elasticsearch.index.analysis.DecimalDigitFilterFactory; +import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory; +import org.elasticsearch.index.analysis.DutchAnalyzerProvider; +import org.elasticsearch.index.analysis.DutchStemTokenFilterFactory; +import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory; +import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory; +import org.elasticsearch.index.analysis.ElisionTokenFilterFactory; +import org.elasticsearch.index.analysis.EnglishAnalyzerProvider; +import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider; +import org.elasticsearch.index.analysis.FingerprintTokenFilterFactory; +import org.elasticsearch.index.analysis.FinnishAnalyzerProvider; +import org.elasticsearch.index.analysis.FrenchAnalyzerProvider; +import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory; +import org.elasticsearch.index.analysis.GalicianAnalyzerProvider; +import org.elasticsearch.index.analysis.GermanAnalyzerProvider; +import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory; +import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory; +import org.elasticsearch.index.analysis.GreekAnalyzerProvider; +import org.elasticsearch.index.analysis.HindiAnalyzerProvider; +import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory; +import org.elasticsearch.index.analysis.HtmlStripCharFilterFactory; +import org.elasticsearch.index.analysis.HungarianAnalyzerProvider; +import org.elasticsearch.index.analysis.HunspellTokenFilterFactory; +import org.elasticsearch.index.analysis.IndicNormalizationFilterFactory; +import org.elasticsearch.index.analysis.IndonesianAnalyzerProvider; +import org.elasticsearch.index.analysis.IrishAnalyzerProvider; +import org.elasticsearch.index.analysis.ItalianAnalyzerProvider; +import org.elasticsearch.index.analysis.KStemTokenFilterFactory; +import org.elasticsearch.index.analysis.KeepTypesFilterFactory; +import org.elasticsearch.index.analysis.KeepWordFilterFactory; +import org.elasticsearch.index.analysis.KeywordAnalyzerProvider; +import org.elasticsearch.index.analysis.KeywordMarkerTokenFilterFactory; +import org.elasticsearch.index.analysis.KeywordTokenizerFactory; +import org.elasticsearch.index.analysis.LatvianAnalyzerProvider; +import org.elasticsearch.index.analysis.LengthTokenFilterFactory; +import org.elasticsearch.index.analysis.LetterTokenizerFactory; +import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory; +import org.elasticsearch.index.analysis.LithuanianAnalyzerProvider; +import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory; +import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory; +import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.NGramTokenFilterFactory; +import org.elasticsearch.index.analysis.NGramTokenizerFactory; +import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider; +import org.elasticsearch.index.analysis.PathHierarchyTokenizerFactory; +import org.elasticsearch.index.analysis.PatternAnalyzerProvider; +import org.elasticsearch.index.analysis.PatternCaptureGroupTokenFilterFactory; +import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory; +import org.elasticsearch.index.analysis.PatternReplaceTokenFilterFactory; +import org.elasticsearch.index.analysis.PatternTokenizerFactory; +import org.elasticsearch.index.analysis.PersianAnalyzerProvider; +import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory; +import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory; +import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider; +import org.elasticsearch.index.analysis.ReverseTokenFilterFactory; +import org.elasticsearch.index.analysis.RomanianAnalyzerProvider; +import org.elasticsearch.index.analysis.RussianAnalyzerProvider; +import org.elasticsearch.index.analysis.RussianStemTokenFilterFactory; +import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory; +import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory; +import org.elasticsearch.index.analysis.SerbianNormalizationFilterFactory; +import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.analysis.SimpleAnalyzerProvider; +import org.elasticsearch.index.analysis.SnowballAnalyzerProvider; +import org.elasticsearch.index.analysis.SnowballTokenFilterFactory; +import org.elasticsearch.index.analysis.SoraniAnalyzerProvider; +import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory; +import org.elasticsearch.index.analysis.SpanishAnalyzerProvider; +import org.elasticsearch.index.analysis.StandardAnalyzerProvider; +import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzerProvider; +import org.elasticsearch.index.analysis.StandardTokenFilterFactory; +import org.elasticsearch.index.analysis.StandardTokenizerFactory; +import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory; +import org.elasticsearch.index.analysis.StemmerTokenFilterFactory; +import org.elasticsearch.index.analysis.StopAnalyzerProvider; +import org.elasticsearch.index.analysis.StopTokenFilterFactory; +import org.elasticsearch.index.analysis.SwedishAnalyzerProvider; +import org.elasticsearch.index.analysis.ThaiAnalyzerProvider; +import org.elasticsearch.index.analysis.ThaiTokenizerFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.index.analysis.TrimTokenFilterFactory; +import org.elasticsearch.index.analysis.TruncateTokenFilterFactory; +import org.elasticsearch.index.analysis.TurkishAnalyzerProvider; +import org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory; +import org.elasticsearch.index.analysis.UniqueTokenFilterFactory; +import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory; +import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider; +import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory; +import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; +import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; +import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; +import org.elasticsearch.plugins.AnalysisPlugin; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; /** - * The AnalysisModule is the main extension point for node and index level analysis components. The lucene classes - * {@link org.apache.lucene.analysis.Analyzer}, {@link org.apache.lucene.analysis.TokenFilter}, {@link org.apache.lucene.analysis.Tokenizer} - * and {@link org.apache.lucene.analysis.CharFilter} can be extended in plugins and registered on node startup when the analysis module - * gets loaded. Since elasticsearch needs to create multiple instances for different configurations dedicated factories need to be provided for - * each of the components: - *
    - *
  • {@link org.apache.lucene.analysis.Analyzer} can be exposed via {@link AnalyzerProvider} and registered on {@link #registerAnalyzer(String, AnalysisProvider)}
    • - *
  • {@link org.apache.lucene.analysis.TokenFilter} can be exposed via {@link TokenFilterFactory} and registered on {@link #registerTokenFilter(String, AnalysisProvider)}
    • - *
  • {@link org.apache.lucene.analysis.Tokenizer} can be exposed via {@link TokenizerFactory} and registered on {@link #registerTokenizer(String, AnalysisProvider)}
    • - *
  • {@link org.apache.lucene.analysis.CharFilter} can be exposed via {@link CharFilterFactory} and registered on {@link #registerCharFilter(String, AnalysisProvider)}
    • - *
- * - * The {@link org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider} is only a functional interface that allows to register factory constructors directly like the plugin example below: - * 
- *     public class MyAnalysisPlugin extends Plugin {
- *       public void onModule(AnalysisModule module) {
- *         module.registerAnalyzer("my-analyzer-name", MyAnalyzer::new);
- *       }
- *     }
- *
+ * Sets up {@link AnalysisRegistry}. */ -public final class AnalysisModule extends AbstractModule { - +public final class AnalysisModule { static { Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) @@ -71,108 +163,195 @@ public final class AnalysisModule extends AbstractModule { NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY); } private static final IndexSettings NA_INDEX_SETTINGS; - private final Environment environment; - private final Map> charFilters = new HashMap<>(); - private final Map> tokenFilters = new HashMap<>(); - private final Map> tokenizers = new HashMap<>(); - private final Map> analyzers = new HashMap<>(); - private final Map knownDictionaries = new HashMap<>(); - /** - * Creates a new AnalysisModule - */ - public AnalysisModule(Environment environment) { - this.environment = environment; + private final HunspellService hunspellService; + private final AnalysisRegistry analysisRegistry; + + public AnalysisModule(Environment environment, List plugins) throws IOException { + NamedRegistry> charFilters = setupCharFilters(plugins); + NamedRegistry hunspellDictionaries = setupHunspellDictionaries(plugins); + hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.registry); + NamedRegistry> tokenFilters = setupTokenFilters(plugins, hunspellService); + NamedRegistry> tokenizers = setupTokenizers(plugins); + NamedRegistry>> analyzers = setupAnalyzers(plugins); + analysisRegistry = new AnalysisRegistry(environment, charFilters.registry, tokenFilters.registry, + tokenizers.registry, analyzers.registry); + } + + HunspellService getHunspellService() { + return hunspellService; + } + + public AnalysisRegistry getAnalysisRegistry() { + return analysisRegistry; + } + + private NamedRegistry> setupCharFilters(List plugins) { + NamedRegistry> charFilters = new NamedRegistry<>("char_filter"); + charFilters.register("html_strip", HtmlStripCharFilterFactory::new); + charFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new)); + charFilters.register("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new)); + charFilters.registerPlugins(plugins, AnalysisPlugin::getCharFilters); + return charFilters; + } + + public NamedRegistry setupHunspellDictionaries(List plugins) { + NamedRegistry hunspellDictionaries = new NamedRegistry<>("dictionary"); + hunspellDictionaries.registerPlugins(plugins, AnalysisPlugin::getHunspellDictionaries); + return hunspellDictionaries; + } + + private NamedRegistry> setupTokenFilters(List plugins, + HunspellService hunspellService) { + NamedRegistry> tokenFilters = new NamedRegistry<>("token_filter"); + tokenFilters.register("stop", StopTokenFilterFactory::new); + tokenFilters.register("reverse", ReverseTokenFilterFactory::new); + tokenFilters.register("asciifolding", ASCIIFoldingTokenFilterFactory::new); + tokenFilters.register("length", LengthTokenFilterFactory::new); + tokenFilters.register("lowercase", LowerCaseTokenFilterFactory::new); + tokenFilters.register("uppercase", UpperCaseTokenFilterFactory::new); + tokenFilters.register("porter_stem", PorterStemTokenFilterFactory::new); + tokenFilters.register("kstem", KStemTokenFilterFactory::new); + tokenFilters.register("standard", StandardTokenFilterFactory::new); + tokenFilters.register("nGram", NGramTokenFilterFactory::new); + tokenFilters.register("ngram", NGramTokenFilterFactory::new); + tokenFilters.register("edgeNGram", EdgeNGramTokenFilterFactory::new); + tokenFilters.register("edge_ngram", EdgeNGramTokenFilterFactory::new); + tokenFilters.register("shingle", ShingleTokenFilterFactory::new); + tokenFilters.register("unique", UniqueTokenFilterFactory::new); + tokenFilters.register("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new)); + tokenFilters.register("trim", TrimTokenFilterFactory::new); + tokenFilters.register("limit", LimitTokenCountFilterFactory::new); + tokenFilters.register("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new)); + tokenFilters.register("snowball", SnowballTokenFilterFactory::new); + tokenFilters.register("stemmer", StemmerTokenFilterFactory::new); + tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new); + tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new); + tokenFilters.register("elision", ElisionTokenFilterFactory::new); + tokenFilters.register("keep", requriesAnalysisSettings(KeepWordFilterFactory::new)); + tokenFilters.register("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new)); + tokenFilters.register("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new)); + tokenFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new)); + tokenFilters.register("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new)); + tokenFilters.register("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new)); + tokenFilters.register("arabic_stem", ArabicStemTokenFilterFactory::new); + tokenFilters.register("brazilian_stem", BrazilianStemTokenFilterFactory::new); + tokenFilters.register("czech_stem", CzechStemTokenFilterFactory::new); + tokenFilters.register("dutch_stem", DutchStemTokenFilterFactory::new); + tokenFilters.register("french_stem", FrenchStemTokenFilterFactory::new); + tokenFilters.register("german_stem", GermanStemTokenFilterFactory::new); + tokenFilters.register("russian_stem", RussianStemTokenFilterFactory::new); + tokenFilters.register("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new)); + tokenFilters.register("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); + tokenFilters.register("arabic_normalization", ArabicNormalizationFilterFactory::new); + tokenFilters.register("german_normalization", GermanNormalizationFilterFactory::new); + tokenFilters.register("hindi_normalization", HindiNormalizationFilterFactory::new); + tokenFilters.register("indic_normalization", IndicNormalizationFilterFactory::new); + tokenFilters.register("sorani_normalization", SoraniNormalizationFilterFactory::new); + tokenFilters.register("persian_normalization", PersianNormalizationFilterFactory::new); + tokenFilters.register("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new); + tokenFilters.register("scandinavian_folding", ScandinavianFoldingFilterFactory::new); + tokenFilters.register("serbian_normalization", SerbianNormalizationFilterFactory::new); + + tokenFilters.register("hunspell", requriesAnalysisSettings( + (indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService))); + tokenFilters.register("cjk_bigram", CJKBigramFilterFactory::new); + tokenFilters.register("cjk_width", CJKWidthFilterFactory::new); + + tokenFilters.register("apostrophe", ApostropheFilterFactory::new); + tokenFilters.register("classic", ClassicFilterFactory::new); + tokenFilters.register("decimal_digit", DecimalDigitFilterFactory::new); + tokenFilters.register("fingerprint", FingerprintTokenFilterFactory::new); + tokenFilters.registerPlugins(plugins, AnalysisPlugin::getTokenFilters); + return tokenFilters; + } + + private NamedRegistry> setupTokenizers(List plugins) { + NamedRegistry> tokenizers = new NamedRegistry<>("tokenizer"); + tokenizers.register("standard", StandardTokenizerFactory::new); + tokenizers.register("uax_url_email", UAX29URLEmailTokenizerFactory::new); + tokenizers.register("path_hierarchy", PathHierarchyTokenizerFactory::new); + tokenizers.register("PathHierarchy", PathHierarchyTokenizerFactory::new); + tokenizers.register("keyword", KeywordTokenizerFactory::new); + tokenizers.register("letter", LetterTokenizerFactory::new); + tokenizers.register("lowercase", LowerCaseTokenizerFactory::new); + tokenizers.register("whitespace", WhitespaceTokenizerFactory::new); + tokenizers.register("nGram", NGramTokenizerFactory::new); + tokenizers.register("ngram", NGramTokenizerFactory::new); + tokenizers.register("edgeNGram", EdgeNGramTokenizerFactory::new); + tokenizers.register("edge_ngram", EdgeNGramTokenizerFactory::new); + tokenizers.register("pattern", PatternTokenizerFactory::new); + tokenizers.register("classic", ClassicTokenizerFactory::new); + tokenizers.register("thai", ThaiTokenizerFactory::new); + tokenizers.registerPlugins(plugins, AnalysisPlugin::getTokenizers); + return tokenizers; + } + + private NamedRegistry>> setupAnalyzers(List plugins) { + NamedRegistry>> analyzers = new NamedRegistry<>("analyzer"); + analyzers.register("default", StandardAnalyzerProvider::new); + analyzers.register("standard", StandardAnalyzerProvider::new); + analyzers.register("standard_html_strip", StandardHtmlStripAnalyzerProvider::new); + analyzers.register("simple", SimpleAnalyzerProvider::new); + analyzers.register("stop", StopAnalyzerProvider::new); + analyzers.register("whitespace", WhitespaceAnalyzerProvider::new); + analyzers.register("keyword", KeywordAnalyzerProvider::new); + analyzers.register("pattern", PatternAnalyzerProvider::new); + analyzers.register("snowball", SnowballAnalyzerProvider::new); + analyzers.register("arabic", ArabicAnalyzerProvider::new); + analyzers.register("armenian", ArmenianAnalyzerProvider::new); + analyzers.register("basque", BasqueAnalyzerProvider::new); + analyzers.register("brazilian", BrazilianAnalyzerProvider::new); + analyzers.register("bulgarian", BulgarianAnalyzerProvider::new); + analyzers.register("catalan", CatalanAnalyzerProvider::new); + analyzers.register("chinese", ChineseAnalyzerProvider::new); + analyzers.register("cjk", CjkAnalyzerProvider::new); + analyzers.register("czech", CzechAnalyzerProvider::new); + analyzers.register("danish", DanishAnalyzerProvider::new); + analyzers.register("dutch", DutchAnalyzerProvider::new); + analyzers.register("english", EnglishAnalyzerProvider::new); + analyzers.register("finnish", FinnishAnalyzerProvider::new); + analyzers.register("french", FrenchAnalyzerProvider::new); + analyzers.register("galician", GalicianAnalyzerProvider::new); + analyzers.register("german", GermanAnalyzerProvider::new); + analyzers.register("greek", GreekAnalyzerProvider::new); + analyzers.register("hindi", HindiAnalyzerProvider::new); + analyzers.register("hungarian", HungarianAnalyzerProvider::new); + analyzers.register("indonesian", IndonesianAnalyzerProvider::new); + analyzers.register("irish", IrishAnalyzerProvider::new); + analyzers.register("italian", ItalianAnalyzerProvider::new); + analyzers.register("latvian", LatvianAnalyzerProvider::new); + analyzers.register("lithuanian", LithuanianAnalyzerProvider::new); + analyzers.register("norwegian", NorwegianAnalyzerProvider::new); + analyzers.register("persian", PersianAnalyzerProvider::new); + analyzers.register("portuguese", PortugueseAnalyzerProvider::new); + analyzers.register("romanian", RomanianAnalyzerProvider::new); + analyzers.register("russian", RussianAnalyzerProvider::new); + analyzers.register("sorani", SoraniAnalyzerProvider::new); + analyzers.register("spanish", SpanishAnalyzerProvider::new); + analyzers.register("swedish", SwedishAnalyzerProvider::new); + analyzers.register("turkish", TurkishAnalyzerProvider::new); + analyzers.register("thai", ThaiAnalyzerProvider::new); + analyzers.register("fingerprint", FingerprintAnalyzerProvider::new); + analyzers.registerPlugins(plugins, AnalysisPlugin::getAnalyzers); + return analyzers; + } + + private static AnalysisModule.AnalysisProvider requriesAnalysisSettings(AnalysisModule.AnalysisProvider provider) { + return new AnalysisModule.AnalysisProvider() { + @Override + public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { + return provider.get(indexSettings, environment, name, settings); + } + @Override + public boolean requiresAnalysisSettings() { + return true; + } + }; } /** - * Registers a new {@link AnalysisProvider} to create - * {@link CharFilterFactory} instance per node as well as per index. - */ - public void registerCharFilter(String name, AnalysisProvider charFilter) { - if (charFilter == null) { - throw new IllegalArgumentException("char_filter provider must not be null"); - } - if (charFilters.putIfAbsent(name, charFilter) != null) { - throw new IllegalArgumentException("char_filter provider for name " + name + " already registered"); - } - } - - /** - * Registers a new {@link AnalysisProvider} to create - * {@link TokenFilterFactory} instance per node as well as per index. - */ - public void registerTokenFilter(String name, AnalysisProvider tokenFilter) { - if (tokenFilter == null) { - throw new IllegalArgumentException("token_filter provider must not be null"); - } - if (tokenFilters.putIfAbsent(name, tokenFilter) != null) { - throw new IllegalArgumentException("token_filter provider for name " + name + " already registered"); - } - } - - /** - * Registers a new {@link AnalysisProvider} to create - * {@link TokenizerFactory} instance per node as well as per index. - */ - public void registerTokenizer(String name, AnalysisProvider tokenizer) { - if (tokenizer == null) { - throw new IllegalArgumentException("tokenizer provider must not be null"); - } - if (tokenizers.putIfAbsent(name, tokenizer) != null) { - throw new IllegalArgumentException("tokenizer provider for name " + name + " already registered"); - } - } - - /** - * Registers a new {@link AnalysisProvider} to create - * {@link AnalyzerProvider} instance per node as well as per index. - */ - public void registerAnalyzer(String name, AnalysisProvider analyzer) { - if (analyzer == null) { - throw new IllegalArgumentException("analyzer provider must not be null"); - } - if (analyzers.putIfAbsent(name, analyzer) != null) { - throw new IllegalArgumentException("analyzer provider for name " + name + " already registered"); - } - } - - /** - * Registers a new hunspell {@link Dictionary} that can be referenced by the given name in - * hunspell analysis configuration. - */ - public void registerHunspellDictionary(String name, Dictionary dictionary) { - if (knownDictionaries.putIfAbsent(name, dictionary) != null) { - throw new IllegalArgumentException("dictionary for [" + name + "] is already registered"); - } - } - - @Override - protected void configure() { - try { - AnalysisRegistry registry = buildRegistry(); - bind(HunspellService.class).toInstance(registry.getHunspellService()); - bind(AnalysisRegistry.class).toInstance(registry); - } catch (IOException e) { - throw new ElasticsearchException("failed to load hunspell service", e); - } - } - - /** - * Builds an {@link AnalysisRegistry} from the current configuration. - */ - public AnalysisRegistry buildRegistry() throws IOException { - return new AnalysisRegistry(new HunspellService(environment.settings(), environment, knownDictionaries), environment, charFilters, tokenFilters, tokenizers, analyzers); - } - - /** - * AnalysisProvider is the basic factory interface for registering analysis components like: - *
    - *
  • {@link TokenizerFactory} - see {@link AnalysisModule#registerTokenizer(String, AnalysisProvider)}
    • - *
  • {@link CharFilterFactory} - see {@link AnalysisModule#registerCharFilter(String, AnalysisProvider)}
    • - *
  • {@link AnalyzerProvider} - see {@link AnalysisModule#registerAnalyzer(String, AnalysisProvider)}
    • - *
  • {@link TokenFilterFactory}- see {@link AnalysisModule#registerTokenFilter(String, AnalysisProvider)} )}
    • - *
+ * The basic factory interface for analysis components. */ public interface AnalysisProvider { @@ -195,7 +374,8 @@ public final class AnalysisModule extends AbstractModule { * @param name the name of the analysis component * @return a new provider instance * @throws IOException if an {@link IOException} occurs - * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns true + * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns + * true */ default T get(Environment environment, String name) throws IOException { if (requiresAnalysisSettings()) { @@ -212,4 +392,29 @@ public final class AnalysisModule extends AbstractModule { return false; } } + + private static class NamedRegistry { + private final Map registry = new HashMap<>(); + private final String targetName; + + public NamedRegistry(String targetName) { + this.targetName = targetName; + } + + private void register(String name, T t) { + requireNonNull(name, "name is required"); + requireNonNull(t, targetName + " is required"); + if (registry.putIfAbsent(name, t) != null) { + throw new IllegalArgumentException(targetName + " for name " + name + " already registered"); + } + } + + private

void registerPlugins(List

plugins, Function> lookup) { + for (P plugin : plugins) { + for (Map.Entry entry : lookup.apply(plugin).entrySet()) { + register(entry.getKey(), entry.getValue()); + } + } + } + } } diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java index 27b7ab23c9f..97667fcdac6 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java @@ -52,7 +52,8 @@ import java.util.function.Function; * The following settings can be set for each dictionary: *

    *
  • {@code ignore_case} - If true, dictionary matching will be case insensitive (defaults to {@code false})
    • - *
  • {@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored (defaults to {@code true})
    • + *
  • {@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored + * (defaults to {@code true})
    • *
*

* These settings can either be configured as node level configuration, such as: @@ -86,7 +87,8 @@ public class HunspellService extends AbstractComponent { private final Path hunspellDir; private final Function loadingFunction; - public HunspellService(final Settings settings, final Environment env, final Map knownDictionaries) throws IOException { + public HunspellService(final Settings settings, final Environment env, final Map knownDictionaries) + throws IOException { super(settings); this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries); this.hunspellDir = resolveHunspellDirectory(env); @@ -166,7 +168,7 @@ public class HunspellService extends AbstractComponent { // merging node settings with hunspell dictionary specific settings Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings); - nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale)); + nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + ".")); boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase); diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index fb38c530d45..afc0e5b4cd8 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -72,6 +72,7 @@ import org.elasticsearch.gateway.GatewayModule; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.http.HttpServer; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -85,6 +86,7 @@ import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.node.service.NodeService; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; @@ -227,6 +229,7 @@ public class Node implements Closeable { final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService, pluginsService.filterPlugins(ScriptPlugin.class)); + AnalysisModule analysisModule = new AnalysisModule(environment, pluginsService.filterPlugins(AnalysisPlugin.class)); additionalSettings.addAll(scriptModule.getSettings()); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already @@ -260,7 +263,6 @@ public class Node implements Closeable { modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false)); modules.add(new GatewayModule()); modules.add(new RepositoriesModule()); - modules.add(new AnalysisModule(environment)); pluginsService.processModules(modules); CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(), settingsModule.getClusterSettings()); @@ -279,6 +281,7 @@ public class Node implements Closeable { b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService); b.bind(BigArrays.class).toInstance(bigArrays); b.bind(ScriptService.class).toInstance(scriptModule.getScriptService()); + b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); } ); injector = modules.createInjector(); diff --git a/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java b/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java new file mode 100644 index 00000000000..ffd278a5653 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.CharFilter; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.Tokenizer; +import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; + +import java.util.Map; + +import static java.util.Collections.emptyMap; + +/** + * An additional extension point for {@link Plugin}s that extends Elasticsearch's analysis functionality. To add an additional + * {@link TokenFilter} just implement the interface and implement the {@link #getTokenFilters()} method: + * + * 

{@code
+ * public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin {
+ *     @Override
+ *     public Map> getTokenFilters() {
+ *         return singletonMap("phonetic", PhoneticTokenFilterFactory::new);
+ *     }
+ * }
+ * }
+ */ +public interface AnalysisPlugin { + /** + * Override to add additional {@link CharFilter}s. + */ + default Map> getCharFilters() { + return emptyMap(); + } + + /** + * Override to add additional {@link TokenFilter}s. + */ + default Map> getTokenFilters() { + return emptyMap(); + } + + /** + * Override to add additional {@link Tokenizer}s. + */ + default Map> getTokenizers() { + return emptyMap(); + } + + /** + * Override to add additional {@link Analyzer}s. + */ + default Map>> getAnalyzers() { + return emptyMap(); + } + + /** + * Override to add additional hunspell {@link org.apache.lucene.analysis.hunspell.Dictionary}s. + */ + default Map getHunspellDictionaries() { + return emptyMap(); + } +} diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 0df9eaf467f..08a8ce7124c 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -103,6 +104,14 @@ public abstract class Plugin { @Deprecated public final void onModule(ScriptModule module) {} + /** + * Old-style analysis extension point. + * + * @deprecated implement {@link AnalysisPlugin} instead + */ + @Deprecated + public final void onModule(AnalysisModule module) {} + /** * Provides the list of this plugin's custom thread pools, empty if * none. diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 71081740d33..4769b5954ba 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.script.NativeScriptFactory; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngineService; @@ -207,8 +208,8 @@ public class PluginsService extends AbstractComponent { } Class moduleClass = method.getParameterTypes()[0]; if (!Module.class.isAssignableFrom(moduleClass)) { - if (moduleClass == ScriptModule.class) { - // This is still part of the Plugin class to point the user to the new implementation + if (method.getDeclaringClass() == Plugin.class) { + // These are still part of the Plugin class to point the user to the new implementations continue; } throw new RuntimeException( diff --git a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java index 7f85b8182a5..c1e2a43c953 100644 --- a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java @@ -27,8 +27,7 @@ import java.util.Collections; import java.util.List; /** - * An additional extension point to {@link Plugin}. Plugins extending the scripting functionality must implement this inteface - * to provide access to script engines or script factories. + * An additional extension point for {@link Plugin}s that extends Elasticsearch's scripting functionality. */ public interface ScriptPlugin { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index ff7faab1f6a..5e2c503eba1 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -29,12 +29,15 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.internal.AllFieldMapper; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; + public class TransportAnalyzeActionTests extends ESTestCase { private AnalysisService analysisService; @@ -56,7 +59,7 @@ public class TransportAnalyzeActionTests extends ESTestCase { .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter").build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); environment = new Environment(settings); - registry = new AnalysisRegistry(null, environment); + registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry(); analysisService = registry.build(idxSettings); } diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 67b6bdadb91..4467ea63082 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -45,9 +44,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.analysis.AnalysisRegistry; -import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.cache.query.IndexQueryCache; import org.elasticsearch.index.cache.query.DisabledQueryCache; +import org.elasticsearch.index.cache.query.IndexQueryCache; +import org.elasticsearch.index.cache.query.QueryCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -61,9 +60,9 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.query.IndicesQueriesRegistry; @@ -73,6 +72,7 @@ import org.elasticsearch.script.ScriptEngineService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.TestSearchContext; @@ -84,11 +84,12 @@ import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static java.util.Collections.emptyMap; + public class IndexModuleTests extends ESTestCase { private Index index; private Settings settings; @@ -147,7 +148,8 @@ public class IndexModuleTests extends ESTestCase { } public void testWrapperIsBound() throws IOException { - IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(indexSettings, null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.setSearcherWrapper((s) -> new Wrapper()); module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class)); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener)); @@ -165,7 +167,8 @@ public class IndexModuleTests extends ESTestCase { .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "foo_store") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); - IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(indexSettings, null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.addIndexStore("foo_store", FooStore::new); try { module.addIndexStore("foo_store", FooStore::new); @@ -188,7 +191,8 @@ public class IndexModuleTests extends ESTestCase { } }; IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); - IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(indexSettings, null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); Consumer listener = (s) -> {}; module.addIndexEventListener(eventListener); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, @@ -204,7 +208,8 @@ public class IndexModuleTests extends ESTestCase { public void testListener() throws IOException { Setting booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); Setting booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope); AtomicBoolean atomicBoolean = new AtomicBoolean(false); module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set); @@ -224,7 +229,8 @@ public class IndexModuleTests extends ESTestCase { } public void testAddIndexOperationListener() throws IOException { - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); AtomicBoolean executed = new AtomicBoolean(false); IndexingOperationListener listener = new IndexingOperationListener() { @Override @@ -254,7 +260,8 @@ public class IndexModuleTests extends ESTestCase { } public void testAddSearchOperationListener() throws IOException { - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); AtomicBoolean executed = new AtomicBoolean(false); SearchOperationListener listener = new SearchOperationListener() { @@ -289,7 +296,8 @@ public class IndexModuleTests extends ESTestCase { .put("index.similarity.my_similarity.key", "there is a key") .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() { @Override public String name() { @@ -313,7 +321,8 @@ public class IndexModuleTests extends ESTestCase { } public void testFrozen() { - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.freeze(); String msg = "Can't modify IndexModule once the index service has been created"; assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage()); @@ -331,7 +340,8 @@ public class IndexModuleTests extends ESTestCase { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); try { module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener)); @@ -346,7 +356,8 @@ public class IndexModuleTests extends ESTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); try { module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener)); @@ -359,7 +370,8 @@ public class IndexModuleTests extends ESTestCase { Settings indexSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()); expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache())); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, @@ -372,7 +384,8 @@ public class IndexModuleTests extends ESTestCase { Settings indexSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener)); assertTrue(indexService.cache().query() instanceof IndexQueryCache); @@ -384,7 +397,8 @@ public class IndexModuleTests extends ESTestCase { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment)); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, + new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener)); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java index b72996bd1a1..52fcdd4bb2e 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -29,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -41,12 +43,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class AnalysisServiceTests extends ESTestCase { - private static AnalyzerProvider analyzerProvider(final String name) { + private static AnalyzerProvider analyzerProvider(final String name) { return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer()); } @@ -58,7 +63,8 @@ public class AnalysisServiceTests extends ESTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()) + .build(idxSettings); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); @@ -68,33 +74,28 @@ public class AnalysisServiceTests extends ESTestCase { Version version = VersionUtils.randomVersion(random()); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), - Collections.singletonMap("default", analyzerProvider("default")), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + singletonMap("default", analyzerProvider("default")), emptyMap(), emptyMap(), emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); } - public void testOverrideDefaultIndexAnalyzer() { + public void testOverrideDefaultIndexAnalyzerIsUnsupported() { Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - try { - AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), - Collections.singletonMap("default_index", new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer())), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); - fail("Expected ISE"); - } catch (IllegalArgumentException e) { - // expected - assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported")); - } + AnalyzerProvider defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), + singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap())); + assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported")); } public void testBackCompatOverrideDefaultIndexAnalyzer() { - Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); + Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), + VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), - Collections.singletonMap("default_index", analyzerProvider("default_index")), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); @@ -104,17 +105,17 @@ public class AnalysisServiceTests extends ESTestCase { Version version = VersionUtils.randomVersion(random()); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), - Collections.singletonMap("default_search", analyzerProvider("default_search")), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); } public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { - Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); + Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), + VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - Map analyzers = new HashMap<>(); + Map> analyzers = new HashMap<>(); analyzers.put("default_index", analyzerProvider("default_index")); analyzers.put("default_search", analyzerProvider("default_search")); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), @@ -125,7 +126,6 @@ public class AnalysisServiceTests extends ESTestCase { } public void testConfigureCamelCaseTokenFilter() throws IOException { - // tests a filter that Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -137,7 +137,9 @@ public class AnalysisServiceTests extends ESTestCase { .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + + AnalysisService analysisService = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry() + .build(idxSettings); try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer")) { assertNotNull(custom_analyser); TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee"); @@ -176,8 +178,10 @@ public class AnalysisServiceTests extends ESTestCase { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); - AnalysisService otherAnalysisSergice = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()) + .build(idxSettings); + AnalysisService otherAnalysisSergice = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), + emptyMap()).build(idxSettings); final int numIters = randomIntBetween(5, 20); for (int i = 0; i < numIters; i++) { PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values()); @@ -196,7 +200,8 @@ public class AnalysisServiceTests extends ESTestCase { .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisRegistry(null, new Environment(settings)).build(idxSettings)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings)); assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer")); } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java index 3e33123f932..40ec2b412ff 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java @@ -23,14 +23,14 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.indices.analysis.HunspellService; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; -import java.util.Collections; + +import static java.util.Collections.emptyList; public class AnalysisTestsHelper { @@ -49,7 +49,6 @@ public class AnalysisTestsHelper { settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); } IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - Environment environment = new Environment(settings); - return new AnalysisRegistry(new HunspellService(settings, environment, Collections.emptyMap()), environment).build(idxSettings); + return new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry().build(idxSettings); } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java index 2b2c9288f17..3f2b1461ef3 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java @@ -26,6 +26,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; +import static org.elasticsearch.test.ESTestCase.createAnalysisService; + /** */ public class CharFilterTests extends ESTokenStreamTestCase { @@ -39,7 +41,7 @@ public class CharFilterTests extends ESTokenStreamTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"}); @@ -56,7 +58,7 @@ public class CharFilterTests extends ESTokenStreamTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); @@ -78,7 +80,7 @@ public class CharFilterTests extends ESTokenStreamTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"}); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java index 0e3af58dc90..0c9010b2c9b 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java @@ -31,15 +31,20 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.instanceOf; @@ -50,8 +55,13 @@ public class CompoundAnalysisTests extends ESTestCase { public void testDefaultsCompoundAnalysis() throws Exception { Settings settings = getJsonSettings(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings), - Collections.emptyMap(),Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings); + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() { + @Override + public Map> getTokenFilters() { + return singletonMap("myfilter", MyFilterTokenFilterFactory::new); + } + })); + AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings); TokenFilterFactory filterFactory = analysisService.tokenFilter("dict_dec"); MatcherAssert.assertThat(filterFactory, instanceOf(DictionaryCompoundWordTokenFilterFactory.class)); @@ -62,14 +72,20 @@ public class CompoundAnalysisTests extends ESTestCase { for (Settings settings : settingsArr) { List terms = analyze(settings, "decompoundingAnalyzer", "donaudampfschiff spargelcremesuppe"); MatcherAssert.assertThat(terms.size(), equalTo(8)); - MatcherAssert.assertThat(terms, hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe")); + MatcherAssert.assertThat(terms, + hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe")); } } private List analyze(Settings settings, String analyzerName, String text) throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings), - Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings); + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() { + @Override + public Map> getTokenFilters() { + return singletonMap("myfilter", MyFilterTokenFilterFactory::new); + } + })); + AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings); Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer(); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java index 3f5ad6fed4f..caefb1039c2 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; +import static org.elasticsearch.test.ESTestCase.createAnalysisService; import static org.hamcrest.Matchers.containsString; public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase { @@ -39,7 +40,7 @@ public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase { .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); NamedAnalyzer analyzer1 = analysisService.analyzer("single"); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java index 39641c281aa..88c5fe692d6 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; +import static org.elasticsearch.test.ESTestCase.createAnalysisService; + public class StopAnalyzerTests extends ESTokenStreamTestCase { public void testDefaultsCompoundAnalysis() throws Exception { String json = "/org/elasticsearch/index/analysis/stop.json"; @@ -36,7 +38,7 @@ public class StopAnalyzerTests extends ESTokenStreamTestCase { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); NamedAnalyzer analyzer1 = analysisService.analyzer("analyzer1"); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java index 104bd17fb33..9e4d5b27ad7 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java @@ -26,13 +26,10 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.lucene.all.AllEntries; import org.elasticsearch.common.lucene.all.AllTokenStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -67,8 +64,7 @@ public class SynonymsAnalysisTests extends ESTestCase { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); - + analysisService = createAnalysisService(idxSettings, settings); match("synonymAnalyzer", "kimchy is the dude abides", "shay is the elasticsearch man!"); match("synonymAnalyzer_file", "kimchy is the dude abides", "shay is the elasticsearch man!"); diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index fdf1411fdf8..80f1cbe46d0 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -36,9 +36,7 @@ import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -97,7 +95,7 @@ public class CodecTests extends ESTestCase { .build(); IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap()); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(settings); + AnalysisService analysisService = createAnalysisService(settings, nodeSettings); MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()); MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry, () -> null); return new CodecService(service, ESLoggerFactory.getLogger("test")); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java similarity index 87% rename from core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java rename to core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index e20e7d1a7c9..869ac622b39 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.indices.analysis; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -36,9 +36,20 @@ import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.Analysis; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.analysis.AnalysisTestsHelper; +import org.elasticsearch.index.analysis.CustomAnalyzer; +import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory; +import org.elasticsearch.index.analysis.StandardTokenizerFactory; +import org.elasticsearch.index.analysis.StopTokenFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.indices.analysis.HunspellService; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; @@ -49,9 +60,11 @@ import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; +import java.util.Map; import java.util.Set; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -72,8 +85,16 @@ public class AnalysisModuleTests extends ModuleTestCase { } public AnalysisRegistry getNewRegistry(Settings settings) { - return new AnalysisRegistry(null, new Environment(settings), - Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new), Collections.emptyMap(), Collections.emptyMap()); + try { + return new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() { + @Override + public Map> getTokenFilters() { + return singletonMap("myfilter", MyFilterTokenFilterFactory::new); + } + })).getAnalysisRegistry(); + } catch (IOException e) { + throw new RuntimeException(e); + } } private Settings loadFromClasspath(String path) throws IOException { @@ -125,7 +146,7 @@ public class AnalysisModuleTests extends ModuleTestCase { assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), analysisService2.analyzer("custom7").analyzer().getVersion()); } - private void assertTokenFilter(String name, Class clazz) throws IOException { + private void assertTokenFilter(String name, Class clazz) throws IOException { Settings settings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); @@ -148,17 +169,9 @@ public class AnalysisModuleTests extends ModuleTestCase { StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0]; assertThat(stop1.stopWords().size(), equalTo(1)); - //assertThat((Iterable) stop1.stopWords(), hasItem("test-stop".toCharArray())); analyzer = analysisService.analyzer("custom2").analyzer(); assertThat(analyzer, instanceOf(CustomAnalyzer.class)); - CustomAnalyzer custom2 = (CustomAnalyzer) analyzer; - -// HtmlStripCharFilterFactory html = (HtmlStripCharFilterFactory) custom2.charFilters()[0]; -// assertThat(html.readAheadLimit(), equalTo(HTMLStripCharFilter.DEFAULT_READ_AHEAD)); -// -// html = (HtmlStripCharFilterFactory) custom2.charFilters()[1]; -// assertThat(html.readAheadLimit(), equalTo(1024)); // verify position increment gap analyzer = analysisService.analyzer("custom6").analyzer(); @@ -248,7 +261,8 @@ public class AnalysisModuleTests extends ModuleTestCase { getAnalysisService(settings); fail("This should fail with IllegalArgumentException because the analyzers name starts with _"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\""))); + assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")) + .or(equalTo("analyzer name must not start with '_'. got \"_invalidName\""))); } } @@ -289,13 +303,18 @@ public class AnalysisModuleTests extends ModuleTestCase { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); Environment environment = new Environment(settings); - AnalysisModule module = new AnalysisModule(environment); InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff"); InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic"); + Dictionary dictionary; try (Directory tmp = new SimpleFSDirectory(environment.tmpFile())) { - Dictionary dictionary = new Dictionary(tmp, "hunspell", aff, dic); - module.registerHunspellDictionary("foo", dictionary); - assertInstanceBinding(module, HunspellService.class, (x) -> x.getDictionary("foo") == dictionary); + dictionary = new Dictionary(tmp, "hunspell", aff, dic); } + AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() { + @Override + public Map getHunspellDictionaries() { + return singletonMap("foo", dictionary); + } + })); + assertSame(dictionary, module.getHunspellService().getDictionary("foo")); } } diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java index 86c54f2ece9..bbfeacfc590 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java @@ -19,15 +19,38 @@ package org.elasticsearch.indices.analysis; +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -public class DummyAnalysisPlugin extends Plugin { +import java.util.Map; - public void onModule(AnalysisModule module) { - module.registerAnalyzer("dummy", (a, b, c, d) -> new DummyAnalyzerProvider()); - module.registerTokenFilter("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory()); - module.registerTokenizer("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory()); - module.registerCharFilter("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory()); +import static java.util.Collections.singletonMap; + +public class DummyAnalysisPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getCharFilters() { + return singletonMap("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory()); + } + + @Override + public Map> getTokenFilters() { + return singletonMap("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory()); + } + + @Override + public Map> getTokenizers() { + return singletonMap("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory()); + } + + @Override + public Map>> getAnalyzers() { + return singletonMap("dummy", (a, b, c, d) -> new DummyAnalyzerProvider()); } } diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java similarity index 58% rename from core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java rename to core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java index 84ce2c5da50..ba4467a5630 100644 --- a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java @@ -19,36 +19,30 @@ package org.elasticsearch.indices.analyze; import org.apache.lucene.analysis.hunspell.Dictionary; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.analysis.HunspellService; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.hamcrest.Matchers; +import org.elasticsearch.test.ESTestCase; +import static java.util.Collections.emptyMap; import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_IGNORE_CASE; import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_LAZY_LOAD; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.notNullValue; -/** - * - */ -@ClusterScope(scope= Scope.TEST, numDataNodes=0) -public class HunspellServiceIT extends ESIntegTestCase { +public class HunspellServiceTests extends ESTestCase { public void testLocaleDirectoryWithNodeLevelConfig() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/conf_dir")) .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean()) .put(HUNSPELL_IGNORE_CASE.getKey(), true) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - internalCluster().startNode(settings); - Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US"); + Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"); assertThat(dictionary, notNullValue()); - assertIgnoreCase(true, dictionary); + assertTrue(dictionary.getIgnoreCase()); } public void testLocaleDirectoryWithLocaleSpecificConfig() throws Exception { @@ -58,58 +52,42 @@ public class HunspellServiceIT extends ESIntegTestCase { .put(HUNSPELL_IGNORE_CASE.getKey(), true) .put("indices.analysis.hunspell.dictionary.en_US.strict_affix_parsing", false) .put("indices.analysis.hunspell.dictionary.en_US.ignore_case", false) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - internalCluster().startNode(settings); - Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US"); + Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"); assertThat(dictionary, notNullValue()); - assertIgnoreCase(false, dictionary); - - + assertFalse(dictionary.getIgnoreCase()); // testing that dictionary specific settings override node level settings - dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US_custom"); + dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US_custom"); assertThat(dictionary, notNullValue()); - assertIgnoreCase(true, dictionary); + assertTrue(dictionary.getIgnoreCase()); } public void testDicWithNoAff() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/no_aff_conf_dir")) .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean()) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - Dictionary dictionary = null; - try { - internalCluster().startNode(settings); - dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US"); - fail("Missing affix file didn't throw an error"); - } - catch (Throwable t) { - assertNull(dictionary); - assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Missing affix file")); - } + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US")); + assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage()); + assertThat(e.getCause(), hasToString(containsString("Missing affix file"))); } public void testDicWithTwoAffs() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/two_aff_conf_dir")) .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean()) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - Dictionary dictionary = null; - try { - internalCluster().startNode(settings); - dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US"); - fail("Multiple affix files didn't throw an error"); - } catch (Throwable t) { - assertNull(dictionary); - assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Too many affix files")); - } - } - - // TODO: on next upgrade of lucene, just use new getter - private void assertIgnoreCase(boolean expected, Dictionary dictionary) throws Exception { - // assertEquals(expected, dictionary.getIgnoreCase()); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US")); + assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage()); + assertThat(e.getCause(), hasToString(containsString("Too many affix files"))); } } diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc index a1c0dad9ca1..9023c84b410 100644 --- a/docs/reference/migration/migrate_5_0/plugins.asciidoc +++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc @@ -122,3 +122,8 @@ been removed. Plugins that register custom scripts should implement `ScriptPlugin` and remove their `onModule(ScriptModule)` implementation. + +==== AnalysisPlugin + +Plugins that register custom analysis components should implement +`AnalysisPlugin` and remove their `onModule(AnalysisModule)` implementation. diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java index 0cca2c41938..8f0a75a282a 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.inject.multibindings.Multibinder; @@ -40,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -129,7 +127,7 @@ public class TemplateQueryParserTests extends ESTestCase { new IndexSettingsModule(index, settings) ).createInjector(); - AnalysisService analysisService = new AnalysisRegistry(null, environment).build(idxSettings); + AnalysisService analysisService = createAnalysisService(idxSettings, settings); SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry(); MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () -> diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java index e0e6c3c2e36..059dabb4f46 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java @@ -19,26 +19,42 @@ package org.elasticsearch.plugin.analysis.icu; +import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory; import org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory; import org.elasticsearch.index.analysis.IcuNormalizerCharFilterFactory; import org.elasticsearch.index.analysis.IcuNormalizerTokenFilterFactory; import org.elasticsearch.index.analysis.IcuTokenizerFactory; import org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -public class AnalysisICUPlugin extends Plugin { +import java.util.HashMap; +import java.util.Map; - /** - * Automatically called with the analysis module. - */ - public void onModule(AnalysisModule module) { - module.registerCharFilter("icu_normalizer", IcuNormalizerCharFilterFactory::new); - module.registerTokenizer("icu_tokenizer", IcuTokenizerFactory::new); - module.registerTokenFilter("icu_normalizer", IcuNormalizerTokenFilterFactory::new); - module.registerTokenFilter("icu_folding", IcuFoldingTokenFilterFactory::new); - module.registerTokenFilter("icu_collation", IcuCollationTokenFilterFactory::new); - module.registerTokenFilter("icu_transform", IcuTransformTokenFilterFactory::new); +import static java.util.Collections.singletonMap; + +public class AnalysisICUPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getCharFilters() { + return singletonMap("icu_normalizer", IcuNormalizerCharFilterFactory::new); + } + + @Override + public Map> getTokenFilters() { + Map> extra = new HashMap<>(); + extra.put("icu_normalizer", IcuNormalizerTokenFilterFactory::new); + extra.put("icu_folding", IcuFoldingTokenFilterFactory::new); + extra.put("icu_collation", IcuCollationTokenFilterFactory::new); + extra.put("icu_transform", IcuTransformTokenFilterFactory::new); + return extra; + } + + @Override + public Map> getTokenizers() { + return singletonMap("icu_tokenizer", IcuTokenizerFactory::new); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java index 1630d514ae3..180c4268612 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java @@ -102,6 +102,6 @@ public class IcuTokenizerFactoryTests extends ESTestCase { .build(); Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build(); - return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin()::onModule); + return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin()); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java index b399dfd34f4..9255a250f16 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java @@ -31,8 +31,7 @@ import static org.hamcrest.Matchers.instanceOf; */ public class SimpleIcuAnalysisTests extends ESTestCase { public void testDefaultsIcuAnalysis() throws IOException { - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), - Settings.EMPTY, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisICUPlugin()); TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer"); assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class)); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java index 71752196af0..62e9c9db145 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java @@ -50,7 +50,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.language", "tr") .put("index.analysis.filter.myCollator.strength", "primary") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "I WİLL USE TURKİSH CASING", "ı will use turkish casıng"); @@ -66,7 +66,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.strength", "primary") .put("index.analysis.filter.myCollator.decomposition", "canonical") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng"); @@ -82,7 +82,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.strength", "secondary") .put("index.analysis.filter.myCollator.decomposition", "no") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "TESTING", "testing"); @@ -99,7 +99,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.strength", "primary") .put("index.analysis.filter.myCollator.alternate", "shifted") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "foo-bar", "foo bar"); @@ -117,7 +117,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.alternate", "shifted") .put("index.analysis.filter.myCollator.variableTop", " ") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "foo bar", "foobar"); @@ -135,7 +135,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.language", "en") .put("index.analysis.filter.myCollator.numeric", "true") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollation(filterFactory, "foobar-9", "foobar-10", -1); @@ -152,7 +152,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.strength", "primary") .put("index.analysis.filter.myCollator.caseLevel", "true") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "résumé", "resume"); @@ -172,7 +172,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.strength", "tertiary") .put("index.analysis.filter.myCollator.caseFirst", "upper") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollation(filterFactory, "Resume", "resume", -1); @@ -200,7 +200,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { .put("index.analysis.filter.myCollator.rules", tailoredRules) .put("index.analysis.filter.myCollator.strength", "primary") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "Töne", "Toene"); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java index 96defd0e076..b82accf0cf8 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java @@ -37,7 +37,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase { Settings settings = Settings.builder() .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar"); String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि"; @@ -61,7 +61,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase { .put("index.analysis.char_filter.myNormalizerChar.name", "nfkc") .put("index.analysis.char_filter.myNormalizerChar.mode", "decompose") .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()); CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar"); String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि"; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java index e92e782a96c..c4b4db53c4a 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java @@ -19,6 +19,9 @@ package org.elasticsearch.plugin.analysis.kuromoji; +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory; import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider; import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory; @@ -28,23 +31,42 @@ import org.elasticsearch.index.analysis.KuromojiNumberFilterFactory; import org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory; import org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory; import org.elasticsearch.index.analysis.KuromojiTokenizerFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -/** - * - */ -public class AnalysisKuromojiPlugin extends Plugin { +import java.util.HashMap; +import java.util.Map; - public void onModule(AnalysisModule module) { - module.registerCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new); - module.registerAnalyzer("kuromoji", KuromojiAnalyzerProvider::new); - module.registerTokenizer("kuromoji_tokenizer", KuromojiTokenizerFactory::new); - module.registerTokenFilter("kuromoji_baseform", KuromojiBaseFormFilterFactory::new); - module.registerTokenFilter("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new); - module.registerTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory::new); - module.registerTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new); - module.registerTokenFilter("ja_stop", JapaneseStopTokenFilterFactory::new); - module.registerTokenFilter("kuromoji_number", KuromojiNumberFilterFactory::new); +import static java.util.Collections.singletonMap; + +public class AnalysisKuromojiPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getCharFilters() { + return singletonMap("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new); + } + + @Override + public Map> getTokenFilters() { + Map> extra = new HashMap<>(); + extra.put("kuromoji_baseform", KuromojiBaseFormFilterFactory::new); + extra.put("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new); + extra.put("kuromoji_readingform", KuromojiReadingFormFilterFactory::new); + extra.put("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new); + extra.put("ja_stop", JapaneseStopTokenFilterFactory::new); + extra.put("kuromoji_number", KuromojiNumberFilterFactory::new); + return extra; + } + + @Override + public Map> getTokenizers() { + return singletonMap("kuromoji_tokenizer", KuromojiTokenizerFactory::new); + } + + @Override + public Map>> getAnalyzers() { + return singletonMap("kuromoji", KuromojiAnalyzerProvider::new); } } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java index 540e11250d0..53196ac7462 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java @@ -198,7 +198,7 @@ public class KuromojiAnalysisTests extends ESTestCase { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build(); - return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin()::onModule); + return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin()); } public static void assertSimpleTSOutput(TokenStream stream, diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java index 3be97b444db..c7355e976ce 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java @@ -20,13 +20,19 @@ package org.elasticsearch.plugin.analysis; import org.elasticsearch.index.analysis.PhoneticTokenFilterFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -public class AnalysisPhoneticPlugin extends Plugin { +import java.util.Map; - public void onModule(AnalysisModule module) { - module.registerTokenFilter("phonetic", PhoneticTokenFilterFactory::new); +import static java.util.Collections.singletonMap; + +public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getTokenFilters() { + return singletonMap("phonetic", PhoneticTokenFilterFactory::new); } } diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java index 18e49fa6e51..3dcfadce781 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java @@ -39,8 +39,7 @@ public class SimplePhoneticAnalysisTests extends ESTestCase { Settings settings = Settings.builder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, - new AnalysisPhoneticPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic"); MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class)); } diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java index 05c779bb5a9..b11a157c149 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java @@ -19,20 +19,40 @@ package org.elasticsearch.plugin.analysis.smartcn; +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider; import org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory; import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -public class AnalysisSmartChinesePlugin extends Plugin { +import java.util.HashMap; +import java.util.Map; - public void onModule(AnalysisModule module) { - module.registerAnalyzer("smartcn", SmartChineseAnalyzerProvider::new); - module.registerTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new); - // This is an alias to "smartcn_tokenizer"; it's here for backwards compat - module.registerTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new); - // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer" - module.registerTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory::new); +import static java.util.Collections.singletonMap; + +public class AnalysisSmartChinesePlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getTokenFilters() { + // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer" + return singletonMap("smartcn_word", SmartChineseNoOpTokenFilterFactory::new); + } + + @Override + public Map> getTokenizers() { + Map> extra = new HashMap<>(); + extra.put("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new); + // This is an alias to "smartcn_tokenizer"; it's here for backwards compat + extra.put("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new); + return extra; + } + + @Override + public Map>> getAnalyzers() { + return singletonMap("smartcn", SmartChineseAnalyzerProvider::new); } } diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java index 0fcc42643d4..08aebdee2bb 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java @@ -29,11 +29,10 @@ import java.io.IOException; import static org.hamcrest.Matchers.instanceOf; -/** - */ public class SimpleSmartChineseAnalysisTests extends ESTestCase { public void testDefaultsIcuAnalysis() throws IOException { - final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisSmartChinesePlugin()::onModule); + final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, + new AnalysisSmartChinesePlugin()); TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer"); MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class)); } diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java index 8549795f4b6..98dd9634fb9 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java @@ -19,15 +19,27 @@ package org.elasticsearch.plugin.analysis.stempel; +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.pl.PolishAnalyzerProvider; import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory; -import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -public class AnalysisStempelPlugin extends Plugin { +import java.util.Map; - public void onModule(AnalysisModule module) { - module.registerAnalyzer("polish", PolishAnalyzerProvider::new); - module.registerTokenFilter("polish_stem", PolishStemTokenFilterFactory::new); +import static java.util.Collections.singletonMap; + +public class AnalysisStempelPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getTokenFilters() { + return singletonMap("polish_stem", PolishStemTokenFilterFactory::new); + } + + @Override + public Map>> getAnalyzers() { + return singletonMap("polish", PolishAnalyzerProvider::new); } } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java index 9bfcc2c2f3f..4f7ee642ebd 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java @@ -36,7 +36,8 @@ import static org.hamcrest.Matchers.instanceOf; */ public class PolishAnalysisTests extends ESTestCase { public void testDefaultsPolishAnalysis() throws IOException { - final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin()::onModule); + final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, + new AnalysisStempelPlugin()); TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem"); MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class)); diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java index 9458b6920c4..3fc12ccdfed 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java @@ -49,7 +49,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase { Settings settings = Settings.builder() .put("index.analysis.filter.myStemmer.type", "polish_stem") .build(); - AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin()); TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer"); @@ -65,8 +65,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase { } private void testAnalyzer(String source, String... expected_terms) throws IOException { - AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, - new AnalysisStempelPlugin()::onModule); + AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin()); Analyzer analyzer = analysisService.analyzer("polish").analyzer(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index fe434246035..1ac0b53cbab 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collections; +import static org.elasticsearch.test.ESTestCase.createAnalysisService; + public class MapperTestUtils { @@ -54,7 +56,7 @@ public class MapperTestUtils { Settings finalSettings = settingsBuilder.build(); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", finalSettings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings); + AnalysisService analysisService = createAnalysisService(indexSettings, finalSettings); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); return new MapperService(indexSettings, analysisService, diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 5eeaa635305..95fadc99e5c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -71,7 +71,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -86,6 +85,7 @@ import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -118,6 +118,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; +import static java.util.Collections.emptyList; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; @@ -917,7 +918,8 @@ public abstract class AbstractQueryTestCase> injector = modulesBuilder.createInjector(); IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class); idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), emptyList()); + AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings); scriptService = scriptModule.getScriptService(); similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 76616fbdb44..50d6dc88aaf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -28,6 +28,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; + import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; @@ -44,7 +45,6 @@ import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -54,9 +54,11 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -768,29 +770,34 @@ public abstract class ESTestCase extends LuceneTestCase { } /** - * Creates an AnalysisService to test analysis factories and analyzers. + * Creates an AnalysisService with all the default analyzers configured. */ - @SafeVarargs - public static AnalysisService createAnalysisService(Index index, Settings settings, Consumer... moduleConsumers) throws IOException { + public static AnalysisService createAnalysisService(Index index, Settings settings, AnalysisPlugin... analysisPlugins) + throws IOException { Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); - return createAnalysisService(index, nodeSettings, settings, moduleConsumers); + return createAnalysisService(index, nodeSettings, settings, analysisPlugins); } /** - * Creates an AnalysisService to test analysis factories and analyzers. + * Creates an AnalysisService with all the default analyzers configured. */ - @SafeVarargs - public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings, Consumer... moduleConsumers) throws IOException { + public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings, + AnalysisPlugin... analysisPlugins) throws IOException { Settings indexSettings = Settings.builder().put(settings) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + return createAnalysisService(IndexSettingsModule.newIndexSettings(index, indexSettings), nodeSettings, analysisPlugins); + } + + /** + * Creates an AnalysisService with all the default analyzers configured. + */ + public static AnalysisService createAnalysisService(IndexSettings indexSettings, Settings nodeSettings, + AnalysisPlugin... analysisPlugins) throws IOException { Environment env = new Environment(nodeSettings); - AnalysisModule analysisModule = new AnalysisModule(env); - for (Consumer consumer : moduleConsumers) { - consumer.accept(analysisModule); - } - SettingsModule settingsModule = new SettingsModule(nodeSettings, InternalSettingsPlugin.VERSION_CREATED); - final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings)); + AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins)); + final AnalysisService analysisService = analysisModule.getAnalysisRegistry() + .build(indexSettings); return analysisService; }