honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch analysis from push to pull

Browse Source 
Instead of plugins calling `registerTokenizer` to extend the analyzer
they now instead have to implement `AnalysisPlugin` and override
`getTokenizer`. This lines up extending plugins in with extending
scripts. This allows `AnalysisModule` to construct the `AnalysisRegistry`
immediately as part of its constructor which makes testing anslysis
much simpler.

This also moves the default analysis configuration into `AnalysisModule`
which is how search is setup.

Like `ScriptModule`, `AnalysisModule` no longer extends `AbstractModule`.
Instead it is only responsible for building `AnslysisRegistry`. We still
bind `AnalysisRegistry` but we only do so in `Node`. This is means it
is available at module construction time so we slowly remove the need to
bind it in guice.
This commit is contained in:
Nik Everett 2016-06-24 16:37:23 -04:00
parent c79e27180e
commit 71b95fb63c
43 changed files with 844 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -477,8 +477,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndexingMemoryController.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesService.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesWarmer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]AnalysisModule.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]HunspellService.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltCacheFactory.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltTokenFilters.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]breaker[/\\]HierarchyCircuitBreakerService.java" checks="LineLength" />
@ -879,9 +877,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicySettingsTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLogTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisModuleTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisServiceTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CompoundAnalysisTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]NGramTokenizerFactoryTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PatternCaptureTokenFilterTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltAnalyzerProviderFactoryTests.java" checks="LineLength" />
@ -1186,8 +1181,6 @@
<suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]JapaneseStopTokenFilterFactory.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]KuromojiAnalysisTests.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-phonetic[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PhoneticTokenFilterFactory.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-smartcn[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]SimpleSmartChineseAnalysisTests.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-stempel[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PolishAnalysisTests.java" checks="LineLength" />
<suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
<suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
<suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />

203
core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
View File

@ -23,14 +23,11 @@ import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.HunspellService;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
import org.elasticsearch.indices.analysis.PreBuiltCharFilters;
import org.elasticsearch.indices.analysis.PreBuiltTokenFilters;
@ -45,48 +42,32 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static java.util.Collections.unmodifiableMap;
/**
* An internal registry for tokenizer, token filter, char filter and analyzer.
* This class exists per node and allows to create per-index {@link AnalysisService} via {@link #build(IndexSettings)}
*/
public final class AnalysisRegistry implements Closeable {
private final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters;
private final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters;
private final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers;
private final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers;
private final PrebuiltAnalysis prebuiltAnalysis = new PrebuiltAnalysis();
private final Map<String, Analyzer> cachedAnalyzer = new ConcurrentHashMap<>();
private final PrebuiltAnalysis prebuiltAnalysis;
private final HunspellService hunspellService;
private final Environment environment;
private final Map<String, AnalysisProvider<CharFilterFactory>> charFilters;
private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters;
private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers;
private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers;
public AnalysisRegistry(HunspellService hunspellService, Environment environment) {
this(hunspellService, environment, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
}
public AnalysisRegistry(HunspellService hunspellService, Environment environment,
Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters,
Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters,
Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers,
Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers) {
prebuiltAnalysis = new PrebuiltAnalysis();
this.hunspellService = hunspellService;
public AnalysisRegistry(Environment environment,
Map<String, AnalysisProvider<CharFilterFactory>> charFilters,
Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters,
Map<String, AnalysisProvider<TokenizerFactory>> tokenizers,
Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers) {
this.environment = environment;
final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilterBuilder = new HashMap<>(charFilters);
final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilterBuilder = new HashMap<>(tokenFilters);
final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizerBuilder = new HashMap<>(tokenizers);
final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzerBuilder= new HashMap<>(analyzers);
registerBuiltInAnalyzer(analyzerBuilder);
registerBuiltInCharFilter(charFilterBuilder);
registerBuiltInTokenizer(tokenizerBuilder);
registerBuiltInTokenFilters(tokenFilterBuilder);
this.tokenFilters = Collections.unmodifiableMap(tokenFilterBuilder);
this.tokenizers = Collections.unmodifiableMap(tokenizerBuilder);
this.charFilters = Collections.unmodifiableMap(charFilterBuilder);
this.analyzers = Collections.unmodifiableMap(analyzerBuilder);
}
public HunspellService getHunspellService() {
return hunspellService;
this.charFilters = unmodifiableMap(charFilters);
this.tokenFilters = unmodifiableMap(tokenFilters);
this.tokenizers = unmodifiableMap(tokenizers);
this.analyzers = unmodifiableMap(analyzers);
}
/**
@ -114,9 +95,9 @@ public final class AnalysisRegistry implements Closeable {
* Returns a registered {@link Analyzer} provider by name or <code>null</code> if the analyzer was not registered
*/
public Analyzer getAnalyzer(String analyzer) throws IOException {
AnalysisModule.AnalysisProvider<AnalyzerProvider> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer);
AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer);
if (analyzerProvider == null) {
AnalysisModule.AnalysisProvider<AnalyzerProvider> provider = analyzers.get(analyzer);
AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> provider = analyzers.get(analyzer);
return provider == null ? null : cachedAnalyzer.computeIfAbsent(analyzer, (key) -> {
try {
return provider.get(environment, key).get();
@ -157,7 +138,8 @@ public final class AnalysisRegistry implements Closeable {
*/
tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, tokenizerFactories, name, settings)));
final Map<String, TokenFilterFactory> tokenFilterFactories = buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
final Map<String, AnalyzerProvider> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
final Map<String, AnalyzerProvider<?>> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings,
analyzers, prebuiltAnalysis.analyzerProviderFactories);
return new AnalysisService(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
}
@ -175,140 +157,9 @@ public final class AnalysisRegistry implements Closeable {
};
}
private void registerBuiltInCharFilter(Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters) {
charFilters.put("html_strip", HtmlStripCharFilterFactory::new);
charFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
charFilters.put("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
}
private void registerBuiltInTokenizer(Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers) {
tokenizers.put("standard", StandardTokenizerFactory::new);
tokenizers.put("uax_url_email", UAX29URLEmailTokenizerFactory::new);
tokenizers.put("path_hierarchy", PathHierarchyTokenizerFactory::new);
tokenizers.put("PathHierarchy", PathHierarchyTokenizerFactory::new);
tokenizers.put("keyword", KeywordTokenizerFactory::new);
tokenizers.put("letter", LetterTokenizerFactory::new);
tokenizers.put("lowercase", LowerCaseTokenizerFactory::new);
tokenizers.put("whitespace", WhitespaceTokenizerFactory::new);
tokenizers.put("nGram", NGramTokenizerFactory::new);
tokenizers.put("ngram", NGramTokenizerFactory::new);
tokenizers.put("edgeNGram", EdgeNGramTokenizerFactory::new);
tokenizers.put("edge_ngram", EdgeNGramTokenizerFactory::new);
tokenizers.put("pattern", PatternTokenizerFactory::new);
tokenizers.put("classic", ClassicTokenizerFactory::new);
tokenizers.put("thai", ThaiTokenizerFactory::new);
}
private void registerBuiltInTokenFilters(Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters) {
tokenFilters.put("stop", StopTokenFilterFactory::new);
tokenFilters.put("reverse", ReverseTokenFilterFactory::new);
tokenFilters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new);
tokenFilters.put("length", LengthTokenFilterFactory::new);
tokenFilters.put("lowercase", LowerCaseTokenFilterFactory::new);
tokenFilters.put("uppercase", UpperCaseTokenFilterFactory::new);
tokenFilters.put("porter_stem", PorterStemTokenFilterFactory::new);
tokenFilters.put("kstem", KStemTokenFilterFactory::new);
tokenFilters.put("standard", StandardTokenFilterFactory::new);
tokenFilters.put("nGram", NGramTokenFilterFactory::new);
tokenFilters.put("ngram", NGramTokenFilterFactory::new);
tokenFilters.put("edgeNGram", EdgeNGramTokenFilterFactory::new);
tokenFilters.put("edge_ngram", EdgeNGramTokenFilterFactory::new);
tokenFilters.put("shingle", ShingleTokenFilterFactory::new);
tokenFilters.put("unique", UniqueTokenFilterFactory::new);
tokenFilters.put("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new));
tokenFilters.put("trim", TrimTokenFilterFactory::new);
tokenFilters.put("limit", LimitTokenCountFilterFactory::new);
tokenFilters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
tokenFilters.put("snowball", SnowballTokenFilterFactory::new);
tokenFilters.put("stemmer", StemmerTokenFilterFactory::new);
tokenFilters.put("word_delimiter", WordDelimiterTokenFilterFactory::new);
tokenFilters.put("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
tokenFilters.put("elision", ElisionTokenFilterFactory::new);
tokenFilters.put("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
tokenFilters.put("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
tokenFilters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
tokenFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
tokenFilters.put("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
tokenFilters.put("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
tokenFilters.put("arabic_stem", ArabicStemTokenFilterFactory::new);
tokenFilters.put("brazilian_stem", BrazilianStemTokenFilterFactory::new);
tokenFilters.put("czech_stem", CzechStemTokenFilterFactory::new);
tokenFilters.put("dutch_stem", DutchStemTokenFilterFactory::new);
tokenFilters.put("french_stem", FrenchStemTokenFilterFactory::new);
tokenFilters.put("german_stem", GermanStemTokenFilterFactory::new);
tokenFilters.put("russian_stem", RussianStemTokenFilterFactory::new);
tokenFilters.put("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
tokenFilters.put("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
tokenFilters.put("arabic_normalization", ArabicNormalizationFilterFactory::new);
tokenFilters.put("german_normalization", GermanNormalizationFilterFactory::new);
tokenFilters.put("hindi_normalization", HindiNormalizationFilterFactory::new);
tokenFilters.put("indic_normalization", IndicNormalizationFilterFactory::new);
tokenFilters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
tokenFilters.put("persian_normalization", PersianNormalizationFilterFactory::new);
tokenFilters.put("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
tokenFilters.put("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
tokenFilters.put("serbian_normalization", SerbianNormalizationFilterFactory::new);
if (hunspellService != null) {
tokenFilters.put("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
}
tokenFilters.put("cjk_bigram", CJKBigramFilterFactory::new);
tokenFilters.put("cjk_width", CJKWidthFilterFactory::new);
tokenFilters.put("apostrophe", ApostropheFilterFactory::new);
tokenFilters.put("classic", ClassicFilterFactory::new);
tokenFilters.put("decimal_digit", DecimalDigitFilterFactory::new);
tokenFilters.put("fingerprint", FingerprintTokenFilterFactory::new);
}
private void registerBuiltInAnalyzer(Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers) {
analyzers.put("default", StandardAnalyzerProvider::new);
analyzers.put("standard", StandardAnalyzerProvider::new);
analyzers.put("standard_html_strip", StandardHtmlStripAnalyzerProvider::new);
analyzers.put("simple", SimpleAnalyzerProvider::new);
analyzers.put("stop", StopAnalyzerProvider::new);
analyzers.put("whitespace", WhitespaceAnalyzerProvider::new);
analyzers.put("keyword", KeywordAnalyzerProvider::new);
analyzers.put("pattern", PatternAnalyzerProvider::new);
analyzers.put("snowball", SnowballAnalyzerProvider::new);
analyzers.put("arabic", ArabicAnalyzerProvider::new);
analyzers.put("armenian", ArmenianAnalyzerProvider::new);
analyzers.put("basque", BasqueAnalyzerProvider::new);
analyzers.put("brazilian", BrazilianAnalyzerProvider::new);
analyzers.put("bulgarian", BulgarianAnalyzerProvider::new);
analyzers.put("catalan", CatalanAnalyzerProvider::new);
analyzers.put("chinese", ChineseAnalyzerProvider::new);
analyzers.put("cjk", CjkAnalyzerProvider::new);
analyzers.put("czech", CzechAnalyzerProvider::new);
analyzers.put("danish", DanishAnalyzerProvider::new);
analyzers.put("dutch", DutchAnalyzerProvider::new);
analyzers.put("english", EnglishAnalyzerProvider::new);
analyzers.put("finnish", FinnishAnalyzerProvider::new);
analyzers.put("french", FrenchAnalyzerProvider::new);
analyzers.put("galician", GalicianAnalyzerProvider::new);
analyzers.put("german", GermanAnalyzerProvider::new);
analyzers.put("greek", GreekAnalyzerProvider::new);
analyzers.put("hindi", HindiAnalyzerProvider::new);
analyzers.put("hungarian", HungarianAnalyzerProvider::new);
analyzers.put("indonesian", IndonesianAnalyzerProvider::new);
analyzers.put("irish", IrishAnalyzerProvider::new);
analyzers.put("italian", ItalianAnalyzerProvider::new);
analyzers.put("latvian", LatvianAnalyzerProvider::new);
analyzers.put("lithuanian", LithuanianAnalyzerProvider::new);
analyzers.put("norwegian", NorwegianAnalyzerProvider::new);
analyzers.put("persian", PersianAnalyzerProvider::new);
analyzers.put("portuguese", PortugueseAnalyzerProvider::new);
analyzers.put("romanian", RomanianAnalyzerProvider::new);
analyzers.put("russian", RussianAnalyzerProvider::new);
analyzers.put("sorani", SoraniAnalyzerProvider::new);
analyzers.put("spanish", SpanishAnalyzerProvider::new);
analyzers.put("swedish", SwedishAnalyzerProvider::new);
analyzers.put("turkish", TurkishAnalyzerProvider::new);
analyzers.put("thai", ThaiAnalyzerProvider::new);
analyzers.put("fingerprint", FingerprintAnalyzerProvider::new);
}
private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap, Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance) throws IOException {
private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap,
Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance)
throws IOException {
Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build();
Map<String, T> factories = new HashMap<>();
for (Map.Entry<String, Settings> entry : settingsMap.entrySet()) {
@ -383,7 +234,7 @@ public final class AnalysisRegistry implements Closeable {
private static class PrebuiltAnalysis implements Closeable {
final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzerProviderFactories;
final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<?>>> analyzerProviderFactories;
final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizerFactories;
final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilterFactories;
final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilterFactories;
@ -446,7 +297,7 @@ public final class AnalysisRegistry implements Closeable {
return tokenizerFactories.get(name);
}
public AnalysisModule.AnalysisProvider<AnalyzerProvider> getAnalyzerProvider(String name) {
public AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> getAnalyzerProvider(String name) {
return analyzerProviderFactories.get(name);
}

6
core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
View File

@ -48,7 +48,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
private final NamedAnalyzer defaultSearchQuoteAnalyzer;
public AnalysisService(IndexSettings indexSettings,
Map<String, AnalyzerProvider> analyzerProviders,
Map<String, AnalyzerProvider<?>> analyzerProviders,
Map<String, TokenizerFactory> tokenizerFactoryFactories,
Map<String, CharFilterFactory> charFilterFactoryFactories,
Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
@ -69,8 +69,8 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
}
Map<String, NamedAnalyzer> analyzers = new HashMap<>();
for (Map.Entry<String, AnalyzerProvider> entry : analyzerProviders.entrySet()) {
AnalyzerProvider analyzerFactory = entry.getValue();
for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
AnalyzerProvider<?> analyzerFactory = entry.getValue();
String name = entry.getKey();
/*
* Lucene defaults positionIncrementGap to 0 in all analyzers but

7
core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
View File

@ -32,7 +32,7 @@ import java.io.IOException;
/**
*
*/
public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider<AnalyzerProvider> {
public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> {
private final PreBuiltAnalyzerProvider analyzerProvider;
@ -40,7 +40,7 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP
analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer);
}
public AnalyzerProvider create(String name, Settings settings) {
public AnalyzerProvider<?> create(String name, Settings settings) {
Version indexVersion = Version.indexCreated(settings);
if (!Version.CURRENT.equals(indexVersion)) {
PreBuiltAnalyzers preBuiltAnalyzers = PreBuiltAnalyzers.getOrDefault(name, null);
@ -54,7 +54,8 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP
}
@Override
public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
public AnalyzerProvider<?> get(IndexSettings indexSettings, Environment environment, String name, Settings settings)
throws IOException {
return create(name, settings);
}

455
core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
View File

@ -19,49 +19,141 @@
package org.elasticsearch.indices.analysis;
import org.apache.lucene.analysis.hunspell.Dictionary;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.ApostropheFilterFactory;
import org.elasticsearch.index.analysis.ArabicAnalyzerProvider;
import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory;
import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory;
import org.elasticsearch.index.analysis.ArmenianAnalyzerProvider;
import org.elasticsearch.index.analysis.BasqueAnalyzerProvider;
import org.elasticsearch.index.analysis.BrazilianAnalyzerProvider;
import org.elasticsearch.index.analysis.BrazilianStemTokenFilterFactory;
import org.elasticsearch.index.analysis.BulgarianAnalyzerProvider;
import org.elasticsearch.index.analysis.CJKBigramFilterFactory;
import org.elasticsearch.index.analysis.CJKWidthFilterFactory;
import org.elasticsearch.index.analysis.CatalanAnalyzerProvider;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.ChineseAnalyzerProvider;
import org.elasticsearch.index.analysis.CjkAnalyzerProvider;
import org.elasticsearch.index.analysis.ClassicFilterFactory;
import org.elasticsearch.index.analysis.ClassicTokenizerFactory;
import org.elasticsearch.index.analysis.CommonGramsTokenFilterFactory;
import org.elasticsearch.index.analysis.CzechAnalyzerProvider;
import org.elasticsearch.index.analysis.CzechStemTokenFilterFactory;
import org.elasticsearch.index.analysis.DanishAnalyzerProvider;
import org.elasticsearch.index.analysis.DecimalDigitFilterFactory;
import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
import org.elasticsearch.index.analysis.DutchAnalyzerProvider;
import org.elasticsearch.index.analysis.DutchStemTokenFilterFactory;
import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
import org.elasticsearch.index.analysis.ElisionTokenFilterFactory;
import org.elasticsearch.index.analysis.EnglishAnalyzerProvider;
import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider;
import org.elasticsearch.index.analysis.FingerprintTokenFilterFactory;
import org.elasticsearch.index.analysis.FinnishAnalyzerProvider;
import org.elasticsearch.index.analysis.FrenchAnalyzerProvider;
import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory;
import org.elasticsearch.index.analysis.GalicianAnalyzerProvider;
import org.elasticsearch.index.analysis.GermanAnalyzerProvider;
import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
import org.elasticsearch.index.analysis.GreekAnalyzerProvider;
import org.elasticsearch.index.analysis.HindiAnalyzerProvider;
import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
import org.elasticsearch.index.analysis.HtmlStripCharFilterFactory;
import org.elasticsearch.index.analysis.HungarianAnalyzerProvider;
import org.elasticsearch.index.analysis.HunspellTokenFilterFactory;
import org.elasticsearch.index.analysis.IndicNormalizationFilterFactory;
import org.elasticsearch.index.analysis.IndonesianAnalyzerProvider;
import org.elasticsearch.index.analysis.IrishAnalyzerProvider;
import org.elasticsearch.index.analysis.ItalianAnalyzerProvider;
import org.elasticsearch.index.analysis.KStemTokenFilterFactory;
import org.elasticsearch.index.analysis.KeepTypesFilterFactory;
import org.elasticsearch.index.analysis.KeepWordFilterFactory;
import org.elasticsearch.index.analysis.KeywordAnalyzerProvider;
import org.elasticsearch.index.analysis.KeywordMarkerTokenFilterFactory;
import org.elasticsearch.index.analysis.KeywordTokenizerFactory;
import org.elasticsearch.index.analysis.LatvianAnalyzerProvider;
import org.elasticsearch.index.analysis.LengthTokenFilterFactory;
import org.elasticsearch.index.analysis.LetterTokenizerFactory;
import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory;
import org.elasticsearch.index.analysis.LithuanianAnalyzerProvider;
import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory;
import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory;
import org.elasticsearch.index.analysis.MappingCharFilterFactory;
import org.elasticsearch.index.analysis.NGramTokenFilterFactory;
import org.elasticsearch.index.analysis.NGramTokenizerFactory;
import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider;
import org.elasticsearch.index.analysis.PathHierarchyTokenizerFactory;
import org.elasticsearch.index.analysis.PatternAnalyzerProvider;
import org.elasticsearch.index.analysis.PatternCaptureGroupTokenFilterFactory;
import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
import org.elasticsearch.index.analysis.PatternReplaceTokenFilterFactory;
import org.elasticsearch.index.analysis.PatternTokenizerFactory;
import org.elasticsearch.index.analysis.PersianAnalyzerProvider;
import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory;
import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider;
import org.elasticsearch.index.analysis.ReverseTokenFilterFactory;
import org.elasticsearch.index.analysis.RomanianAnalyzerProvider;
import org.elasticsearch.index.analysis.RussianAnalyzerProvider;
import org.elasticsearch.index.analysis.RussianStemTokenFilterFactory;
import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory;
import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory;
import org.elasticsearch.index.analysis.SerbianNormalizationFilterFactory;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.analysis.SimpleAnalyzerProvider;
import org.elasticsearch.index.analysis.SnowballAnalyzerProvider;
import org.elasticsearch.index.analysis.SnowballTokenFilterFactory;
import org.elasticsearch.index.analysis.SoraniAnalyzerProvider;
import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory;
import org.elasticsearch.index.analysis.SpanishAnalyzerProvider;
import org.elasticsearch.index.analysis.StandardAnalyzerProvider;
import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzerProvider;
import org.elasticsearch.index.analysis.StandardTokenFilterFactory;
import org.elasticsearch.index.analysis.StandardTokenizerFactory;
import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory;
import org.elasticsearch.index.analysis.StemmerTokenFilterFactory;
import org.elasticsearch.index.analysis.StopAnalyzerProvider;
import org.elasticsearch.index.analysis.StopTokenFilterFactory;
import org.elasticsearch.index.analysis.SwedishAnalyzerProvider;
import org.elasticsearch.index.analysis.ThaiAnalyzerProvider;
import org.elasticsearch.index.analysis.ThaiTokenizerFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.index.analysis.TrimTokenFilterFactory;
import org.elasticsearch.index.analysis.TruncateTokenFilterFactory;
import org.elasticsearch.index.analysis.TurkishAnalyzerProvider;
import org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory;
import org.elasticsearch.index.analysis.UniqueTokenFilterFactory;
import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory;
import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider;
import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
import org.elasticsearch.plugins.AnalysisPlugin;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import static java.util.Objects.requireNonNull;
/**
* The AnalysisModule is the main extension point for node and index level analysis components. The lucene classes
* {@link org.apache.lucene.analysis.Analyzer}, {@link org.apache.lucene.analysis.TokenFilter}, {@link org.apache.lucene.analysis.Tokenizer}
* and {@link org.apache.lucene.analysis.CharFilter} can be extended in plugins and registered on node startup when the analysis module
* gets loaded. Since elasticsearch needs to create multiple instances for different configurations dedicated factories need to be provided for
* each of the components:
* <ul>
* <li> {@link org.apache.lucene.analysis.Analyzer} can be exposed via {@link AnalyzerProvider} and registered on {@link #registerAnalyzer(String, AnalysisProvider)}</li>
* <li> {@link org.apache.lucene.analysis.TokenFilter} can be exposed via {@link TokenFilterFactory} and registered on {@link #registerTokenFilter(String, AnalysisProvider)}</li>
* <li> {@link org.apache.lucene.analysis.Tokenizer} can be exposed via {@link TokenizerFactory} and registered on {@link #registerTokenizer(String, AnalysisProvider)}</li>
* <li> {@link org.apache.lucene.analysis.CharFilter} can be exposed via {@link CharFilterFactory} and registered on {@link #registerCharFilter(String, AnalysisProvider)}</li>
* </ul>
*
* The {@link org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider} is only a functional interface that allows to register factory constructors directly like the plugin example below:
* <pre>
* public class MyAnalysisPlugin extends Plugin {
* public void onModule(AnalysisModule module) {
* module.registerAnalyzer("my-analyzer-name", MyAnalyzer::new);
* }
* }
* </pre>
* Sets up {@link AnalysisRegistry}.
*/
public final class AnalysisModule extends AbstractModule {
public final class AnalysisModule {
static {
Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@ -71,108 +163,195 @@ public final class AnalysisModule extends AbstractModule {
NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY);
}
private static final IndexSettings NA_INDEX_SETTINGS;
private final Environment environment;
private final Map<String, AnalysisProvider<CharFilterFactory>> charFilters = new HashMap<>();
private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>();
private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers = new HashMap<>();
private final Map<String, AnalysisProvider<AnalyzerProvider>> analyzers = new HashMap<>();
private final Map<String, org.apache.lucene.analysis.hunspell.Dictionary> knownDictionaries = new HashMap<>();
/**
* Creates a new AnalysisModule
*/
public AnalysisModule(Environment environment) {
this.environment = environment;
private final HunspellService hunspellService;
private final AnalysisRegistry analysisRegistry;
public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) throws IOException {
NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = setupCharFilters(plugins);
NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = setupHunspellDictionaries(plugins);
hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.registry);
NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
analysisRegistry = new AnalysisRegistry(environment, charFilters.registry, tokenFilters.registry,
tokenizers.registry, analyzers.registry);
}
HunspellService getHunspellService() {
return hunspellService;
}
public AnalysisRegistry getAnalysisRegistry() {
return analysisRegistry;
}
private NamedRegistry<AnalysisProvider<CharFilterFactory>> setupCharFilters(List<AnalysisPlugin> plugins) {
NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = new NamedRegistry<>("char_filter");
charFilters.register("html_strip", HtmlStripCharFilterFactory::new);
charFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
charFilters.register("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
charFilters.registerPlugins(plugins, AnalysisPlugin::getCharFilters);
return charFilters;
}
public NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> setupHunspellDictionaries(List<AnalysisPlugin> plugins) {
NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = new NamedRegistry<>("dictionary");
hunspellDictionaries.registerPlugins(plugins, AnalysisPlugin::getHunspellDictionaries);
return hunspellDictionaries;
}
private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(List<AnalysisPlugin> plugins,
HunspellService hunspellService) {
NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = new NamedRegistry<>("token_filter");
tokenFilters.register("stop", StopTokenFilterFactory::new);
tokenFilters.register("reverse", ReverseTokenFilterFactory::new);
tokenFilters.register("asciifolding", ASCIIFoldingTokenFilterFactory::new);
tokenFilters.register("length", LengthTokenFilterFactory::new);
tokenFilters.register("lowercase", LowerCaseTokenFilterFactory::new);
tokenFilters.register("uppercase", UpperCaseTokenFilterFactory::new);
tokenFilters.register("porter_stem", PorterStemTokenFilterFactory::new);
tokenFilters.register("kstem", KStemTokenFilterFactory::new);
tokenFilters.register("standard", StandardTokenFilterFactory::new);
tokenFilters.register("nGram", NGramTokenFilterFactory::new);
tokenFilters.register("ngram", NGramTokenFilterFactory::new);
tokenFilters.register("edgeNGram", EdgeNGramTokenFilterFactory::new);
tokenFilters.register("edge_ngram", EdgeNGramTokenFilterFactory::new);
tokenFilters.register("shingle", ShingleTokenFilterFactory::new);
tokenFilters.register("unique", UniqueTokenFilterFactory::new);
tokenFilters.register("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new));
tokenFilters.register("trim", TrimTokenFilterFactory::new);
tokenFilters.register("limit", LimitTokenCountFilterFactory::new);
tokenFilters.register("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
tokenFilters.register("snowball", SnowballTokenFilterFactory::new);
tokenFilters.register("stemmer", StemmerTokenFilterFactory::new);
tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new);
tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
tokenFilters.register("elision", ElisionTokenFilterFactory::new);
tokenFilters.register("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
tokenFilters.register("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
tokenFilters.register("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
tokenFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
tokenFilters.register("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
tokenFilters.register("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
tokenFilters.register("arabic_stem", ArabicStemTokenFilterFactory::new);
tokenFilters.register("brazilian_stem", BrazilianStemTokenFilterFactory::new);
tokenFilters.register("czech_stem", CzechStemTokenFilterFactory::new);
tokenFilters.register("dutch_stem", DutchStemTokenFilterFactory::new);
tokenFilters.register("french_stem", FrenchStemTokenFilterFactory::new);
tokenFilters.register("german_stem", GermanStemTokenFilterFactory::new);
tokenFilters.register("russian_stem", RussianStemTokenFilterFactory::new);
tokenFilters.register("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
tokenFilters.register("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
tokenFilters.register("arabic_normalization", ArabicNormalizationFilterFactory::new);
tokenFilters.register("german_normalization", GermanNormalizationFilterFactory::new);
tokenFilters.register("hindi_normalization", HindiNormalizationFilterFactory::new);
tokenFilters.register("indic_normalization", IndicNormalizationFilterFactory::new);
tokenFilters.register("sorani_normalization", SoraniNormalizationFilterFactory::new);
tokenFilters.register("persian_normalization", PersianNormalizationFilterFactory::new);
tokenFilters.register("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
tokenFilters.register("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
tokenFilters.register("serbian_normalization", SerbianNormalizationFilterFactory::new);
tokenFilters.register("hunspell", requriesAnalysisSettings(
(indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
tokenFilters.register("cjk_bigram", CJKBigramFilterFactory::new);
tokenFilters.register("cjk_width", CJKWidthFilterFactory::new);
tokenFilters.register("apostrophe", ApostropheFilterFactory::new);
tokenFilters.register("classic", ClassicFilterFactory::new);
tokenFilters.register("decimal_digit", DecimalDigitFilterFactory::new);
tokenFilters.register("fingerprint", FingerprintTokenFilterFactory::new);
tokenFilters.registerPlugins(plugins, AnalysisPlugin::getTokenFilters);
return tokenFilters;
}
private NamedRegistry<AnalysisProvider<TokenizerFactory>> setupTokenizers(List<AnalysisPlugin> plugins) {
NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = new NamedRegistry<>("tokenizer");
tokenizers.register("standard", StandardTokenizerFactory::new);
tokenizers.register("uax_url_email", UAX29URLEmailTokenizerFactory::new);
tokenizers.register("path_hierarchy", PathHierarchyTokenizerFactory::new);
tokenizers.register("PathHierarchy", PathHierarchyTokenizerFactory::new);
tokenizers.register("keyword", KeywordTokenizerFactory::new);
tokenizers.register("letter", LetterTokenizerFactory::new);
tokenizers.register("lowercase", LowerCaseTokenizerFactory::new);
tokenizers.register("whitespace", WhitespaceTokenizerFactory::new);
tokenizers.register("nGram", NGramTokenizerFactory::new);
tokenizers.register("ngram", NGramTokenizerFactory::new);
tokenizers.register("edgeNGram", EdgeNGramTokenizerFactory::new);
tokenizers.register("edge_ngram", EdgeNGramTokenizerFactory::new);
tokenizers.register("pattern", PatternTokenizerFactory::new);
tokenizers.register("classic", ClassicTokenizerFactory::new);
tokenizers.register("thai", ThaiTokenizerFactory::new);
tokenizers.registerPlugins(plugins, AnalysisPlugin::getTokenizers);
return tokenizers;
}
private NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> setupAnalyzers(List<AnalysisPlugin> plugins) {
NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = new NamedRegistry<>("analyzer");
analyzers.register("default", StandardAnalyzerProvider::new);
analyzers.register("standard", StandardAnalyzerProvider::new);
analyzers.register("standard_html_strip", StandardHtmlStripAnalyzerProvider::new);
analyzers.register("simple", SimpleAnalyzerProvider::new);
analyzers.register("stop", StopAnalyzerProvider::new);
analyzers.register("whitespace", WhitespaceAnalyzerProvider::new);
analyzers.register("keyword", KeywordAnalyzerProvider::new);
analyzers.register("pattern", PatternAnalyzerProvider::new);
analyzers.register("snowball", SnowballAnalyzerProvider::new);
analyzers.register("arabic", ArabicAnalyzerProvider::new);
analyzers.register("armenian", ArmenianAnalyzerProvider::new);
analyzers.register("basque", BasqueAnalyzerProvider::new);
analyzers.register("brazilian", BrazilianAnalyzerProvider::new);
analyzers.register("bulgarian", BulgarianAnalyzerProvider::new);
analyzers.register("catalan", CatalanAnalyzerProvider::new);
analyzers.register("chinese", ChineseAnalyzerProvider::new);
analyzers.register("cjk", CjkAnalyzerProvider::new);
analyzers.register("czech", CzechAnalyzerProvider::new);
analyzers.register("danish", DanishAnalyzerProvider::new);
analyzers.register("dutch", DutchAnalyzerProvider::new);
analyzers.register("english", EnglishAnalyzerProvider::new);
analyzers.register("finnish", FinnishAnalyzerProvider::new);
analyzers.register("french", FrenchAnalyzerProvider::new);
analyzers.register("galician", GalicianAnalyzerProvider::new);
analyzers.register("german", GermanAnalyzerProvider::new);
analyzers.register("greek", GreekAnalyzerProvider::new);
analyzers.register("hindi", HindiAnalyzerProvider::new);
analyzers.register("hungarian", HungarianAnalyzerProvider::new);
analyzers.register("indonesian", IndonesianAnalyzerProvider::new);
analyzers.register("irish", IrishAnalyzerProvider::new);
analyzers.register("italian", ItalianAnalyzerProvider::new);
analyzers.register("latvian", LatvianAnalyzerProvider::new);
analyzers.register("lithuanian", LithuanianAnalyzerProvider::new);
analyzers.register("norwegian", NorwegianAnalyzerProvider::new);
analyzers.register("persian", PersianAnalyzerProvider::new);
analyzers.register("portuguese", PortugueseAnalyzerProvider::new);
analyzers.register("romanian", RomanianAnalyzerProvider::new);
analyzers.register("russian", RussianAnalyzerProvider::new);
analyzers.register("sorani", SoraniAnalyzerProvider::new);
analyzers.register("spanish", SpanishAnalyzerProvider::new);
analyzers.register("swedish", SwedishAnalyzerProvider::new);
analyzers.register("turkish", TurkishAnalyzerProvider::new);
analyzers.register("thai", ThaiAnalyzerProvider::new);
analyzers.register("fingerprint", FingerprintAnalyzerProvider::new);
analyzers.registerPlugins(plugins, AnalysisPlugin::getAnalyzers);
return analyzers;
}
private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
return new AnalysisModule.AnalysisProvider<T>() {
@Override
public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
return provider.get(indexSettings, environment, name, settings);
}
@Override
public boolean requiresAnalysisSettings() {
return true;
}
};
}
/**
* Registers a new {@link AnalysisProvider} to create
* {@link CharFilterFactory} instance per node as well as per index.
*/
public void registerCharFilter(String name, AnalysisProvider<CharFilterFactory> charFilter) {
if (charFilter == null) {
throw new IllegalArgumentException("char_filter provider must not be null");
}
if (charFilters.putIfAbsent(name, charFilter) != null) {
throw new IllegalArgumentException("char_filter provider for name " + name + " already registered");
}
}
/**
* Registers a new {@link AnalysisProvider} to create
* {@link TokenFilterFactory} instance per node as well as per index.
*/
public void registerTokenFilter(String name, AnalysisProvider<TokenFilterFactory> tokenFilter) {
if (tokenFilter == null) {
throw new IllegalArgumentException("token_filter provider must not be null");
}
if (tokenFilters.putIfAbsent(name, tokenFilter) != null) {
throw new IllegalArgumentException("token_filter provider for name " + name + " already registered");
}
}
/**
* Registers a new {@link AnalysisProvider} to create
* {@link TokenizerFactory} instance per node as well as per index.
*/
public void registerTokenizer(String name, AnalysisProvider<TokenizerFactory> tokenizer) {
if (tokenizer == null) {
throw new IllegalArgumentException("tokenizer provider must not be null");
}
if (tokenizers.putIfAbsent(name, tokenizer) != null) {
throw new IllegalArgumentException("tokenizer provider for name " + name + " already registered");
}
}
/**
* Registers a new {@link AnalysisProvider} to create
* {@link AnalyzerProvider} instance per node as well as per index.
*/
public void registerAnalyzer(String name, AnalysisProvider<AnalyzerProvider> analyzer) {
if (analyzer == null) {
throw new IllegalArgumentException("analyzer provider must not be null");
}
if (analyzers.putIfAbsent(name, analyzer) != null) {
throw new IllegalArgumentException("analyzer provider for name " + name + " already registered");
}
}
/**
* Registers a new hunspell {@link Dictionary} that can be referenced by the given name in
* hunspell analysis configuration.
*/
public void registerHunspellDictionary(String name, Dictionary dictionary) {
if (knownDictionaries.putIfAbsent(name, dictionary) != null) {
throw new IllegalArgumentException("dictionary for [" + name + "] is already registered");
}
}
@Override
protected void configure() {
try {
AnalysisRegistry registry = buildRegistry();
bind(HunspellService.class).toInstance(registry.getHunspellService());
bind(AnalysisRegistry.class).toInstance(registry);
} catch (IOException e) {
throw new ElasticsearchException("failed to load hunspell service", e);
}
}
/**
* Builds an {@link AnalysisRegistry} from the current configuration.
*/
public AnalysisRegistry buildRegistry() throws IOException {
return new AnalysisRegistry(new HunspellService(environment.settings(), environment, knownDictionaries), environment, charFilters, tokenFilters, tokenizers, analyzers);
}
/**
* AnalysisProvider is the basic factory interface for registering analysis components like:
* <ul>
* <li>{@link TokenizerFactory} - see {@link AnalysisModule#registerTokenizer(String, AnalysisProvider)}</li>
* <li>{@link CharFilterFactory} - see {@link AnalysisModule#registerCharFilter(String, AnalysisProvider)}</li>
* <li>{@link AnalyzerProvider} - see {@link AnalysisModule#registerAnalyzer(String, AnalysisProvider)}</li>
* <li>{@link TokenFilterFactory}- see {@link AnalysisModule#registerTokenFilter(String, AnalysisProvider)} )}</li>
* </ul>
* The basic factory interface for analysis components.
*/
public interface AnalysisProvider<T> {
@ -195,7 +374,8 @@ public final class AnalysisModule extends AbstractModule {
* @param name the name of the analysis component
* @return a new provider instance
* @throws IOException if an {@link IOException} occurs
* @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns <code>true</code>
* @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns
* <code>true</code>
*/
default T get(Environment environment, String name) throws IOException {
if (requiresAnalysisSettings()) {
@ -212,4 +392,29 @@ public final class AnalysisModule extends AbstractModule {
return false;
}
}
private static class NamedRegistry<T> {
private final Map<String, T> registry = new HashMap<>();
private final String targetName;
public NamedRegistry(String targetName) {
this.targetName = targetName;
}
private void register(String name, T t) {
requireNonNull(name, "name is required");
requireNonNull(t, targetName + " is required");
if (registry.putIfAbsent(name, t) != null) {
throw new IllegalArgumentException(targetName + " for name " + name + " already registered");
}
}
private <P> void registerPlugins(List<P> plugins, Function<P, Map<String, T>> lookup) {
for (P plugin : plugins) {
for (Map.Entry<String, T> entry : lookup.apply(plugin).entrySet()) {
register(entry.getKey(), entry.getValue());
}
}
}
}
}

8
core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
View File

@ -52,7 +52,8 @@ import java.util.function.Function;
* The following settings can be set for each dictionary:
* <ul>
* <li>{@code ignore_case} - If true, dictionary matching will be case insensitive (defaults to {@code false})</li>
* <li>{@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored (defaults to {@code true})</li>
* <li>{@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored
* (defaults to {@code true})</li>
* </ul>
* <p>
* These settings can either be configured as node level configuration, such as:
@ -86,7 +87,8 @@ public class HunspellService extends AbstractComponent {
private final Path hunspellDir;
private final Function<String, Dictionary> loadingFunction;
public HunspellService(final Settings settings, final Environment env, final Map<String, Dictionary> knownDictionaries) throws IOException {
public HunspellService(final Settings settings, final Environment env, final Map<String, Dictionary> knownDictionaries)
throws IOException {
super(settings);
this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries);
this.hunspellDir = resolveHunspellDirectory(env);
@ -166,7 +168,7 @@ public class HunspellService extends AbstractComponent {
// merging node settings with hunspell dictionary specific settings
Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale));
nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));
boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);

5
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -72,6 +72,7 @@ import org.elasticsearch.gateway.GatewayModule;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.http.HttpServer;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.analysis.AnalysisModule;
@ -85,6 +86,7 @@ import org.elasticsearch.monitor.MonitorService;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.elasticsearch.node.service.NodeService;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.plugins.ScriptPlugin;
@ -227,6 +229,7 @@ public class Node implements Closeable {
final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService,
pluginsService.filterPlugins(ScriptPlugin.class));
AnalysisModule analysisModule = new AnalysisModule(environment, pluginsService.filterPlugins(AnalysisPlugin.class));
additionalSettings.addAll(scriptModule.getSettings());
// this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
// so we might be late here already
@ -260,7 +263,6 @@ public class Node implements Closeable {
modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
modules.add(new GatewayModule());
modules.add(new RepositoriesModule());
modules.add(new AnalysisModule(environment));
pluginsService.processModules(modules);
CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
settingsModule.getClusterSettings());
@ -279,6 +281,7 @@ public class Node implements Closeable {
b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
b.bind(BigArrays.class).toInstance(bigArrays);
b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry());
}
);
injector = modules.createInjector();

84
core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import java.util.Map;
import static java.util.Collections.emptyMap;
/**
* An additional extension point for {@link Plugin}s that extends Elasticsearch's analysis functionality. To add an additional
* {@link TokenFilter} just implement the interface and implement the {@link #getTokenFilters()} method:
*
* <pre>{@code
* public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin {
* &#64;Override
* public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
* return singletonMap("phonetic", PhoneticTokenFilterFactory::new);
* }
* }
* }</pre>
*/
public interface AnalysisPlugin {
/**
* Override to add additional {@link CharFilter}s.
*/
default Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
return emptyMap();
}
/**
* Override to add additional {@link TokenFilter}s.
*/
default Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return emptyMap();
}
/**
* Override to add additional {@link Tokenizer}s.
*/
default Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
return emptyMap();
}
/**
* Override to add additional {@link Analyzer}s.
*/
default Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
return emptyMap();
}
/**
* Override to add additional hunspell {@link org.apache.lucene.analysis.hunspell.Dictionary}s.
*/
default Map<String, org.apache.lucene.analysis.hunspell.Dictionary> getHunspellDictionaries() {
return emptyMap();
}
}

9
core/src/main/java/org/elasticsearch/plugins/Plugin.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.threadpool.ExecutorBuilder;
@ -103,6 +104,14 @@ public abstract class Plugin {
@Deprecated
public final void onModule(ScriptModule module) {}
/**
* Old-style analysis extension point.
*
* @deprecated implement {@link AnalysisPlugin} instead
*/
@Deprecated
public final void onModule(AnalysisModule module) {}
/**
* Provides the list of this plugin's custom thread pools, empty if
* none.

5
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.script.NativeScriptFactory;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptEngineService;
@ -207,8 +208,8 @@ public class PluginsService extends AbstractComponent {
}
Class moduleClass = method.getParameterTypes()[0];
if (!Module.class.isAssignableFrom(moduleClass)) {
if (moduleClass == ScriptModule.class) {
// This is still part of the Plugin class to point the user to the new implementation
if (method.getDeclaringClass() == Plugin.class) {
// These are still part of the Plugin class to point the user to the new implementations
continue;
}
throw new RuntimeException(

3
core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
View File

@ -27,8 +27,7 @@ import java.util.Collections;
import java.util.List;
/**
* An additional extension point to {@link Plugin}. Plugins extending the scripting functionality must implement this inteface
* to provide access to script engines or script factories.
* An additional extension point for {@link Plugin}s that extends Elasticsearch's scripting functionality.
*/
public interface ScriptPlugin {

5
core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
View File

@ -29,12 +29,15 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import java.io.IOException;
import java.util.List;
import static java.util.Collections.emptyList;
public class TransportAnalyzeActionTests extends ESTestCase {
private AnalysisService analysisService;
@ -56,7 +59,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
.putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter").build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
environment = new Environment(settings);
registry = new AnalysisRegistry(null, environment);
registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry();
analysisService = registry.build(idxSettings);
}

50
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -36,7 +36,6 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
@ -45,9 +44,9 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.cache.query.QueryCache;
import org.elasticsearch.index.cache.query.IndexQueryCache;
import org.elasticsearch.index.cache.query.DisabledQueryCache;
import org.elasticsearch.index.cache.query.IndexQueryCache;
import org.elasticsearch.index.cache.query.QueryCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@ -61,9 +60,9 @@ import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreConfig;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.IndicesQueryCache;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.IndicesQueryCache;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@ -73,6 +72,7 @@ import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.TestSearchContext;
@ -84,11 +84,12 @@ import org.elasticsearch.watcher.ResourceWatcherService;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import static java.util.Collections.emptyMap;
public class IndexModuleTests extends ESTestCase {
private Index index;
private Settings settings;
@ -147,7 +148,8 @@ public class IndexModuleTests extends ESTestCase {
}
public void testWrapperIsBound() throws IOException {
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(indexSettings, null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.setSearcherWrapper((s) -> new Wrapper());
module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class));
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener));
@ -165,7 +167,8 @@ public class IndexModuleTests extends ESTestCase {
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "foo_store")
.build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(indexSettings, null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.addIndexStore("foo_store", FooStore::new);
try {
module.addIndexStore("foo_store", FooStore::new);
@ -188,7 +191,8 @@ public class IndexModuleTests extends ESTestCase {
}
};
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(indexSettings, null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
Consumer<Settings> listener = (s) -> {};
module.addIndexEventListener(eventListener);
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
@ -204,7 +208,8 @@ public class IndexModuleTests extends ESTestCase {
public void testListener() throws IOException {
Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
AtomicBoolean atomicBoolean = new AtomicBoolean(false);
module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
@ -224,7 +229,8 @@ public class IndexModuleTests extends ESTestCase {
}
public void testAddIndexOperationListener() throws IOException {
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
AtomicBoolean executed = new AtomicBoolean(false);
IndexingOperationListener listener = new IndexingOperationListener() {
@Override
@ -254,7 +260,8 @@ public class IndexModuleTests extends ESTestCase {
}
public void testAddSearchOperationListener() throws IOException {
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
AtomicBoolean executed = new AtomicBoolean(false);
SearchOperationListener listener = new SearchOperationListener() {
@ -289,7 +296,8 @@ public class IndexModuleTests extends ESTestCase {
.put("index.similarity.my_similarity.key", "there is a key")
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() {
@Override
public String name() {
@ -313,7 +321,8 @@ public class IndexModuleTests extends ESTestCase {
}
public void testFrozen() {
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.freeze();
String msg = "Can't modify IndexModule once the index service has been created";
assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage());
@ -331,7 +340,8 @@ public class IndexModuleTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
try {
module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener));
@ -346,7 +356,8 @@ public class IndexModuleTests extends ESTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
try {
module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener));
@ -359,7 +370,8 @@ public class IndexModuleTests extends ESTestCase {
Settings indexSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
@ -372,7 +384,8 @@ public class IndexModuleTests extends ESTestCase {
Settings indexSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener));
assertTrue(indexService.cache().query() instanceof IndexQueryCache);
@ -384,7 +397,8 @@ public class IndexModuleTests extends ESTestCase {
.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener));

57
core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.analysis;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@ -29,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -41,12 +43,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
public class AnalysisServiceTests extends ESTestCase {
private static AnalyzerProvider analyzerProvider(final String name) {
private static AnalyzerProvider<?> analyzerProvider(final String name) {
return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer());
}
@ -58,7 +63,8 @@ public class AnalysisServiceTests extends ESTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
.build(idxSettings);
assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
@ -68,33 +74,28 @@ public class AnalysisServiceTests extends ESTestCase {
Version version = VersionUtils.randomVersion(random());
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
Collections.singletonMap("default", analyzerProvider("default")),
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
singletonMap("default", analyzerProvider("default")), emptyMap(), emptyMap(), emptyMap());
assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
}
public void testOverrideDefaultIndexAnalyzer() {
public void testOverrideDefaultIndexAnalyzerIsUnsupported() {
Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.CURRENT);
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
try {
AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
Collections.singletonMap("default_index", new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer())),
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
fail("Expected ISE");
} catch (IllegalArgumentException e) {
// expected
assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
}
AnalyzerProvider<?> defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap()));
assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
}
public void testBackCompatOverrideDefaultIndexAnalyzer() {
Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
Collections.singletonMap("default_index", analyzerProvider("default_index")),
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap());
assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
@ -104,17 +105,17 @@ public class AnalysisServiceTests extends ESTestCase {
Version version = VersionUtils.randomVersion(random());
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
Collections.singletonMap("default_search", analyzerProvider("default_search")),
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap());
assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
}
public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
Map<String, AnalyzerProvider> analyzers = new HashMap<>();
Map<String, AnalyzerProvider<?>> analyzers = new HashMap<>();
analyzers.put("default_index", analyzerProvider("default_index"));
analyzers.put("default_search", analyzerProvider("default_search"));
AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
@ -125,7 +126,6 @@ public class AnalysisServiceTests extends ESTestCase {
}
public void testConfigureCamelCaseTokenFilter() throws IOException {
// tests a filter that
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@ -137,7 +137,9 @@ public class AnalysisServiceTests extends ESTestCase {
.putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry()
.build(idxSettings);
try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer")) {
assertNotNull(custom_analyser);
TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
@ -176,8 +178,10 @@ public class AnalysisServiceTests extends ESTestCase {
Settings indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService otherAnalysisSergice = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
.build(idxSettings);
AnalysisService otherAnalysisSergice = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(),
emptyMap()).build(idxSettings);
final int numIters = randomIntBetween(5, 20);
for (int i = 0; i < numIters; i++) {
PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
@ -196,7 +200,8 @@ public class AnalysisServiceTests extends ESTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisRegistry(null, new Environment(settings)).build(idxSettings));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
}
}

9
core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
View File

@ -23,14 +23,14 @@ import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.analysis.HunspellService;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.test.IndexSettingsModule;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import static java.util.Collections.emptyList;
public class AnalysisTestsHelper {
@ -49,7 +49,6 @@ public class AnalysisTestsHelper {
settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
}
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
Environment environment = new Environment(settings);
return new AnalysisRegistry(new HunspellService(settings, environment, Collections.emptyMap()), environment).build(idxSettings);
return new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry().build(idxSettings);
}
}

8
core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
View File

@ -26,6 +26,8 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.test.ESTokenStreamTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import static org.elasticsearch.test.ESTestCase.createAnalysisService;
/**
*/
public class CharFilterTests extends ESTokenStreamTestCase {
@ -39,7 +41,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"});
@ -56,7 +58,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
@ -78,7 +80,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"});

28
core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
View File

@ -31,15 +31,20 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.hamcrest.MatcherAssert;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.instanceOf;
@ -50,8 +55,13 @@ public class CompoundAnalysisTests extends ESTestCase {
public void testDefaultsCompoundAnalysis() throws Exception {
Settings settings = getJsonSettings();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings),
Collections.emptyMap(),Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings);
AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
}
}));
AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
TokenFilterFactory filterFactory = analysisService.tokenFilter("dict_dec");
MatcherAssert.assertThat(filterFactory, instanceOf(DictionaryCompoundWordTokenFilterFactory.class));
@ -62,14 +72,20 @@ public class CompoundAnalysisTests extends ESTestCase {
for (Settings settings : settingsArr) {
List<String> terms = analyze(settings, "decompoundingAnalyzer", "donaudampfschiff spargelcremesuppe");
MatcherAssert.assertThat(terms.size(), equalTo(8));
MatcherAssert.assertThat(terms, hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe"));
MatcherAssert.assertThat(terms,
hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe"));
}
}
private List<String> analyze(Settings settings, String analyzerName, String text) throws IOException {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings),
Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings);
AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
}
}));
AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer();

3
core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.test.ESTokenStreamTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import static org.elasticsearch.test.ESTestCase.createAnalysisService;
import static org.hamcrest.Matchers.containsString;
public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase {
@ -39,7 +40,7 @@ public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
NamedAnalyzer analyzer1 = analysisService.analyzer("single");

4
core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
View File

@ -27,6 +27,8 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.test.ESTokenStreamTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import static org.elasticsearch.test.ESTestCase.createAnalysisService;
public class StopAnalyzerTests extends ESTokenStreamTestCase {
public void testDefaultsCompoundAnalysis() throws Exception {
String json = "/org/elasticsearch/index/analysis/stop.json";
@ -36,7 +38,7 @@ public class StopAnalyzerTests extends ESTokenStreamTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
NamedAnalyzer analyzer1 = analysisService.analyzer("analyzer1");

6
core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
View File

@ -26,13 +26,10 @@ import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.all.AllEntries;
import org.elasticsearch.common.lucene.all.AllTokenStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -67,8 +64,7 @@ public class SynonymsAnalysisTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
analysisService = createAnalysisService(idxSettings, settings);
match("synonymAnalyzer", "kimchy is the dude abides", "shay is the elasticsearch man!");
match("synonymAnalyzer_file", "kimchy is the dude abides", "shay is the elasticsearch man!");

4
core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
View File

@ -36,9 +36,7 @@ import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.similarity.SimilarityService;
@ -97,7 +95,7 @@ public class CodecTests extends ESTestCase {
.build();
IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap());
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(settings);
AnalysisService analysisService = createAnalysisService(settings, nodeSettings);
MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap());
MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry, () -> null);
return new CodecService(service, ESLoggerFactory.getLogger("test"));

59
core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java → core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.analysis;
package org.elasticsearch.indices.analysis;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@ -36,9 +36,20 @@ import org.elasticsearch.common.inject.ModuleTestCase;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.Analysis;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.AnalysisTestsHelper;
import org.elasticsearch.index.analysis.CustomAnalyzer;
import org.elasticsearch.index.analysis.MappingCharFilterFactory;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
import org.elasticsearch.index.analysis.StandardTokenizerFactory;
import org.elasticsearch.index.analysis.StopTokenFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.HunspellService;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.test.IndexSettingsModule;
import org.hamcrest.MatcherAssert;
@ -49,9 +60,11 @@ import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
@ -72,8 +85,16 @@ public class AnalysisModuleTests extends ModuleTestCase {
}
public AnalysisRegistry getNewRegistry(Settings settings) {
return new AnalysisRegistry(null, new Environment(settings),
Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new), Collections.emptyMap(), Collections.emptyMap());
try {
return new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
}
})).getAnalysisRegistry();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Settings loadFromClasspath(String path) throws IOException {
@ -125,7 +146,7 @@ public class AnalysisModuleTests extends ModuleTestCase {
assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), analysisService2.analyzer("custom7").analyzer().getVersion());
}
private void assertTokenFilter(String name, Class clazz) throws IOException {
private void assertTokenFilter(String name, Class<?> clazz) throws IOException {
Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
@ -148,17 +169,9 @@ public class AnalysisModuleTests extends ModuleTestCase {
StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0];
assertThat(stop1.stopWords().size(), equalTo(1));
//assertThat((Iterable<char[]>) stop1.stopWords(), hasItem("test-stop".toCharArray()));
analyzer = analysisService.analyzer("custom2").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom2 = (CustomAnalyzer) analyzer;
// HtmlStripCharFilterFactory html = (HtmlStripCharFilterFactory) custom2.charFilters()[0];
// assertThat(html.readAheadLimit(), equalTo(HTMLStripCharFilter.DEFAULT_READ_AHEAD));
//
// html = (HtmlStripCharFilterFactory) custom2.charFilters()[1];
// assertThat(html.readAheadLimit(), equalTo(1024));
// verify position increment gap
analyzer = analysisService.analyzer("custom6").analyzer();
@ -248,7 +261,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
getAnalysisService(settings);
fail("This should fail with IllegalArgumentException because the analyzers name starts with _");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\"")));
assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\""))
.or(equalTo("analyzer name must not start with '_'. got \"_invalidName\"")));
}
}
@ -289,13 +303,18 @@ public class AnalysisModuleTests extends ModuleTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
Environment environment = new Environment(settings);
AnalysisModule module = new AnalysisModule(environment);
InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff");
InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic");
Dictionary dictionary;
try (Directory tmp = new SimpleFSDirectory(environment.tmpFile())) {
Dictionary dictionary = new Dictionary(tmp, "hunspell", aff, dic);
module.registerHunspellDictionary("foo", dictionary);
assertInstanceBinding(module, HunspellService.class, (x) -> x.getDictionary("foo") == dictionary);
dictionary = new Dictionary(tmp, "hunspell", aff, dic);
}
AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() {
@Override
public Map<String, Dictionary> getHunspellDictionaries() {
return singletonMap("foo", dictionary);
}
}));
assertSame(dictionary, module.getHunspellService().getDictionary("foo"));
}
}

35
core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
View File

@ -19,15 +19,38 @@
package org.elasticsearch.indices.analysis;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
public class DummyAnalysisPlugin extends Plugin {
import java.util.Map;
public void onModule(AnalysisModule module) {
module.registerAnalyzer("dummy", (a, b, c, d) -> new DummyAnalyzerProvider());
module.registerTokenFilter("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory());
module.registerTokenizer("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory());
module.registerCharFilter("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory());
import static java.util.Collections.singletonMap;
public class DummyAnalysisPlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
return singletonMap("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory());
}
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory());
}
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
return singletonMap("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory());
}
@Override
public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
return singletonMap("dummy", (a, b, c, d) -> new DummyAnalyzerProvider());
}
}

68
core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java → core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java
View File

@ -19,36 +19,30 @@
package org.elasticsearch.indices.analyze;
import org.apache.lucene.analysis.hunspell.Dictionary;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.indices.analysis.HunspellService;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.hamcrest.Matchers;
import org.elasticsearch.test.ESTestCase;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_IGNORE_CASE;
import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_LAZY_LOAD;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.notNullValue;
/**
*
*/
@ClusterScope(scope= Scope.TEST, numDataNodes=0)
public class HunspellServiceIT extends ESIntegTestCase {
public class HunspellServiceTests extends ESTestCase {
public void testLocaleDirectoryWithNodeLevelConfig() throws Exception {
Settings settings = Settings.builder()
.put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/conf_dir"))
.put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
.put(HUNSPELL_IGNORE_CASE.getKey(), true)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.build();
internalCluster().startNode(settings);
Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US");
assertThat(dictionary, notNullValue());
assertIgnoreCase(true, dictionary);
assertTrue(dictionary.getIgnoreCase());
}
public void testLocaleDirectoryWithLocaleSpecificConfig() throws Exception {
@ -58,58 +52,42 @@ public class HunspellServiceIT extends ESIntegTestCase {
.put(HUNSPELL_IGNORE_CASE.getKey(), true)
.put("indices.analysis.hunspell.dictionary.en_US.strict_affix_parsing", false)
.put("indices.analysis.hunspell.dictionary.en_US.ignore_case", false)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.build();
internalCluster().startNode(settings);
Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US");
assertThat(dictionary, notNullValue());
assertIgnoreCase(false, dictionary);
assertFalse(dictionary.getIgnoreCase());
// testing that dictionary specific settings override node level settings
dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US_custom");
dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US_custom");
assertThat(dictionary, notNullValue());
assertIgnoreCase(true, dictionary);
assertTrue(dictionary.getIgnoreCase());
}
public void testDicWithNoAff() throws Exception {
Settings settings = Settings.builder()
.put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/no_aff_conf_dir"))
.put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.build();
Dictionary dictionary = null;
try {
internalCluster().startNode(settings);
dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
fail("Missing affix file didn't throw an error");
}
catch (Throwable t) {
assertNull(dictionary);
assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Missing affix file"));
}
IllegalStateException e = expectThrows(IllegalStateException.class,
() -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"));
assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage());
assertThat(e.getCause(), hasToString(containsString("Missing affix file")));
}
public void testDicWithTwoAffs() throws Exception {
Settings settings = Settings.builder()
.put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/two_aff_conf_dir"))
.put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.build();
Dictionary dictionary = null;
try {
internalCluster().startNode(settings);
dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
fail("Multiple affix files didn't throw an error");
} catch (Throwable t) {
assertNull(dictionary);
assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Too many affix files"));
}
}
// TODO: on next upgrade of lucene, just use new getter
private void assertIgnoreCase(boolean expected, Dictionary dictionary) throws Exception {
// assertEquals(expected, dictionary.getIgnoreCase());
IllegalStateException e = expectThrows(IllegalStateException.class,
() -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"));
assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage());
assertThat(e.getCause(), hasToString(containsString("Too many affix files")));
}
}

5
docs/reference/migration/migrate_5_0/plugins.asciidoc
View File

@ -122,3 +122,8 @@ been removed.
Plugins that register custom scripts should implement `ScriptPlugin` and remove
their `onModule(ScriptModule)` implementation.
==== AnalysisPlugin
Plugins that register custom analysis components should implement
`AnalysisPlugin` and remove their `onModule(AnalysisModule)` implementation.

4
modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.inject.multibindings.Multibinder;
@ -40,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@ -129,7 +127,7 @@ public class TemplateQueryParserTests extends ESTestCase {
new IndexSettingsModule(index, settings)
).createInjector();
AnalysisService analysisService = new AnalysisRegistry(null, environment).build(idxSettings);
AnalysisService analysisService = createAnalysisService(idxSettings, settings);
SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->

40
plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
View File

@ -19,26 +19,42 @@
package org.elasticsearch.plugin.analysis.icu;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory;
import org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory;
import org.elasticsearch.index.analysis.IcuNormalizerCharFilterFactory;
import org.elasticsearch.index.analysis.IcuNormalizerTokenFilterFactory;
import org.elasticsearch.index.analysis.IcuTokenizerFactory;
import org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
public class AnalysisICUPlugin extends Plugin {
import java.util.HashMap;
import java.util.Map;
/**
* Automatically called with the analysis module.
*/
public void onModule(AnalysisModule module) {
module.registerCharFilter("icu_normalizer", IcuNormalizerCharFilterFactory::new);
module.registerTokenizer("icu_tokenizer", IcuTokenizerFactory::new);
module.registerTokenFilter("icu_normalizer", IcuNormalizerTokenFilterFactory::new);
module.registerTokenFilter("icu_folding", IcuFoldingTokenFilterFactory::new);
module.registerTokenFilter("icu_collation", IcuCollationTokenFilterFactory::new);
module.registerTokenFilter("icu_transform", IcuTransformTokenFilterFactory::new);
import static java.util.Collections.singletonMap;
public class AnalysisICUPlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
return singletonMap("icu_normalizer", IcuNormalizerCharFilterFactory::new);
}
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
extra.put("icu_normalizer", IcuNormalizerTokenFilterFactory::new);
extra.put("icu_folding", IcuFoldingTokenFilterFactory::new);
extra.put("icu_collation", IcuCollationTokenFilterFactory::new);
extra.put("icu_transform", IcuTransformTokenFilterFactory::new);
return extra;
}
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
return singletonMap("icu_tokenizer", IcuTokenizerFactory::new);
}
}

2
plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
View File

@ -102,6 +102,6 @@ public class IcuTokenizerFactoryTests extends ESTestCase {
.build();
Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin()::onModule);
return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin());
}
}

3
plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
View File

@ -31,8 +31,7 @@ import static org.hamcrest.Matchers.instanceOf;
*/
public class SimpleIcuAnalysisTests extends ESTestCase {
public void testDefaultsIcuAnalysis() throws IOException {
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"),
Settings.EMPTY, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisICUPlugin());
TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class));

18
plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
View File

@ -50,7 +50,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.language", "tr")
.put("index.analysis.filter.myCollator.strength", "primary")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "I WİLL USE TURKİSH CASING", "ı will use turkish casıng");
@ -66,7 +66,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.strength", "primary")
.put("index.analysis.filter.myCollator.decomposition", "canonical")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng");
@ -82,7 +82,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.strength", "secondary")
.put("index.analysis.filter.myCollator.decomposition", "no")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "TESTING", "testing");
@ -99,7 +99,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.strength", "primary")
.put("index.analysis.filter.myCollator.alternate", "shifted")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "foo-bar", "foo bar");
@ -117,7 +117,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.alternate", "shifted")
.put("index.analysis.filter.myCollator.variableTop", " ")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "foo bar", "foobar");
@ -135,7 +135,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.language", "en")
.put("index.analysis.filter.myCollator.numeric", "true")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollation(filterFactory, "foobar-9", "foobar-10", -1);
@ -152,7 +152,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.strength", "primary")
.put("index.analysis.filter.myCollator.caseLevel", "true")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "résumé", "resume");
@ -172,7 +172,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.strength", "tertiary")
.put("index.analysis.filter.myCollator.caseFirst", "upper")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollation(filterFactory, "Resume", "resume", -1);
@ -200,7 +200,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
.put("index.analysis.filter.myCollator.rules", tailoredRules)
.put("index.analysis.filter.myCollator.strength", "primary")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
assertCollatesToSame(filterFactory, "Töne", "Toene");

4
plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
View File

@ -37,7 +37,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
Settings settings = Settings.builder()
.put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";
@ -61,7 +61,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
.put("index.analysis.char_filter.myNormalizerChar.name", "nfkc")
.put("index.analysis.char_filter.myNormalizerChar.mode", "decompose")
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";

52
plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
View File

@ -19,6 +19,9 @@
package org.elasticsearch.plugin.analysis.kuromoji;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory;
import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider;
import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory;
@ -28,23 +31,42 @@ import org.elasticsearch.index.analysis.KuromojiNumberFilterFactory;
import org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory;
import org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory;
import org.elasticsearch.index.analysis.KuromojiTokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
/**
*
*/
public class AnalysisKuromojiPlugin extends Plugin {
import java.util.HashMap;
import java.util.Map;
public void onModule(AnalysisModule module) {
module.registerCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new);
module.registerAnalyzer("kuromoji", KuromojiAnalyzerProvider::new);
module.registerTokenizer("kuromoji_tokenizer", KuromojiTokenizerFactory::new);
module.registerTokenFilter("kuromoji_baseform", KuromojiBaseFormFilterFactory::new);
module.registerTokenFilter("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new);
module.registerTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory::new);
module.registerTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new);
module.registerTokenFilter("ja_stop", JapaneseStopTokenFilterFactory::new);
module.registerTokenFilter("kuromoji_number", KuromojiNumberFilterFactory::new);
import static java.util.Collections.singletonMap;
public class AnalysisKuromojiPlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
return singletonMap("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new);
}
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
extra.put("kuromoji_baseform", KuromojiBaseFormFilterFactory::new);
extra.put("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new);
extra.put("kuromoji_readingform", KuromojiReadingFormFilterFactory::new);
extra.put("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new);
extra.put("ja_stop", JapaneseStopTokenFilterFactory::new);
extra.put("kuromoji_number", KuromojiNumberFilterFactory::new);
return extra;
}
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
return singletonMap("kuromoji_tokenizer", KuromojiTokenizerFactory::new);
}
@Override
public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
return singletonMap("kuromoji", KuromojiAnalyzerProvider::new);
}
}

2
plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
View File

@ -198,7 +198,7 @@ public class KuromojiAnalysisTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin()::onModule);
return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin());
}
public static void assertSimpleTSOutput(TokenStream stream,

14
plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
View File

@ -20,13 +20,19 @@
package org.elasticsearch.plugin.analysis;
import org.elasticsearch.index.analysis.PhoneticTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
public class AnalysisPhoneticPlugin extends Plugin {
import java.util.Map;
public void onModule(AnalysisModule module) {
module.registerTokenFilter("phonetic", PhoneticTokenFilterFactory::new);
import static java.util.Collections.singletonMap;
public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("phonetic", PhoneticTokenFilterFactory::new);
}
}

3
plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
View File

@ -39,8 +39,7 @@ public class SimplePhoneticAnalysisTests extends ESTestCase {
Settings settings = Settings.builder().loadFromStream(yaml, getClass().getResourceAsStream(yaml))
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings,
new AnalysisPhoneticPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic");
MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class));
}

38
plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
View File

@ -19,20 +19,40 @@
package org.elasticsearch.plugin.analysis.smartcn;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider;
import org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory;
import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
public class AnalysisSmartChinesePlugin extends Plugin {
import java.util.HashMap;
import java.util.Map;
public void onModule(AnalysisModule module) {
module.registerAnalyzer("smartcn", SmartChineseAnalyzerProvider::new);
module.registerTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
// This is an alias to "smartcn_tokenizer"; it's here for backwards compat
module.registerTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
// This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
module.registerTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
import static java.util.Collections.singletonMap;
public class AnalysisSmartChinesePlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
// This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
return singletonMap("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
}
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
Map<String, AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();
extra.put("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
// This is an alias to "smartcn_tokenizer"; it's here for backwards compat
extra.put("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
return extra;
}
@Override
public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
return singletonMap("smartcn", SmartChineseAnalyzerProvider::new);
}
}

5
plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
View File

@ -29,11 +29,10 @@ import java.io.IOException;
import static org.hamcrest.Matchers.instanceOf;
/**
*/
public class SimpleSmartChineseAnalysisTests extends ESTestCase {
public void testDefaultsIcuAnalysis() throws IOException {
final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisSmartChinesePlugin()::onModule);
final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
new AnalysisSmartChinesePlugin());
TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer");
MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class));
}

22
plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
View File

@ -19,15 +19,27 @@
package org.elasticsearch.plugin.analysis.stempel;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.pl.PolishAnalyzerProvider;
import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
public class AnalysisStempelPlugin extends Plugin {
import java.util.Map;
public void onModule(AnalysisModule module) {
module.registerAnalyzer("polish", PolishAnalyzerProvider::new);
module.registerTokenFilter("polish_stem", PolishStemTokenFilterFactory::new);
import static java.util.Collections.singletonMap;
public class AnalysisStempelPlugin extends Plugin implements AnalysisPlugin {
@Override
public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
return singletonMap("polish_stem", PolishStemTokenFilterFactory::new);
}
@Override
public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
return singletonMap("polish", PolishAnalyzerProvider::new);
}
}

3
plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
View File

@ -36,7 +36,8 @@ import static org.hamcrest.Matchers.instanceOf;
*/
public class PolishAnalysisTests extends ESTestCase {
public void testDefaultsPolishAnalysis() throws IOException {
final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin()::onModule);
final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
new AnalysisStempelPlugin());
TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem");
MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class));

5
plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
View File

@ -49,7 +49,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
Settings settings = Settings.builder()
.put("index.analysis.filter.myStemmer.type", "polish_stem")
.build();
AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin());
TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer");
@ -65,8 +65,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
}
private void testAnalyzer(String source, String... expected_terms) throws IOException {
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
new AnalysisStempelPlugin()::onModule);
AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin());
Analyzer analyzer = analysisService.analyzer("polish").analyzer();

4
test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View File

@ -36,6 +36,8 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import static org.elasticsearch.test.ESTestCase.createAnalysisService;
public class MapperTestUtils {
@ -54,7 +56,7 @@ public class MapperTestUtils {
Settings finalSettings = settingsBuilder.build();
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", finalSettings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings);
AnalysisService analysisService = createAnalysisService(indexSettings, finalSettings);
SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
return new MapperService(indexSettings,
analysisService,

6
test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
View File

@ -71,7 +71,6 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@ -86,6 +85,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@ -118,6 +118,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static java.util.Collections.emptyList;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
@ -917,7 +918,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
injector = modulesBuilder.createInjector();
IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class);
idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), emptyList());
AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
scriptService = scriptModule.getScriptService();
similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);

39
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View File

@ -28,6 +28,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@ -44,7 +45,6 @@ import org.elasticsearch.common.io.PathUtilsForTesting;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.MockPageCacheRecycler;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -54,9 +54,11 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.ScriptService;
@ -768,29 +770,34 @@ public abstract class ESTestCase extends LuceneTestCase {
}
/**
* Creates an AnalysisService to test analysis factories and analyzers.
* Creates an AnalysisService with all the default analyzers configured.
*/
@SafeVarargs
public static AnalysisService createAnalysisService(Index index, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
public static AnalysisService createAnalysisService(Index index, Settings settings, AnalysisPlugin... analysisPlugins)
throws IOException {
Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
return createAnalysisService(index, nodeSettings, settings, moduleConsumers);
return createAnalysisService(index, nodeSettings, settings, analysisPlugins);
}
/**
* Creates an AnalysisService to test analysis factories and analyzers.
* Creates an AnalysisService with all the default analyzers configured.
*/
@SafeVarargs
public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings,
AnalysisPlugin... analysisPlugins) throws IOException {
Settings indexSettings = Settings.builder().put(settings)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
return createAnalysisService(IndexSettingsModule.newIndexSettings(index, indexSettings), nodeSettings, analysisPlugins);
}
/**
* Creates an AnalysisService with all the default analyzers configured.
*/
public static AnalysisService createAnalysisService(IndexSettings indexSettings, Settings nodeSettings,
AnalysisPlugin... analysisPlugins) throws IOException {
Environment env = new Environment(nodeSettings);
AnalysisModule analysisModule = new AnalysisModule(env);
for (Consumer<AnalysisModule> consumer : moduleConsumers) {
consumer.accept(analysisModule);
}
SettingsModule settingsModule = new SettingsModule(nodeSettings, InternalSettingsPlugin.VERSION_CREATED);
final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings));
AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins));
final AnalysisService analysisService = analysisModule.getAnalysisRegistry()
.build(indexSettings);
return analysisService;
}