diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java index 9c53fc1f63e..477675a1888 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; @@ -84,12 +85,15 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { if (preserveOriginal) { filters.add(IDENTITY_FILTER); } + // also merge and transfer token filter analysis modes with analyzer + AnalysisMode mode = AnalysisMode.ALL; for (String filter : filterNames) { String[] parts = Strings.tokenizeToStringArray(filter, ","); if (parts.length == 1) { TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]); factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters); filters.add(factory); + mode = mode.merge(factory.getAnalysisMode()); } else { List existingChain = new ArrayList<>(previousTokenFilters); List chain = new ArrayList<>(); @@ -98,10 +102,12 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters); chain.add(factory); existingChain.add(factory); + mode = mode.merge(factory.getAnalysisMode()); } filters.add(chainFilters(filter, chain)); } } + final AnalysisMode analysisMode = mode; return new TokenFilterFactory() { @Override @@ -133,6 +139,11 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { + "] cannot be used to parse synonyms unless [preserve_original] is [true]"); } } + + @Override + public AnalysisMode getAnalysisMode() { + return analysisMode; + } }; } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java index ea9e1e0c6aa..5b781ea7a25 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java @@ -78,5 +78,5 @@ public enum AnalysisMode { *
  • INDEX_TIME.merge(SEARCH_TIME) throws an {@link IllegalStateException}
    • * */ - abstract AnalysisMode merge(AnalysisMode other); + public abstract AnalysisMode merge(AnalysisMode other); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java index 750b7671152..c70554f14f3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java @@ -43,35 +43,25 @@ public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase { public void testSynonymsUpdateable() throws FileNotFoundException, IOException { String synonymsFileName = "synonyms.txt"; - Path configDir = node().getEnvironment().configFile(); - if (Files.exists(configDir) == false) { - Files.createDirectory(configDir); - } - Path synonymsFile = configDir.resolve(synonymsFileName); - if (Files.exists(synonymsFile) == false) { - Files.createFile(synonymsFile); - } - try (PrintWriter out = new PrintWriter( - new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { - out.println("foo, baz"); - } + Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz"); final String indexName = "test"; final String synonymAnalyzerName = "synonym_analyzer"; final String synonymGraphAnalyzerName = "synonym_graph_analyzer"; - assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder() - .put("index.number_of_shards", 5) - .put("index.number_of_replicas", 0) - .put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard") - .putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter") - .put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard") - .putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter") - .put("analysis.filter.synonym_filter.type", "synonym") - .put("analysis.filter.synonym_filter.updateable", "true") - .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName) - .put("analysis.filter.synonym_graph_filter.type", "synonym_graph") - .put("analysis.filter.synonym_graph_filter.updateable", "true") - .put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName)) + assertAcked(client().admin().indices().prepareCreate(indexName) + .setSettings(Settings.builder() + .put("index.number_of_shards", 5) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard") + .putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter") + .put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard") + .putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter") + .put("analysis.filter.synonym_filter.type", "synonym") + .put("analysis.filter.synonym_filter.updateable", "true") + .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName) + .put("analysis.filter.synonym_graph_filter.type", "synonym_graph") + .put("analysis.filter.synonym_graph_filter.updateable", "true") + .put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName)) .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName)); client().prepareIndex(indexName, "_doc", "1").setSource("field", "Foo").get(); @@ -84,8 +74,7 @@ public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase { { for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) { - Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName) - .get(); + Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get(); assertEquals(2, analyzeResponse.getTokens().size()); Set tokens = new HashSet<>(); analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); @@ -109,8 +98,7 @@ public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase { { for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) { - Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName) - .get(); + Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get(); assertEquals(3, analyzeResponse.getTokens().size()); Set tokens = new HashSet<>(); analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); @@ -126,8 +114,69 @@ public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase { assertHitCount(response, 1L); } + public void testSynonymsInMultiplexerUpdateable() throws FileNotFoundException, IOException { + String synonymsFileName = "synonyms.txt"; + Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz"); + + final String indexName = "test"; + final String synonymAnalyzerName = "synonym_in_multiplexer_analyzer"; + assertAcked(client().admin().indices().prepareCreate(indexName) + .setSettings(Settings.builder() + .put("index.number_of_shards", 5) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "whitespace") + .putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "my_multiplexer") + .put("analysis.filter.synonym_filter.type", "synonym") + .put("analysis.filter.synonym_filter.updateable", "true") + .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName) + .put("analysis.filter.my_multiplexer.type", "multiplexer") + .putList("analysis.filter.my_multiplexer.filters", "synonym_filter")) + .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName)); + + client().prepareIndex(indexName, "_doc", "1").setSource("field", "foo").get(); + assertNoFailures(client().admin().indices().prepareRefresh(indexName).execute().actionGet()); + + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 0L); + + Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get(); + assertEquals(2, analyzeResponse.getTokens().size()); + final Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + + // now update synonyms file and trigger reloading + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, buzz"); + } + ReloadAnalyzersResponse reloadResponse = client().execute(ReloadAnalyzerAction.INSTANCE, new ReloadAnalyzersRequest(indexName)) + .actionGet(); + assertNoFailures(reloadResponse); + Set reloadedAnalyzers = reloadResponse.getReloadDetails().get(indexName).getReloadedAnalyzers(); + assertEquals(1, reloadedAnalyzers.size()); + assertTrue(reloadedAnalyzers.contains(synonymAnalyzerName)); + + analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get(); + assertEquals(3, analyzeResponse.getTokens().size()); + tokens.clear(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains("buzz")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 1L); + } + public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundException, IOException { String synonymsFileName = "synonyms.txt"; + setupSynonymsFile(synonymsFileName, "foo, baz"); Path configDir = node().getEnvironment().configFile(); if (Files.exists(configDir) == false) { Files.createDirectory(configDir); @@ -143,20 +192,53 @@ public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase { final String indexName = "test"; final String analyzerName = "my_synonym_analyzer"; + MapperException ex = expectThrows(MapperException.class, () -> client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put("index.number_of_shards", 5) - .put("index.number_of_replicas", 0) - .put("analysis.analyzer." + analyzerName + ".tokenizer", "standard") - .putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter") - .put("analysis.filter.synonym_filter.type", "synonym") - .put("analysis.filter.synonym_filter.updateable", "true") - .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)) + .put("index.number_of_shards", 5) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer." + analyzerName + ".tokenizer", "standard") + .putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter") + .put("analysis.filter.synonym_filter.type", "synonym") + .put("analysis.filter.synonym_filter.updateable", "true") + .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)) .addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get()); - assertEquals( - "Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] " - + "contains filters [synonym_filter] that are not allowed to run in all mode.", - ex.getMessage()); + assertEquals("Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] " + + "contains filters [synonym_filter] that are not allowed to run in all mode.", ex.getMessage()); + + // same for synonym filters in multiplexer chain + ex = expectThrows(MapperException.class, + () -> client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder() + .put("index.number_of_shards", 5) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer." + analyzerName + ".tokenizer", "whitespace") + .putList("analysis.analyzer." + analyzerName + ".filter", "my_multiplexer") + .put("analysis.filter.synonym_filter.type", "synonym") + .put("analysis.filter.synonym_filter.updateable", "true") + .put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName) + .put("analysis.filter.my_multiplexer.type", "multiplexer") + .putList("analysis.filter.my_multiplexer.filters", "synonym_filter")) + .addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get()); + + assertEquals("Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] " + + "contains filters [my_multiplexer] that are not allowed to run in all mode.", ex.getMessage()); } -} \ No newline at end of file + + private Path setupSynonymsFile(String synonymsFileName, String content) throws IOException { + Path configDir = node().getEnvironment().configFile(); + if (Files.exists(configDir) == false) { + Files.createDirectory(configDir); + } + Path synonymsFile = configDir.resolve(synonymsFileName); + if (Files.exists(synonymsFile) == false) { + Files.createFile(synonymsFile); + } + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println(content); + } + return synonymsFile; + } + +}