diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index b9d3891c751..ec837e980e5 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -25,6 +25,8 @@
+
+
highlighting (type=" + highlighterType + ") and searching on field1");
- SearchSourceBuilder source = searchSource()
- .query(matchQuery("field1", "quick brown fox").operator(Operator.AND))
- .highlighter(
- highlight()
- .field("field1")
- .order("score")
- .preTags("")
- .postTags("")
- .highlighterType(highlighterType));
+ SearchSourceBuilder source = searchSource().query(matchQuery("field1", "quick brown fox").operator(Operator.AND))
+ .highlighter(highlight().field("field1").order("score").preTags("").postTags("").highlighterType(highlighterType));
SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field1", 0, 1,
- equalTo("The quick brown fox jumps over the lazy dog"));
+ assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
- source = searchSource()
- .query(matchQuery("field1", "fast brown fox").operator(Operator.AND))
+ source = searchSource().query(matchQuery("field1", "fast brown fox").operator(Operator.AND))
.highlighter(highlight().field("field1").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("test").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field1", 0, 1,
- equalTo("The quick brown fox jumps over the lazy dog"));
+ assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
}
}
@@ -230,93 +264,139 @@ public class HighlighterWithAnalyzersTests extends OpenSearchIntegTestCase {
ensureGreen();
- client().prepareIndex("first_test_index", "type1", "0").setSource(
- "field0", "The quick brown fox jumps over the lazy dog",
- "field1", "The quick brown fox jumps over the lazy dog").get();
- client().prepareIndex("first_test_index", "type1", "1").setSource("field1",
- "The quick browse button is a fancy thing, right bro?").get();
+ client().prepareIndex("first_test_index", "type1", "0")
+ .setSource("field0", "The quick brown fox jumps over the lazy dog", "field1", "The quick brown fox jumps over the lazy dog")
+ .get();
+ client().prepareIndex("first_test_index", "type1", "1")
+ .setSource("field1", "The quick browse button is a fancy thing, right bro?")
+ .get();
refresh();
logger.info("--> highlighting and searching on field0");
- SearchSourceBuilder source = searchSource()
- .query(matchPhrasePrefixQuery("field0", "bro"))
+ SearchSourceBuilder source = searchSource().query(matchPhrasePrefixQuery("field0", "bro"))
.highlighter(highlight().field("field0").order("score").preTags("").postTags(""));
SearchResponse searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
- source = searchSource()
- .query(matchPhrasePrefixQuery("field0", "quick bro"))
+ source = searchSource().query(matchPhrasePrefixQuery("field0", "quick bro"))
.highlighter(highlight().field("field0").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field0", 0, 1,
- equalTo("The quick brown fox jumps over the lazy dog"));
+ assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
logger.info("--> highlighting and searching on field1");
- source = searchSource()
- .query(boolQuery()
- .should(matchPhrasePrefixQuery("field1", "test"))
- .should(matchPhrasePrefixQuery("field1", "bro"))
- )
- .highlighter(highlight().field("field1").order("score").preTags("").postTags(""));
+ source = searchSource().query(
+ boolQuery().should(matchPhrasePrefixQuery("field1", "test")).should(matchPhrasePrefixQuery("field1", "bro"))
+ ).highlighter(highlight().field("field1").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
for (int i = 0; i < 2; i++) {
- assertHighlight(searchResponse, i, "field1", 0, 1, anyOf(
- equalTo("The quick browse button is a fancy thing, right bro?"),
- equalTo("The quick brown fox jumps over the lazy dog")));
+ assertHighlight(
+ searchResponse,
+ i,
+ "field1",
+ 0,
+ 1,
+ anyOf(
+ equalTo("The quick browse button is a fancy thing, right bro?"),
+ equalTo("The quick brown fox jumps over the lazy dog")
+ )
+ );
}
- source = searchSource()
- .query(matchPhrasePrefixQuery("field1", "quick bro"))
+ source = searchSource().query(matchPhrasePrefixQuery("field1", "quick bro"))
.highlighter(highlight().field("field1").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
- equalTo("The quick browse button is a fancy thing, right bro?"),
- equalTo("The quick brown fox jumps over the lazy dog")));
- assertHighlight(searchResponse, 1, "field1", 0, 1, anyOf(
- equalTo("The quick browse button is a fancy thing, right bro?"),
- equalTo("The quick brown fox jumps over the lazy dog")));
+ assertHighlight(
+ searchResponse,
+ 0,
+ "field1",
+ 0,
+ 1,
+ anyOf(
+ equalTo("The quick browse button is a fancy thing, right bro?"),
+ equalTo("The quick brown fox jumps over the lazy dog")
+ )
+ );
+ assertHighlight(
+ searchResponse,
+ 1,
+ "field1",
+ 0,
+ 1,
+ anyOf(
+ equalTo("The quick browse button is a fancy thing, right bro?"),
+ equalTo("The quick brown fox jumps over the lazy dog")
+ )
+ );
- assertAcked(prepareCreate("second_test_index").setSettings(builder.build()).addMapping("doc",
- "field4", "type=text,term_vector=with_positions_offsets,analyzer=synonym",
- "field3", "type=text,analyzer=synonym"));
+ assertAcked(
+ prepareCreate("second_test_index").setSettings(builder.build())
+ .addMapping(
+ "doc",
+ "field4",
+ "type=text,term_vector=with_positions_offsets,analyzer=synonym",
+ "field3",
+ "type=text,analyzer=synonym"
+ )
+ );
// with synonyms
- client().prepareIndex("second_test_index", "doc", "0").setSource(
- "type", "type2",
- "field4", "The quick brown fox jumps over the lazy dog",
- "field3", "The quick brown fox jumps over the lazy dog").get();
- client().prepareIndex("second_test_index", "doc", "1").setSource(
- "type", "type2",
- "field4", "The quick browse button is a fancy thing, right bro?").get();
- client().prepareIndex("second_test_index", "doc", "2").setSource(
- "type", "type2",
- "field4", "a quick fast blue car").get();
+ client().prepareIndex("second_test_index", "doc", "0")
+ .setSource(
+ "type",
+ "type2",
+ "field4",
+ "The quick brown fox jumps over the lazy dog",
+ "field3",
+ "The quick brown fox jumps over the lazy dog"
+ )
+ .get();
+ client().prepareIndex("second_test_index", "doc", "1")
+ .setSource("type", "type2", "field4", "The quick browse button is a fancy thing, right bro?")
+ .get();
+ client().prepareIndex("second_test_index", "doc", "2").setSource("type", "type2", "field4", "a quick fast blue car").get();
refresh();
- source = searchSource().postFilter(termQuery("type", "type2")).query(matchPhrasePrefixQuery("field3", "fast bro"))
+ source = searchSource().postFilter(termQuery("type", "type2"))
+ .query(matchPhrasePrefixQuery("field3", "fast bro"))
.highlighter(highlight().field("field3").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("second_test_index").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field3", 0, 1,
- equalTo("The quick brown fox jumps over the lazy dog"));
+ assertHighlight(searchResponse, 0, "field3", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
logger.info("--> highlighting and searching on field4");
- source = searchSource().postFilter(termQuery("type", "type2")).query(matchPhrasePrefixQuery("field4", "the fast bro"))
+ source = searchSource().postFilter(termQuery("type", "type2"))
+ .query(matchPhrasePrefixQuery("field4", "the fast bro"))
.highlighter(highlight().field("field4").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("second_test_index").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(
- equalTo("The quick browse button is a fancy thing, right bro?"),
- equalTo("The quick brown fox jumps over the lazy dog")));
- assertHighlight(searchResponse, 1, "field4", 0, 1, anyOf(
- equalTo("The quick browse button is a fancy thing, right bro?"),
- equalTo("The quick brown fox jumps over the lazy dog")));
+ assertHighlight(
+ searchResponse,
+ 0,
+ "field4",
+ 0,
+ 1,
+ anyOf(
+ equalTo("The quick browse button is a fancy thing, right bro?"),
+ equalTo("The quick brown fox jumps over the lazy dog")
+ )
+ );
+ assertHighlight(
+ searchResponse,
+ 1,
+ "field4",
+ 0,
+ 1,
+ anyOf(
+ equalTo("The quick browse button is a fancy thing, right bro?"),
+ equalTo("The quick brown fox jumps over the lazy dog")
+ )
+ );
logger.info("--> highlighting and searching on field4");
source = searchSource().postFilter(termQuery("type", "type2"))
@@ -324,17 +404,31 @@ public class HighlighterWithAnalyzersTests extends OpenSearchIntegTestCase {
.highlighter(highlight().field("field4").order("score").preTags("").postTags(""));
searchResponse = client().search(searchRequest("second_test_index").source(source)).actionGet();
- assertHighlight(searchResponse, 0, "field4", 0, 1,
- anyOf(equalTo("a quick fast blue car"),
- equalTo("a quick fast blue car")));
+ assertHighlight(
+ searchResponse,
+ 0,
+ "field4",
+ 0,
+ 1,
+ anyOf(equalTo("a quick fast blue car"), equalTo("a quick fast blue car"))
+ );
}
public static XContentBuilder type1TermVectorMapping() throws IOException {
- return XContentFactory.jsonBuilder().startObject().startObject("type1")
+ return XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("type1")
.startObject("properties")
- .startObject("field1").field("type", "text").field("term_vector", "with_positions_offsets").endObject()
- .startObject("field2").field("type", "text").field("term_vector", "with_positions_offsets").endObject()
+ .startObject("field1")
+ .field("type", "text")
+ .field("term_vector", "with_positions_offsets")
.endObject()
- .endObject().endObject();
+ .startObject("field2")
+ .field("type", "text")
+ .field("term_vector", "with_positions_offsets")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject();
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HtmlStripCharFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HtmlStripCharFilterFactoryTests.java
index 1e5cd039b53..63a7ffdfc18 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HtmlStripCharFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HtmlStripCharFilterFactoryTests.java
@@ -47,25 +47,29 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
-
public class HtmlStripCharFilterFactoryTests extends OpenSearchTestCase {
/**
* Check that the deprecated name "htmlStrip" issues a deprecation warning for indices created since 6.3.0
*/
public void testDeprecationWarning() throws IOException {
- Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
- .put(IndexMetadata.SETTING_VERSION_CREATED,
- VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_6_3_0, Version.CURRENT))
- .build();
+ Settings settings = Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
+ VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_6_3_0, Version.CURRENT)
+ )
+ .build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
Map charFilters = createTestAnalysis(idxSettings, settings, commonAnalysisPlugin).charFilter;
CharFilterFactory charFilterFactory = charFilters.get("htmlStrip");
assertNotNull(charFilterFactory.create(new StringReader("input")));
- assertWarnings("The [htmpStrip] char filter name is deprecated and will be removed in a future version. "
- + "Please change the filter name to [html_strip] instead.");
+ assertWarnings(
+ "The [htmpStrip] char filter name is deprecated and will be removed in a future version. "
+ + "Please change the filter name to [html_strip] instead."
+ );
}
}
@@ -73,10 +77,13 @@ public class HtmlStripCharFilterFactoryTests extends OpenSearchTestCase {
* Check that the deprecated name "htmlStrip" does NOT issues a deprecation warning for indices created before 6.3.0
*/
public void testNoDeprecationWarningPre6_3() throws IOException {
- Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
- .put(IndexMetadata.SETTING_VERSION_CREATED,
- VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_6_0_0, LegacyESVersion.V_6_2_4))
- .build();
+ Settings settings = Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
+ VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_6_0_0, LegacyESVersion.V_6_2_4)
+ )
+ .build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepFilterFactoryTests.java
index e12bd3185f7..0b094e52df8 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepFilterFactoryTests.java
@@ -52,51 +52,49 @@ public class KeepFilterFactoryTests extends OpenSearchTokenStreamTestCase {
public void testLoadWithoutSettings() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(
- createTempDir(), RESOURCE, new CommonAnalysisPlugin());
+ createTempDir(),
+ RESOURCE,
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep");
Assert.assertNull(tokenFilter);
}
public void testLoadOverConfiguredSettings() {
Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put("index.analysis.filter.broken_keep_filter.type", "keep")
- .put("index.analysis.filter.broken_keep_filter.keep_words_path", "does/not/exists.txt")
- .put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
- .build();
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put("index.analysis.filter.broken_keep_filter.type", "keep")
+ .put("index.analysis.filter.broken_keep_filter.keep_words_path", "does/not/exists.txt")
+ .put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
+ .build();
try {
AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
Assert.fail("path and array are configured");
- } catch (IllegalArgumentException e) {
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {} catch (IOException e) {
fail("expected IAE");
}
}
public void testKeepWordsPathSettings() {
Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put("index.analysis.filter.non_broken_keep_filter.type", "keep")
- .put("index.analysis.filter.non_broken_keep_filter.keep_words_path", "does/not/exists.txt")
- .build();
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put("index.analysis.filter.non_broken_keep_filter.type", "keep")
+ .put("index.analysis.filter.non_broken_keep_filter.keep_words_path", "does/not/exists.txt")
+ .build();
try {
// test our none existing setup is picked up
AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
fail("expected an exception due to non existent keep_words_path");
- } catch (IllegalArgumentException e) {
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {} catch (IOException e) {
fail("expected IAE");
}
- settings = Settings.builder().put(settings)
- .putList("index.analysis.filter.non_broken_keep_filter.keep_words", "test")
- .build();
+ settings = Settings.builder().put(settings).putList("index.analysis.filter.non_broken_keep_filter.keep_words", "test").build();
try {
// test our none existing setup is picked up
AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
fail("expected an exception indicating that you can't use [keep_words_path] with [keep_words] ");
- } catch (IllegalArgumentException e) {
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {} catch (IOException e) {
fail("expected IAE");
}
@@ -104,25 +102,31 @@ public class KeepFilterFactoryTests extends OpenSearchTokenStreamTestCase {
public void testCaseInsensitiveMapping() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(
- createTempDir(), RESOURCE, new CommonAnalysisPlugin());
+ createTempDir(),
+ RESOURCE,
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_keep_filter");
assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
String source = "hello small world";
- String[] expected = new String[]{"hello", "world"};
+ String[] expected = new String[] { "hello", "world" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
- assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[]{1, 2});
+ assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[] { 1, 2 });
}
public void testCaseSensitiveMapping() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(
- createTempDir(), RESOURCE, new CommonAnalysisPlugin());
+ createTempDir(),
+ RESOURCE,
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_case_sensitive_keep_filter");
assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
String source = "Hello small world";
- String[] expected = new String[]{"Hello"};
+ String[] expected = new String[] { "Hello" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
- assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[]{1});
+ assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[] { 1 });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepTypesFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepTypesFilterFactoryTests.java
index 26c4516dc7d..1f1021b4bfe 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepTypesFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeepTypesFilterFactoryTests.java
@@ -51,13 +51,16 @@ public class KeepTypesFilterFactoryTests extends OpenSearchTokenStreamTestCase {
private static final String BASE_SETTING = "index.analysis.filter.keep_numbers";
public void testKeepTypesInclude() throws IOException {
- Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put(BASE_SETTING + ".type", "keep_types")
- .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" });
+ Settings.Builder settingsBuilder = Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put(BASE_SETTING + ".type", "keep_types")
+ .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" });
// either use default mode or set "include" mode explicitly
if (random().nextBoolean()) {
- settingsBuilder.put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY,
- KeepTypesFilterFactory.KeepTypesMode.INCLUDE);
+ settingsBuilder.put(
+ BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY,
+ KeepTypesFilterFactory.KeepTypesMode.INCLUDE
+ );
}
Settings settings = settingsBuilder.build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
@@ -71,10 +74,12 @@ public class KeepTypesFilterFactoryTests extends OpenSearchTokenStreamTestCase {
}
public void testKeepTypesExclude() throws IOException {
- Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put(BASE_SETTING + ".type", "keep_types")
- .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" })
- .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, KeepTypesFilterFactory.KeepTypesMode.EXCLUDE).build();
+ Settings settings = Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put(BASE_SETTING + ".type", "keep_types")
+ .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" })
+ .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, KeepTypesFilterFactory.KeepTypesMode.EXCLUDE)
+ .build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep_numbers");
assertThat(tokenFilter, instanceOf(KeepTypesFilterFactory.class));
@@ -86,12 +91,16 @@ public class KeepTypesFilterFactoryTests extends OpenSearchTokenStreamTestCase {
}
public void testKeepTypesException() throws IOException {
- Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put(BASE_SETTING + ".type", "keep_types")
- .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" })
- .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, "bad_parameter").build();
- IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
- () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin()));
+ Settings settings = Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put(BASE_SETTING + ".type", "keep_types")
+ .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "", "" })
+ .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, "bad_parameter")
+ .build();
+ IllegalArgumentException ex = expectThrows(
+ IllegalArgumentException.class,
+ () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin())
+ );
assertEquals("`keep_types` tokenfilter mode can only be [include] or [exclude] but was [bad_parameter].", ex.getMessage());
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeywordMarkerFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeywordMarkerFilterFactoryTests.java
index a4507729fa2..40e354785dd 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeywordMarkerFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/KeywordMarkerFilterFactoryTests.java
@@ -72,8 +72,7 @@ public class KeywordMarkerFilterFactoryTests extends OpenSearchTokenStreamTestCa
assertThat(filter, instanceOf(SetKeywordMarkerFilter.class));
NamedAnalyzer analyzer = analysis.indexAnalyzers.get("my_keyword");
// jogging is not part of the keywords set, so verify that its the only stemmed word
- assertAnalyzesTo(analyzer, "running jogging sleeping",
- new String[] { "running", "jog", "sleeping" });
+ assertAnalyzesTo(analyzer, "running jogging sleeping", new String[] { "running", "jog", "sleeping" });
}
/**
@@ -111,9 +110,10 @@ public class KeywordMarkerFilterFactoryTests extends OpenSearchTokenStreamTestCa
.put("index.analysis.analyzer.my_keyword.filter", "my_keyword, porter_stem")
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin()));
- assertEquals("cannot specify both `keywords_pattern` and `keywords` or `keywords_path`",
- e.getMessage());
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin())
+ );
+ assertEquals("cannot specify both `keywords_pattern` and `keywords` or `keywords_path`", e.getMessage());
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/LimitTokenCountFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/LimitTokenCountFilterFactoryTests.java
index bece55849ad..76471fd98e5 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/LimitTokenCountFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/LimitTokenCountFilterFactoryTests.java
@@ -47,9 +47,9 @@ import java.io.StringReader;
public class LimitTokenCountFilterFactoryTests extends OpenSearchTokenStreamTestCase {
public void testDefault() throws IOException {
Settings settings = Settings.builder()
- .put("index.analysis.filter.limit_default.type", "limit")
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.limit_default.type", "limit")
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = createTestAnalysisFromSettings(settings);
{
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_default");
@@ -72,11 +72,11 @@ public class LimitTokenCountFilterFactoryTests extends OpenSearchTokenStreamTest
public void testSettings() throws IOException {
{
Settings settings = Settings.builder()
- .put("index.analysis.filter.limit_1.type", "limit")
- .put("index.analysis.filter.limit_1.max_token_count", 3)
- .put("index.analysis.filter.limit_1.consume_all_tokens", true)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.limit_1.type", "limit")
+ .put("index.analysis.filter.limit_1.max_token_count", 3)
+ .put("index.analysis.filter.limit_1.consume_all_tokens", true)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = createTestAnalysisFromSettings(settings);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
String source = "the quick brown fox";
@@ -87,11 +87,11 @@ public class LimitTokenCountFilterFactoryTests extends OpenSearchTokenStreamTest
}
{
Settings settings = Settings.builder()
- .put("index.analysis.filter.limit_1.type", "limit")
- .put("index.analysis.filter.limit_1.max_token_count", 3)
- .put("index.analysis.filter.limit_1.consume_all_tokens", false)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.limit_1.type", "limit")
+ .put("index.analysis.filter.limit_1.max_token_count", 3)
+ .put("index.analysis.filter.limit_1.consume_all_tokens", false)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = createTestAnalysisFromSettings(settings);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
String source = "the quick brown fox";
@@ -103,11 +103,11 @@ public class LimitTokenCountFilterFactoryTests extends OpenSearchTokenStreamTest
{
Settings settings = Settings.builder()
- .put("index.analysis.filter.limit_1.type", "limit")
- .put("index.analysis.filter.limit_1.max_token_count", 17)
- .put("index.analysis.filter.limit_1.consume_all_tokens", true)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.limit_1.type", "limit")
+ .put("index.analysis.filter.limit_1.max_token_count", 17)
+ .put("index.analysis.filter.limit_1.consume_all_tokens", true)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = createTestAnalysisFromSettings(settings);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("limit_1");
String source = "the quick brown fox";
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MassiveWordListTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MassiveWordListTests.java
index a89693ce609..390e36c4ca0 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MassiveWordListTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MassiveWordListTests.java
@@ -51,13 +51,18 @@ public class MassiveWordListTests extends OpenSearchSingleNodeTestCase {
for (int i = 0; i < wordList.length; i++) {
wordList[i] = "hello world";
}
- client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
- .put("index.number_of_shards", 1)
- .put("analysis.analyzer.test_analyzer.type", "custom")
- .put("analysis.analyzer.test_analyzer.tokenizer", "standard")
- .putList("analysis.analyzer.test_analyzer.filter", "dictionary_decompounder", "lowercase")
- .put("analysis.filter.dictionary_decompounder.type", "dictionary_decompounder")
- .putList("analysis.filter.dictionary_decompounder.word_list", wordList)
- ).get();
+ client().admin()
+ .indices()
+ .prepareCreate("test")
+ .setSettings(
+ Settings.builder()
+ .put("index.number_of_shards", 1)
+ .put("analysis.analyzer.test_analyzer.type", "custom")
+ .put("analysis.analyzer.test_analyzer.tokenizer", "standard")
+ .putList("analysis.analyzer.test_analyzer.filter", "dictionary_decompounder", "lowercase")
+ .put("analysis.filter.dictionary_decompounder.type", "dictionary_decompounder")
+ .putList("analysis.filter.dictionary_decompounder.word_list", wordList)
+ )
+ .get();
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MinHashFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MinHashFilterFactoryTests.java
index a16d74dbfa9..514c53f1745 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MinHashFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MinHashFilterFactoryTests.java
@@ -49,9 +49,7 @@ public class MinHashFilterFactoryTests extends OpenSearchTokenStreamTestCase {
int default_hash_count = 1;
int default_bucket_size = 512;
int default_hash_set_size = 1;
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("min_hash");
String source = "the quick brown fox";
@@ -60,8 +58,7 @@ public class MinHashFilterFactoryTests extends OpenSearchTokenStreamTestCase {
// with_rotation is true by default, and hash_set_size is 1, so even though the source doesn't
// have enough tokens to fill all the buckets, we still expect 512 tokens.
- assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer),
- default_hash_count * default_bucket_size * default_hash_set_size);
+ assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), default_hash_count * default_bucket_size * default_hash_set_size);
}
public void testSettings() throws IOException {
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MultiplexerTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MultiplexerTokenFilterTests.java
index 922c10e8e85..167f61464da 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MultiplexerTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MultiplexerTokenFilterTests.java
@@ -50,9 +50,7 @@ import java.util.Collections;
public class MultiplexerTokenFilterTests extends OpenSearchTokenStreamTestCase {
public void testMultiplexingFilter() throws IOException {
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.filter.t.type", "truncate")
@@ -65,30 +63,27 @@ public class MultiplexerTokenFilterTests extends OpenSearchTokenStreamTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
- IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
- Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings);
+ IndexAnalyzers indexAnalyzers = new AnalysisModule(
+ TestEnvironment.newEnvironment(settings),
+ Collections.singletonList(new CommonAnalysisPlugin())
+ ).getAnalysisRegistry().build(idxSettings);
try (NamedAnalyzer analyzer = indexAnalyzers.get("myAnalyzer")) {
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "ONe tHree", new String[]{
- "ONe", "on", "ONE", "tHree", "th", "THREE"
- }, new int[]{
- 1, 0, 0, 1, 0, 0
- });
+ assertAnalyzesTo(
+ analyzer,
+ "ONe tHree",
+ new String[] { "ONe", "on", "ONE", "tHree", "th", "THREE" },
+ new int[] { 1, 0, 0, 1, 0, 0 }
+ );
// Duplicates are removed
- assertAnalyzesTo(analyzer, "ONe THREE", new String[]{
- "ONe", "on", "ONE", "THREE", "th"
- }, new int[]{
- 1, 0, 0, 1, 0, 0
- });
+ assertAnalyzesTo(analyzer, "ONe THREE", new String[] { "ONe", "on", "ONE", "THREE", "th" }, new int[] { 1, 0, 0, 1, 0, 0 });
}
}
public void testMultiplexingNoOriginal() throws IOException {
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.filter.t.type", "truncate")
@@ -102,16 +97,14 @@ public class MultiplexerTokenFilterTests extends OpenSearchTokenStreamTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
- IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
- Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings);
+ IndexAnalyzers indexAnalyzers = new AnalysisModule(
+ TestEnvironment.newEnvironment(settings),
+ Collections.singletonList(new CommonAnalysisPlugin())
+ ).getAnalysisRegistry().build(idxSettings);
try (NamedAnalyzer analyzer = indexAnalyzers.get("myAnalyzer")) {
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "ONe tHree", new String[]{
- "on", "ONE", "th", "THREE"
- }, new int[]{
- 1, 0, 1, 0,
- });
+ assertAnalyzesTo(analyzer, "ONe tHree", new String[] { "on", "ONE", "th", "THREE" }, new int[] { 1, 0, 1, 0, });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenFilterFactoryTests.java
index 45ac5f58138..85090648096 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenFilterFactoryTests.java
@@ -52,10 +52,11 @@ public class NGramTokenFilterFactoryTests extends OpenSearchTokenStreamTestCase
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put("index.analysis.filter.my_ngram.type", "ngram")
.build(),
- new CommonAnalysisPlugin());
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ngram");
String source = "foo";
- String[] expected = new String[]{"f", "fo", "o", "oo", "o"};
+ String[] expected = new String[] { "f", "fo", "o", "oo", "o" };
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
@@ -68,10 +69,11 @@ public class NGramTokenFilterFactoryTests extends OpenSearchTokenStreamTestCase
.put("index.analysis.filter.my_ngram.type", "ngram")
.put("index.analysis.filter.my_ngram.preserve_original", true)
.build(),
- new CommonAnalysisPlugin());
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ngram");
String source = "foo";
- String[] expected = new String[]{"f", "fo", "o", "oo", "o", "foo"};
+ String[] expected = new String[] { "f", "fo", "o", "oo", "o", "foo" };
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java
index 656c8afb27f..c23b2c6082e 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java
@@ -61,30 +61,43 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
final Settings indexSettings = newAnalysisSettingsBuilder().build();
final IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(index, indexSettings);
for (String tokenChars : Arrays.asList("letter", " digit ", "punctuation", "DIGIT", "CoNtRoL", "dash_punctuation")) {
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .put("token_chars", tokenChars).build();
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", tokenChars)
+ .build();
new NGramTokenizerFactory(indexProperties, null, name, settings).create();
// no exception
}
{
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .put("token_chars", "DIRECTIONALITY_UNDEFINED").build();
- IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
- () -> new NGramTokenizerFactory(indexProperties, null, name, settings).create());
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", "DIRECTIONALITY_UNDEFINED")
+ .build();
+ IllegalArgumentException ex = expectThrows(
+ IllegalArgumentException.class,
+ () -> new NGramTokenizerFactory(indexProperties, null, name, settings).create()
+ );
assertEquals("Unknown token type: 'directionality_undefined'", ex.getMessage().substring(0, 46));
assertTrue(ex.getMessage().contains("custom"));
}
{
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "custom")
- .put("custom_token_chars", "_-").build();
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", "custom")
+ .put("custom_token_chars", "_-")
+ .build();
new NGramTokenizerFactory(indexProperties, null, name, settings).create();
// no exception
}
{
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "custom")
- .build();
- IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
- () -> new NGramTokenizerFactory(indexProperties, null, name, settings).create());
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", "custom")
+ .build();
+ IllegalArgumentException ex = expectThrows(
+ IllegalArgumentException.class,
+ () -> new NGramTokenizerFactory(indexProperties, null, name, settings).create()
+ );
assertEquals("Token type: 'custom' requires setting `custom_token_chars`", ex.getMessage());
}
}
@@ -94,12 +107,14 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
final String name = "ngr";
final Settings indexSettings = newAnalysisSettingsBuilder().put(IndexSettings.MAX_NGRAM_DIFF_SETTING.getKey(), 2).build();
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 4)
- .putList("token_chars", new String[0]).build();
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 4)
+ .putList("token_chars", new String[0])
+ .build();
Tokenizer tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings)
.create();
tokenizer.setReader(new StringReader("1.34"));
- assertTokenStreamContents(tokenizer, new String[] {"1.", "1.3", "1.34", ".3", ".34", "34"});
+ assertTokenStreamContents(tokenizer, new String[] { "1.", "1.3", "1.34", ".3", ".34", "34" });
}
public void testCustomTokenChars() throws IOException {
@@ -107,12 +122,15 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
final String name = "ngr";
final Settings indexSettings = newAnalysisSettingsBuilder().put(IndexSettings.MAX_NGRAM_DIFF_SETTING.getKey(), 2).build();
- final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .putList("token_chars", "letter", "custom").put("custom_token_chars","_-").build();
+ final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .putList("token_chars", "letter", "custom")
+ .put("custom_token_chars", "_-")
+ .build();
Tokenizer tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings)
.create();
tokenizer.setReader(new StringReader("Abc -gh _jk =lm"));
- assertTokenStreamContents(tokenizer, new String[] {"Ab", "Abc", "bc", "-g", "-gh", "gh", "_j", "_jk", "jk", "lm"});
+ assertTokenStreamContents(tokenizer, new String[] { "Ab", "Abc", "bc", "-g", "-gh", "gh", "_j", "_jk", "jk", "lm" });
}
public void testPreTokenization() throws IOException {
@@ -120,19 +138,21 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
final Index index = new Index("test", "_na_");
final String name = "ngr";
final Settings indexSettings = newAnalysisSettingsBuilder().build();
- Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .put("token_chars", "letter,digit").build();
+ Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit").build();
Tokenizer tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings)
.create();
tokenizer.setReader(new StringReader("Åbc déf g\uD801\uDC00f "));
- assertTokenStreamContents(tokenizer,
- new String[] {"Åb", "Åbc", "bc", "dé", "déf", "éf", "g\uD801\uDC00", "g\uD801\uDC00f", "\uD801\uDC00f"});
- settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .put("token_chars", "letter,digit,punctuation,whitespace,symbol").build();
+ assertTokenStreamContents(
+ tokenizer,
+ new String[] { "Åb", "Åbc", "bc", "dé", "déf", "éf", "g\uD801\uDC00", "g\uD801\uDC00f", "\uD801\uDC00f" }
+ );
+ settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", "letter,digit,punctuation,whitespace,symbol")
+ .build();
tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create();
tokenizer.setReader(new StringReader(" a!$ 9"));
- assertTokenStreamContents(tokenizer,
- new String[] {" a", " a!", "a!", "a!$", "!$", "!$ ", "$ ", "$ 9", " 9"});
+ assertTokenStreamContents(tokenizer, new String[] { " a", " a!", "a!", "a!$", "!$", "!$ ", "$ ", "$ 9", " 9" });
}
public void testPreTokenizationEdge() throws IOException {
@@ -141,18 +161,22 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
final String name = "ngr";
final Settings indexSettings = newAnalysisSettingsBuilder().build();
Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit").build();
- Tokenizer tokenizer =
- new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create();
+ Tokenizer tokenizer = new EdgeNGramTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ name,
+ settings
+ ).create();
tokenizer.setReader(new StringReader("Åbc déf g\uD801\uDC00f "));
- assertTokenStreamContents(tokenizer,
- new String[] {"Åb", "Åbc", "dé", "déf", "g\uD801\uDC00", "g\uD801\uDC00f"});
- settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3)
- .put("token_chars", "letter,digit,punctuation,whitespace,symbol").build();
+ assertTokenStreamContents(tokenizer, new String[] { "Åb", "Åbc", "dé", "déf", "g\uD801\uDC00", "g\uD801\uDC00f" });
+ settings = newAnalysisSettingsBuilder().put("min_gram", 2)
+ .put("max_gram", 3)
+ .put("token_chars", "letter,digit,punctuation,whitespace,symbol")
+ .build();
tokenizer = new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings)
.create();
tokenizer.setReader(new StringReader(" a!$ 9"));
- assertTokenStreamContents(tokenizer,
- new String[] {" a", " a!"});
+ assertTokenStreamContents(tokenizer, new String[] { " a", " a!" });
}
public void testBackwardsCompatibilityEdgeNgramTokenFilter() throws Exception {
@@ -170,9 +194,12 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetadata.SETTING_VERSION_CREATED, v.id).build();
Tokenizer tokenizer = new MockTokenizer();
tokenizer.setReader(new StringReader("foo bar"));
- TokenStream edgeNGramTokenFilter =
- new EdgeNGramTokenFilterFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings)
- .create(tokenizer);
+ TokenStream edgeNGramTokenFilter = new EdgeNGramTokenFilterFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ name,
+ settings
+ ).create(tokenizer);
if (reverse) {
assertThat(edgeNGramTokenFilter, instanceOf(ReverseStringFilter.class));
} else {
@@ -185,7 +212,7 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
* test that throws an error when trying to get a NGramTokenizer where difference between max_gram and min_gram
* is greater than the allowed value of max_ngram_diff
*/
- public void testMaxNGramDiffException() throws Exception{
+ public void testMaxNGramDiffException() throws Exception {
final Index index = new Index("test", "_na_");
final String name = "ngr";
final Settings indexSettings = newAnalysisSettingsBuilder().build();
@@ -197,12 +224,19 @@ public class NGramTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
int max_gram = min_gram + ngramDiff;
final Settings settings = newAnalysisSettingsBuilder().put("min_gram", min_gram).put("max_gram", max_gram).build();
- IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
- new NGramTokenizerFactory(indexProperties, null, name, settings).create());
+ IllegalArgumentException ex = expectThrows(
+ IllegalArgumentException.class,
+ () -> new NGramTokenizerFactory(indexProperties, null, name, settings).create()
+ );
assertEquals(
"The difference between max_gram and min_gram in NGram Tokenizer must be less than or equal to: ["
- + maxAllowedNgramDiff + "] but was [" + ngramDiff + "]. This limit can be set by changing the ["
- + IndexSettings.MAX_NGRAM_DIFF_SETTING.getKey() + "] index level setting.",
- ex.getMessage());
+ + maxAllowedNgramDiff
+ + "] but was ["
+ + ngramDiff
+ + "]. This limit can be set by changing the ["
+ + IndexSettings.MAX_NGRAM_DIFF_SETTING.getKey()
+ + "] index level setting.",
+ ex.getMessage()
+ );
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchSolrSynonymParserTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchSolrSynonymParserTests.java
index 4e3bf297750..4a5ee5d3856 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchSolrSynonymParserTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchSolrSynonymParserTests.java
@@ -52,23 +52,20 @@ public class OpenSearchSolrSynonymParserTests extends OpenSearchTokenStreamTestC
public void testLenientParser() throws IOException, ParseException {
OpenSearchSolrSynonymParser parser = new OpenSearchSolrSynonymParser(true, false, true, new StandardAnalyzer());
- String rules =
- "&,and\n" +
- "come,advance,approach\n";
+ String rules = "&,and\n" + "come,advance,approach\n";
StringReader rulesReader = new StringReader(rules);
parser.parse(rulesReader);
SynonymMap synonymMap = parser.build();
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader("approach quietly then advance & destroy"));
TokenStream ts = new SynonymFilter(tokenizer, synonymMap, false);
- assertTokenStreamContents(ts, new String[]{"come", "quietly", "then", "come", "destroy"});
+ assertTokenStreamContents(ts, new String[] { "come", "quietly", "then", "come", "destroy" });
}
public void testLenientParserWithSomeIncorrectLines() throws IOException, ParseException {
CharArraySet stopSet = new CharArraySet(1, true);
stopSet.add("bar");
- OpenSearchSolrSynonymParser parser =
- new OpenSearchSolrSynonymParser(true, false, true, new StandardAnalyzer(stopSet));
+ OpenSearchSolrSynonymParser parser = new OpenSearchSolrSynonymParser(true, false, true, new StandardAnalyzer(stopSet));
String rules = "foo,bar,baz";
StringReader rulesReader = new StringReader(rules);
parser.parse(rulesReader);
@@ -76,14 +73,12 @@ public class OpenSearchSolrSynonymParserTests extends OpenSearchTokenStreamTestC
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader("first word is foo, then bar and lastly baz"));
TokenStream ts = new SynonymFilter(new StopFilter(tokenizer, stopSet), synonymMap, false);
- assertTokenStreamContents(ts, new String[]{"first", "word", "is", "foo", "then", "and", "lastly", "foo"});
+ assertTokenStreamContents(ts, new String[] { "first", "word", "is", "foo", "then", "and", "lastly", "foo" });
}
public void testNonLenientParser() {
OpenSearchSolrSynonymParser parser = new OpenSearchSolrSynonymParser(true, false, false, new StandardAnalyzer());
- String rules =
- "&,and=>and\n" +
- "come,advance,approach\n";
+ String rules = "&,and=>and\n" + "come,advance,approach\n";
StringReader rulesReader = new StringReader(rules);
ParseException ex = expectThrows(ParseException.class, () -> parser.parse(rulesReader));
assertThat(ex.getMessage(), containsString("Invalid synonym rule at line 1"));
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchWordnetSynonymParserTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchWordnetSynonymParserTests.java
index ac5f8c24056..d74641bd16d 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchWordnetSynonymParserTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/OpenSearchWordnetSynonymParserTests.java
@@ -52,47 +52,41 @@ public class OpenSearchWordnetSynonymParserTests extends OpenSearchTokenStreamTe
public void testLenientParser() throws IOException, ParseException {
OpenSearchWordnetSynonymParser parser = new OpenSearchWordnetSynonymParser(true, false, true, new StandardAnalyzer());
- String rules =
- "s(100000001,1,'&',a,1,0).\n" +
- "s(100000001,2,'and',a,1,0).\n" +
- "s(100000002,1,'come',v,1,0).\n" +
- "s(100000002,2,'advance',v,1,0).\n" +
- "s(100000002,3,'approach',v,1,0).";
+ String rules = "s(100000001,1,'&',a,1,0).\n"
+ + "s(100000001,2,'and',a,1,0).\n"
+ + "s(100000002,1,'come',v,1,0).\n"
+ + "s(100000002,2,'advance',v,1,0).\n"
+ + "s(100000002,3,'approach',v,1,0).";
StringReader rulesReader = new StringReader(rules);
parser.parse(rulesReader);
SynonymMap synonymMap = parser.build();
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader("approach quietly then advance & destroy"));
TokenStream ts = new SynonymFilter(tokenizer, synonymMap, false);
- assertTokenStreamContents(ts, new String[]{"come", "quietly", "then", "come", "destroy"});
+ assertTokenStreamContents(ts, new String[] { "come", "quietly", "then", "come", "destroy" });
}
public void testLenientParserWithSomeIncorrectLines() throws IOException, ParseException {
CharArraySet stopSet = new CharArraySet(1, true);
stopSet.add("bar");
- OpenSearchWordnetSynonymParser parser =
- new OpenSearchWordnetSynonymParser(true, false, true, new StandardAnalyzer(stopSet));
- String rules =
- "s(100000001,1,'foo',v,1,0).\n" +
- "s(100000001,2,'bar',v,1,0).\n" +
- "s(100000001,3,'baz',v,1,0).";
+ OpenSearchWordnetSynonymParser parser = new OpenSearchWordnetSynonymParser(true, false, true, new StandardAnalyzer(stopSet));
+ String rules = "s(100000001,1,'foo',v,1,0).\n" + "s(100000001,2,'bar',v,1,0).\n" + "s(100000001,3,'baz',v,1,0).";
StringReader rulesReader = new StringReader(rules);
parser.parse(rulesReader);
SynonymMap synonymMap = parser.build();
Tokenizer tokenizer = new StandardTokenizer();
tokenizer.setReader(new StringReader("first word is foo, then bar and lastly baz"));
TokenStream ts = new SynonymFilter(new StopFilter(tokenizer, stopSet), synonymMap, false);
- assertTokenStreamContents(ts, new String[]{"first", "word", "is", "foo", "then", "and", "lastly", "foo"});
+ assertTokenStreamContents(ts, new String[] { "first", "word", "is", "foo", "then", "and", "lastly", "foo" });
}
public void testNonLenientParser() {
OpenSearchWordnetSynonymParser parser = new OpenSearchWordnetSynonymParser(true, false, false, new StandardAnalyzer());
- String rules =
- "s(100000001,1,'&',a,1,0).\n" +
- "s(100000001,2,'and',a,1,0).\n" +
- "s(100000002,1,'come',v,1,0).\n" +
- "s(100000002,2,'advance',v,1,0).\n" +
- "s(100000002,3,'approach',v,1,0).";
+ String rules = "s(100000001,1,'&',a,1,0).\n"
+ + "s(100000001,2,'and',a,1,0).\n"
+ + "s(100000002,1,'come',v,1,0).\n"
+ + "s(100000002,2,'advance',v,1,0).\n"
+ + "s(100000002,3,'approach',v,1,0).";
StringReader rulesReader = new StringReader(rules);
ParseException ex = expectThrows(ParseException.class, () -> parser.parse(rulesReader));
assertThat(ex.getMessage(), containsString("Invalid synonym rule at line 1"));
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
index 958f04ed339..73c104a5b72 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
@@ -48,73 +48,105 @@ public class PathHierarchyTokenizerFactoryTests extends OpenSearchTokenStreamTes
public void testDefaults() throws IOException {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
- Tokenizer tokenizer = new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", Settings.EMPTY).create();
+ Tokenizer tokenizer = new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ Settings.EMPTY
+ ).create();
tokenizer.setReader(new StringReader("/one/two/three"));
- assertTokenStreamContents(tokenizer, new String[] {"/one", "/one/two", "/one/two/three"});
+ assertTokenStreamContents(tokenizer, new String[] { "/one", "/one/two", "/one/two/three" });
}
public void testReverse() throws IOException {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
Settings settings = newAnalysisSettingsBuilder().put("reverse", true).build();
- Tokenizer tokenizer = new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create();
+ Tokenizer tokenizer = new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create();
tokenizer.setReader(new StringReader("/one/two/three"));
- assertTokenStreamContents(tokenizer, new String[] {"/one/two/three", "one/two/three", "two/three", "three"});
+ assertTokenStreamContents(tokenizer, new String[] { "/one/two/three", "one/two/three", "two/three", "three" });
}
public void testDelimiter() throws IOException {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
Settings settings = newAnalysisSettingsBuilder().put("delimiter", "-").build();
- Tokenizer tokenizer = new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create();
+ Tokenizer tokenizer = new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create();
tokenizer.setReader(new StringReader("/one/two/three"));
- assertTokenStreamContents(tokenizer, new String[] {"/one/two/three"});
+ assertTokenStreamContents(tokenizer, new String[] { "/one/two/three" });
tokenizer.setReader(new StringReader("one-two-three"));
- assertTokenStreamContents(tokenizer, new String[] {"one", "one-two", "one-two-three"});
+ assertTokenStreamContents(tokenizer, new String[] { "one", "one-two", "one-two-three" });
}
public void testReplace() throws IOException {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
Settings settings = newAnalysisSettingsBuilder().put("replacement", "-").build();
- Tokenizer tokenizer = new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create();
+ Tokenizer tokenizer = new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create();
tokenizer.setReader(new StringReader("/one/two/three"));
- assertTokenStreamContents(tokenizer, new String[] {"-one", "-one-two", "-one-two-three"});
+ assertTokenStreamContents(tokenizer, new String[] { "-one", "-one-two", "-one-two-three" });
tokenizer.setReader(new StringReader("one-two-three"));
- assertTokenStreamContents(tokenizer, new String[] {"one-two-three"});
+ assertTokenStreamContents(tokenizer, new String[] { "one-two-three" });
}
public void testSkip() throws IOException {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
Settings settings = newAnalysisSettingsBuilder().put("skip", 2).build();
- Tokenizer tokenizer = new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create();
+ Tokenizer tokenizer = new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create();
tokenizer.setReader(new StringReader("/one/two/three/four/five"));
- assertTokenStreamContents(tokenizer, new String[] {"/three", "/three/four", "/three/four/five"});
+ assertTokenStreamContents(tokenizer, new String[] { "/three", "/three/four", "/three/four/five" });
}
public void testDelimiterExceptions() {
final Index index = new Index("test", "_na_");
final Settings indexSettings = newAnalysisSettingsBuilder().build();
{
- String delimiter = RandomPicks.randomFrom(random(), new String[] {"--", ""});
+ String delimiter = RandomPicks.randomFrom(random(), new String[] { "--", "" });
Settings settings = newAnalysisSettingsBuilder().put("delimiter", delimiter).build();
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create());
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create()
+ );
assertEquals("delimiter must be a one char value", e.getMessage());
}
{
- String replacement = RandomPicks.randomFrom(random(), new String[] {"--", ""});
+ String replacement = RandomPicks.randomFrom(random(), new String[] { "--", "" });
Settings settings = newAnalysisSettingsBuilder().put("replacement", replacement).build();
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> new PathHierarchyTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null,
- "path-hierarchy-tokenizer", settings).create());
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> new PathHierarchyTokenizerFactory(
+ IndexSettingsModule.newIndexSettings(index, indexSettings),
+ null,
+ "path-hierarchy-tokenizer",
+ settings
+ ).create()
+ );
assertEquals("replacement must be a one char value", e.getMessage());
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternAnalyzerTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternAnalyzerTests.java
index eef8c8b86a3..162c7e19efb 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternAnalyzerTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternAnalyzerTests.java
@@ -46,90 +46,96 @@ import java.util.regex.Pattern;
*/
public class PatternAnalyzerTests extends OpenSearchTokenStreamTestCase {
- /**
- * Test PatternAnalyzer when it is configured with a non-word pattern.
- */
- public void testNonWordPattern() throws IOException {
- // Split on non-letter pattern, do not lowercase, no stopwords
- PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\W+"), false, null);
- assertAnalyzesTo(a, "The quick brown Fox,the abcd1234 (56.78) dc.",
- new String[] { "The", "quick", "brown", "Fox", "the", "abcd1234", "56", "78", "dc" });
+ /**
+ * Test PatternAnalyzer when it is configured with a non-word pattern.
+ */
+ public void testNonWordPattern() throws IOException {
+ // Split on non-letter pattern, do not lowercase, no stopwords
+ PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\W+"), false, null);
+ assertAnalyzesTo(
+ a,
+ "The quick brown Fox,the abcd1234 (56.78) dc.",
+ new String[] { "The", "quick", "brown", "Fox", "the", "abcd1234", "56", "78", "dc" }
+ );
- // split on non-letter pattern, lowercase, english stopwords
- PatternAnalyzer b = new PatternAnalyzer(Pattern.compile("\\W+"), true,
- EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
- assertAnalyzesTo(b, "The quick brown Fox,the abcd1234 (56.78) dc.",
- new String[] { "quick", "brown", "fox", "abcd1234", "56", "78", "dc" });
- }
+ // split on non-letter pattern, lowercase, english stopwords
+ PatternAnalyzer b = new PatternAnalyzer(Pattern.compile("\\W+"), true, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+ assertAnalyzesTo(
+ b,
+ "The quick brown Fox,the abcd1234 (56.78) dc.",
+ new String[] { "quick", "brown", "fox", "abcd1234", "56", "78", "dc" }
+ );
+ }
- /**
- * Test PatternAnalyzer when it is configured with a whitespace pattern.
- * Behavior can be similar to WhitespaceAnalyzer (depending upon options)
- */
- public void testWhitespacePattern() throws IOException {
- // Split on whitespace patterns, do not lowercase, no stopwords
- PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
- assertAnalyzesTo(a, "The quick brown Fox,the abcd1234 (56.78) dc.",
- new String[] { "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." });
+ /**
+ * Test PatternAnalyzer when it is configured with a whitespace pattern.
+ * Behavior can be similar to WhitespaceAnalyzer (depending upon options)
+ */
+ public void testWhitespacePattern() throws IOException {
+ // Split on whitespace patterns, do not lowercase, no stopwords
+ PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
+ assertAnalyzesTo(
+ a,
+ "The quick brown Fox,the abcd1234 (56.78) dc.",
+ new String[] { "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." }
+ );
- // Split on whitespace patterns, lowercase, english stopwords
- PatternAnalyzer b = new PatternAnalyzer(Pattern.compile("\\s+"), true,
- EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
- assertAnalyzesTo(b, "The quick brown Fox,the abcd1234 (56.78) dc.",
- new String[] { "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." });
- }
+ // Split on whitespace patterns, lowercase, english stopwords
+ PatternAnalyzer b = new PatternAnalyzer(Pattern.compile("\\s+"), true, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+ assertAnalyzesTo(
+ b,
+ "The quick brown Fox,the abcd1234 (56.78) dc.",
+ new String[] { "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." }
+ );
+ }
- /**
- * Test PatternAnalyzer when it is configured with a custom pattern. In this
- * case, text is tokenized on the comma ","
- */
- public void testCustomPattern() throws IOException {
- // Split on comma, do not lowercase, no stopwords
- PatternAnalyzer a = new PatternAnalyzer(Pattern.compile(","), false, null);
- assertAnalyzesTo(a, "Here,Are,some,Comma,separated,words,",
- new String[] { "Here", "Are", "some", "Comma", "separated", "words" });
+ /**
+ * Test PatternAnalyzer when it is configured with a custom pattern. In this
+ * case, text is tokenized on the comma ","
+ */
+ public void testCustomPattern() throws IOException {
+ // Split on comma, do not lowercase, no stopwords
+ PatternAnalyzer a = new PatternAnalyzer(Pattern.compile(","), false, null);
+ assertAnalyzesTo(a, "Here,Are,some,Comma,separated,words,", new String[] { "Here", "Are", "some", "Comma", "separated", "words" });
- // split on comma, lowercase, english stopwords
- PatternAnalyzer b = new PatternAnalyzer(Pattern.compile(","), true,
- EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
- assertAnalyzesTo(b, "Here,Are,some,Comma,separated,words,",
- new String[] { "here", "some", "comma", "separated", "words" });
- }
+ // split on comma, lowercase, english stopwords
+ PatternAnalyzer b = new PatternAnalyzer(Pattern.compile(","), true, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+ assertAnalyzesTo(b, "Here,Are,some,Comma,separated,words,", new String[] { "here", "some", "comma", "separated", "words" });
+ }
- /**
- * Test PatternAnalyzer against a large document.
- */
- public void testHugeDocument() throws IOException {
- StringBuilder document = new StringBuilder();
- // 5000 a's
- char largeWord[] = new char[5000];
- Arrays.fill(largeWord, 'a');
- document.append(largeWord);
+ /**
+ * Test PatternAnalyzer against a large document.
+ */
+ public void testHugeDocument() throws IOException {
+ StringBuilder document = new StringBuilder();
+ // 5000 a's
+ char largeWord[] = new char[5000];
+ Arrays.fill(largeWord, 'a');
+ document.append(largeWord);
- // a space
- document.append(' ');
+ // a space
+ document.append(' ');
- // 2000 b's
- char largeWord2[] = new char[2000];
- Arrays.fill(largeWord2, 'b');
- document.append(largeWord2);
+ // 2000 b's
+ char largeWord2[] = new char[2000];
+ Arrays.fill(largeWord2, 'b');
+ document.append(largeWord2);
- // Split on whitespace patterns, do not lowercase, no stopwords
- PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
- assertAnalyzesTo(a, document.toString(),
- new String[] { new String(largeWord), new String(largeWord2) });
- }
+ // Split on whitespace patterns, do not lowercase, no stopwords
+ PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
+ assertAnalyzesTo(a, document.toString(), new String[] { new String(largeWord), new String(largeWord2) });
+ }
- /** blast some random strings through the analyzer */
- public void testRandomStrings() throws Exception {
- Analyzer a = new PatternAnalyzer(Pattern.compile(","), true, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
- }
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new PatternAnalyzer(Pattern.compile(","), true, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+ checkRandomData(random(), a, 10000 * RANDOM_MULTIPLIER);
+ }
- public void testNormalize() {
- PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
- assertEquals(new BytesRef("FooBar"), a.normalize("dummy", "FooBar"));
- a = new PatternAnalyzer(Pattern.compile("\\s+"), true, null);
- assertEquals(new BytesRef("foobar"), a.normalize("dummy", "FooBar"));
- }
+ public void testNormalize() {
+ PatternAnalyzer a = new PatternAnalyzer(Pattern.compile("\\s+"), false, null);
+ assertEquals(new BytesRef("FooBar"), a.normalize("dummy", "FooBar"));
+ a = new PatternAnalyzer(Pattern.compile("\\s+"), true, null);
+ assertEquals(new BytesRef("foobar"), a.normalize("dummy", "FooBar"));
+ }
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternCaptureTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternCaptureTokenFilterTests.java
index 2e03dac4329..5cd18a5b01f 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternCaptureTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PatternCaptureTokenFilterTests.java
@@ -49,31 +49,35 @@ public class PatternCaptureTokenFilterTests extends OpenSearchTokenStreamTestCas
public void testPatternCaptureTokenFilter() throws Exception {
String json = "/org/opensearch/analysis/common/pattern_capture.json";
Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
- .loadFromStream(json, getClass().getResourceAsStream(json), false)
- .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
- .build();
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+ .loadFromStream(json, getClass().getResourceAsStream(json), false)
+ .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+ .build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
IndexAnalyzers indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
NamedAnalyzer analyzer1 = indexAnalyzers.get("single");
- assertTokenStreamContents(analyzer1.tokenStream("test", "foobarbaz"), new String[]{"foobarbaz","foobar","foo"});
+ assertTokenStreamContents(analyzer1.tokenStream("test", "foobarbaz"), new String[] { "foobarbaz", "foobar", "foo" });
NamedAnalyzer analyzer2 = indexAnalyzers.get("multi");
- assertTokenStreamContents(analyzer2.tokenStream("test", "abc123def"), new String[]{"abc123def","abc","123","def"});
+ assertTokenStreamContents(analyzer2.tokenStream("test", "abc123def"), new String[] { "abc123def", "abc", "123", "def" });
NamedAnalyzer analyzer3 = indexAnalyzers.get("preserve");
- assertTokenStreamContents(analyzer3.tokenStream("test", "foobarbaz"), new String[]{"foobar","foo"});
+ assertTokenStreamContents(analyzer3.tokenStream("test", "foobarbaz"), new String[] { "foobar", "foo" });
}
public void testNoPatterns() {
try {
- new PatternCaptureGroupTokenFilterFactory(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), null,
- "pattern_capture", Settings.builder().put("pattern", "foobar").build());
- fail ("Expected IllegalArgumentException");
+ new PatternCaptureGroupTokenFilterFactory(
+ IndexSettingsModule.newIndexSettings("test", Settings.EMPTY),
+ null,
+ "pattern_capture",
+ Settings.builder().put("pattern", "foobar").build()
+ );
+ fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("required setting 'patterns' is missing"));
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PredicateTokenScriptFilterTests.java
index 5e7ae3e29c8..c16f4f37846 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PredicateTokenScriptFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PredicateTokenScriptFilterTests.java
@@ -53,9 +53,7 @@ import java.util.Collections;
public class PredicateTokenScriptFilterTests extends OpenSearchTokenStreamTestCase {
public void testSimpleFilter() throws IOException {
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.filter.f.type", "predicate_token_filter")
@@ -74,7 +72,7 @@ public class PredicateTokenScriptFilterTests extends OpenSearchTokenStreamTestCa
};
@SuppressWarnings("unchecked")
- ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){
+ ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) {
@Override
public FactoryType compile(Script script, ScriptContext context) {
assertEquals(context, AnalysisPredicateScript.CONTEXT);
@@ -85,16 +83,13 @@ public class PredicateTokenScriptFilterTests extends OpenSearchTokenStreamTestCa
CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
plugin.createComponents(null, null, null, null, scriptService, null, null, null, null, null, null);
- AnalysisModule module
- = new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin));
+ AnalysisModule module = new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin));
IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings);
try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) {
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "Oh what a wonderful thing to be", new String[]{
- "Oh", "what", "to", "be"
- });
+ assertAnalyzesTo(analyzer, "Oh what a wonderful thing to be", new String[] { "Oh", "what", "to", "be" });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java
index 7aed6f021b8..cb22835c364 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java
@@ -64,11 +64,7 @@ public class RemoveDuplicatesFilterFactoryTests extends OpenSearchTokenStreamTes
new Token("d", 1, 4, 5)
);
- assertTokenStreamContents(tokenFilter.create(cts), new String[]{
- "a", "b", "c", "d"
- }, new int[]{
- 1, 1, 0, 1
- });
+ assertTokenStreamContents(tokenFilter.create(cts), new String[] { "a", "b", "c", "d" }, new int[] { 1, 1, 0, 1 });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ScriptedConditionTokenFilterTests.java
index f2a5af92e91..9212fcad285 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ScriptedConditionTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ScriptedConditionTokenFilterTests.java
@@ -52,9 +52,7 @@ import java.util.Collections;
public class ScriptedConditionTokenFilterTests extends OpenSearchTokenStreamTestCase {
public void testSimpleCondition() throws Exception {
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.filter.cond.type", "condition")
@@ -74,7 +72,7 @@ public class ScriptedConditionTokenFilterTests extends OpenSearchTokenStreamTest
};
@SuppressWarnings("unchecked")
- ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){
+ ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) {
@Override
public FactoryType compile(Script script, ScriptContext context) {
assertEquals(context, AnalysisPredicateScript.CONTEXT);
@@ -85,16 +83,13 @@ public class ScriptedConditionTokenFilterTests extends OpenSearchTokenStreamTest
CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
plugin.createComponents(null, null, null, null, scriptService, null, null, null, null, null, null);
- AnalysisModule module
- = new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin));
+ AnalysisModule module = new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin));
IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings);
try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) {
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[]{
- "Vorsprung", "Durch", "TECHNIK"
- });
+ assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[] { "Vorsprung", "Durch", "TECHNIK" });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java
index f62d7fc55d8..0646fed4730 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java
@@ -52,7 +52,8 @@ public class ShingleTokenFilterTests extends OpenSearchTokenStreamTestCase {
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put("index.analysis.filter.my_ascii_folding.type", "asciifolding")
.build(),
- new CommonAnalysisPlugin());
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("shingle");
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("this is a test"));
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SnowballAnalyzerTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SnowballAnalyzerTests.java
index 6660f0837be..ebf72425ebb 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SnowballAnalyzerTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SnowballAnalyzerTests.java
@@ -38,35 +38,29 @@ import org.opensearch.test.OpenSearchTokenStreamTestCase;
public class SnowballAnalyzerTests extends OpenSearchTokenStreamTestCase {
- public void testEnglish() throws Exception {
- Analyzer a = new SnowballAnalyzer("English");
- assertAnalyzesTo(a, "he abhorred accents",
- new String[]{"he", "abhor", "accent"});
- }
+ public void testEnglish() throws Exception {
+ Analyzer a = new SnowballAnalyzer("English");
+ assertAnalyzesTo(a, "he abhorred accents", new String[] { "he", "abhor", "accent" });
+ }
- public void testStopwords() throws Exception {
- Analyzer a = new SnowballAnalyzer("English",
- EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
- assertAnalyzesTo(a, "the quick brown fox jumped",
- new String[]{"quick", "brown", "fox", "jump"});
- }
+ public void testStopwords() throws Exception {
+ Analyzer a = new SnowballAnalyzer("English", EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+ assertAnalyzesTo(a, "the quick brown fox jumped", new String[] { "quick", "brown", "fox", "jump" });
+ }
- /**
- * Test turkish lowercasing
- */
- public void testTurkish() throws Exception {
- Analyzer a = new SnowballAnalyzer("Turkish");
+ /**
+ * Test turkish lowercasing
+ */
+ public void testTurkish() throws Exception {
+ Analyzer a = new SnowballAnalyzer("Turkish");
- assertAnalyzesTo(a, "ağacı", new String[] { "ağaç" });
- assertAnalyzesTo(a, "AĞACI", new String[] { "ağaç" });
- }
+ assertAnalyzesTo(a, "ağacı", new String[] { "ağaç" });
+ assertAnalyzesTo(a, "AĞACI", new String[] { "ağaç" });
+ }
-
- public void testReusableTokenStream() throws Exception {
- Analyzer a = new SnowballAnalyzer("English");
- assertAnalyzesTo(a, "he abhorred accents",
- new String[]{"he", "abhor", "accent"});
- assertAnalyzesTo(a, "she abhorred him",
- new String[]{"she", "abhor", "him"});
- }
+ public void testReusableTokenStream() throws Exception {
+ Analyzer a = new SnowballAnalyzer("English");
+ assertAnalyzesTo(a, "he abhorred accents", new String[] { "he", "abhor", "accent" });
+ assertAnalyzesTo(a, "she abhorred him", new String[] { "she", "abhor", "him" });
+ }
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java
index 92d8fd5807d..96e05efa977 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java
@@ -59,7 +59,8 @@ public class StemmerOverrideTokenFilterFactoryTests extends OpenSearchTokenStrea
.putList("index.analysis.filter.my_stemmer_override.rules", rules)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build(),
- new CommonAnalysisPlugin());
+ new CommonAnalysisPlugin()
+ );
return analysis.tokenFilter.get("my_stemmer_override");
}
@@ -75,19 +76,18 @@ public class StemmerOverrideTokenFilterFactoryTests extends OpenSearchTokenStrea
"=>a", // no keys
"a,=>b" // empty key
)) {
- expectThrows(RuntimeException.class, String.format(
- Locale.ROOT, "Should fail for invalid rule: '%s'", rule
- ), () -> create(rule));
+ expectThrows(
+ RuntimeException.class,
+ String.format(Locale.ROOT, "Should fail for invalid rule: '%s'", rule),
+ () -> create(rule)
+ );
}
}
public void testRulesOk() throws IOException {
- TokenFilterFactory tokenFilterFactory = create(
- "a => 1",
- "b,c => 2"
- );
+ TokenFilterFactory tokenFilterFactory = create("a => 1", "b,c => 2");
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("a b c"));
- assertTokenStreamContents(tokenFilterFactory.create(tokenizer), new String[]{"1", "2", "2"});
+ assertTokenStreamContents(tokenFilterFactory.create(tokenizer), new String[] { "1", "2", "2" });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerTokenFilterFactoryTests.java
index a65ec662792..fca64f4915c 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/StemmerTokenFilterFactoryTests.java
@@ -63,13 +63,13 @@ public class StemmerTokenFilterFactoryTests extends OpenSearchTokenStreamTestCas
for (int i = 0; i < iters; i++) {
Version v = VersionUtils.randomVersion(random());
Settings settings = Settings.builder()
- .put("index.analysis.filter.my_english.type", "stemmer")
- .put("index.analysis.filter.my_english.language", "english")
- .put("index.analysis.analyzer.my_english.tokenizer","whitespace")
- .put("index.analysis.analyzer.my_english.filter","my_english")
- .put(SETTING_VERSION_CREATED,v)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.my_english.type", "stemmer")
+ .put("index.analysis.filter.my_english.language", "english")
+ .put("index.analysis.analyzer.my_english.tokenizer", "whitespace")
+ .put("index.analysis.analyzer.my_english.filter", "my_english")
+ .put(SETTING_VERSION_CREATED, v)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_english");
@@ -80,7 +80,7 @@ public class StemmerTokenFilterFactoryTests extends OpenSearchTokenStreamTestCas
IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
NamedAnalyzer analyzer = indexAnalyzers.get("my_english");
assertThat(create, instanceOf(PorterStemFilter.class));
- assertAnalyzesTo(analyzer, "consolingly", new String[]{"consolingli"});
+ assertAnalyzesTo(analyzer, "consolingly", new String[] { "consolingli" });
}
}
@@ -90,13 +90,13 @@ public class StemmerTokenFilterFactoryTests extends OpenSearchTokenStreamTestCas
Version v = VersionUtils.randomVersion(random());
Settings settings = Settings.builder()
- .put("index.analysis.filter.my_porter2.type", "stemmer")
- .put("index.analysis.filter.my_porter2.language", "porter2")
- .put("index.analysis.analyzer.my_porter2.tokenizer","whitespace")
- .put("index.analysis.analyzer.my_porter2.filter","my_porter2")
- .put(SETTING_VERSION_CREATED,v)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.my_porter2.type", "stemmer")
+ .put("index.analysis.filter.my_porter2.language", "porter2")
+ .put("index.analysis.analyzer.my_porter2.tokenizer", "whitespace")
+ .put("index.analysis.analyzer.my_porter2.filter", "my_porter2")
+ .put(SETTING_VERSION_CREATED, v)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_porter2");
@@ -107,18 +107,23 @@ public class StemmerTokenFilterFactoryTests extends OpenSearchTokenStreamTestCas
IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
NamedAnalyzer analyzer = indexAnalyzers.get("my_porter2");
assertThat(create, instanceOf(SnowballFilter.class));
- assertAnalyzesTo(analyzer, "possibly", new String[]{"possibl"});
+ assertAnalyzesTo(analyzer, "possibly", new String[] { "possibl" });
}
}
public void testMultipleLanguagesThrowsException() throws IOException {
Version v = VersionUtils.randomVersion(random());
- Settings settings = Settings.builder().put("index.analysis.filter.my_english.type", "stemmer")
- .putList("index.analysis.filter.my_english.language", "english", "light_english").put(SETTING_VERSION_CREATED, v)
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+ Settings settings = Settings.builder()
+ .put("index.analysis.filter.my_english.type", "stemmer")
+ .putList("index.analysis.filter.my_english.language", "english", "light_english")
+ .put(SETTING_VERSION_CREATED, v)
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN));
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN)
+ );
assertEquals("Invalid stemmer class specified: [english, light_english]", e.getMessage());
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java
index bb1ceb4663d..9df477c0f4a 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java
@@ -80,10 +80,11 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
Files.copy(synonymsWordnet, config.resolve("synonyms_wordnet.txt"));
String json = "/org/opensearch/analysis/common/synonyms.json";
- Settings settings = Settings.builder().
- loadFromStream(json, getClass().getResourceAsStream(json), false)
- .put(Environment.PATH_HOME_SETTING.getKey(), home)
- .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
+ Settings settings = Settings.builder()
+ .loadFromStream(json, getClass().getResourceAsStream(json), false)
+ .put(Environment.PATH_HOME_SETTING.getKey(), home)
+ .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+ .build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
@@ -110,7 +111,7 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
.put("index.analysis.filter.stop_within_synonym.type", "stop")
.putList("index.analysis.filter.stop_within_synonym.stopwords", "foobar", "opensearch")
.put("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.tokenizer", "whitespace")
- .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym","synonym")
+ .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym", "synonym")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
try {
@@ -131,7 +132,7 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
.put("index.analysis.filter.stop_within_synonym.type", "stop")
.putList("index.analysis.filter.stop_within_synonym.stopwords", "foobar", "opensearch")
.put("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.tokenizer", "whitespace")
- .putList("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.filter", "stop_within_synonym","synonym_expand")
+ .putList("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.filter", "stop_within_synonym", "synonym_expand")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
try {
@@ -159,9 +160,12 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
- BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("synonymAnalyzer"), "Some developers are odd",
- new String[]{ "some", "developers", "develop", "programm", "are", "odd" },
- new int[]{ 1, 1, 0, 0, 1, 1 });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ indexAnalyzers.get("synonymAnalyzer"),
+ "Some developers are odd",
+ new String[] { "some", "developers", "develop", "programm", "are", "odd" },
+ new int[] { 1, 1, 0, 0, 1, 1 }
+ );
}
public void testAsciiFoldingFilterForSynonyms() throws IOException {
@@ -176,9 +180,12 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
- BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("synonymAnalyzer"), "høj",
- new String[]{ "hoj", "height" },
- new int[]{ 1, 0 });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ indexAnalyzers.get("synonymAnalyzer"),
+ "høj",
+ new String[] { "hoj", "height" },
+ new int[] { 1, 0 }
+ );
}
public void testPreconfigured() throws IOException {
@@ -193,9 +200,12 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
- BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("my_analyzer"), "würst",
- new String[]{ "wurst", "sausage"},
- new int[]{ 1, 0 });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ indexAnalyzers.get("my_analyzer"),
+ "würst",
+ new String[] { "wurst", "sausage" },
+ new int[] { 1, 0 }
+ );
}
public void testChainedSynonymFilters() throws IOException {
@@ -212,15 +222,21 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
- BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("syn"), "term1",
- new String[]{ "term1", "term3", "term2" }, new int[]{ 1, 0, 0 });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ indexAnalyzers.get("syn"),
+ "term1",
+ new String[] { "term1", "term3", "term2" },
+ new int[] { 1, 0, 0 }
+ );
}
public void testShingleFilters() {
Settings settings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
- VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT))
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
+ VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
+ )
.put("path.home", createTempDir().toString())
.put("index.analysis.filter.synonyms.type", "synonym")
.putList("index.analysis.filter.synonyms.synonyms", "programmer, developer")
@@ -230,9 +246,10 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
- expectThrows(IllegalArgumentException.class, () -> {
- indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
- });
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> { indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers; }
+ );
}
@@ -246,9 +263,7 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
- String[] bypassingFactories = new String[]{
- "dictionary_decompounder"
- };
+ String[] bypassingFactories = new String[] { "dictionary_decompounder" };
CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
for (String factory : bypassingFactories) {
@@ -265,14 +280,25 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
}
public void testPreconfiguredTokenFilters() throws IOException {
- Set disallowedFilters = new HashSet<>(Arrays.asList(
- "common_grams", "edge_ngram", "edgeNGram", "keyword_repeat", "ngram", "nGram",
- "shingle", "word_delimiter", "word_delimiter_graph"
- ));
+ Set disallowedFilters = new HashSet<>(
+ Arrays.asList(
+ "common_grams",
+ "edge_ngram",
+ "edgeNGram",
+ "keyword_repeat",
+ "ngram",
+ "nGram",
+ "shingle",
+ "word_delimiter",
+ "word_delimiter_graph"
+ )
+ );
Settings settings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
- VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT))
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
+ VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
+ )
.put("path.home", createTempDir().toString())
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
@@ -281,23 +307,26 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
for (PreConfiguredTokenFilter tf : plugin.getPreConfiguredTokenFilters()) {
if (disallowedFilters.contains(tf.getName())) {
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- "Expected exception for factory " + tf.getName(), () -> {
- tf.get(idxSettings, null, tf.getName(), settings).getSynonymFilter();
- });
- assertEquals(tf.getName(), "Token filter [" + tf.getName()
- + "] cannot be used to parse synonyms",
- e.getMessage());
- }
- else {
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ "Expected exception for factory " + tf.getName(),
+ () -> { tf.get(idxSettings, null, tf.getName(), settings).getSynonymFilter(); }
+ );
+ assertEquals(tf.getName(), "Token filter [" + tf.getName() + "] cannot be used to parse synonyms", e.getMessage());
+ } else {
tf.get(idxSettings, null, tf.getName(), settings).getSynonymFilter();
}
}
Settings settings2 = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
VersionUtils.randomVersionBetween(
- random(), LegacyESVersion.V_6_0_0, VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)))
+ random(),
+ LegacyESVersion.V_6_0_0,
+ VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)
+ )
+ )
.put("path.home", createTempDir().toString())
.putList("common_words", "a", "b")
.put("output_unigrams", "true")
@@ -309,8 +338,7 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
if (disallowedFilters.contains(tf.getName())) {
tf.get(idxSettings2, null, tf.getName(), settings2).getSynonymFilter();
expectedWarnings.add("Token filter [" + tf.getName() + "] will not be usable to parse synonyms after v7.0");
- }
- else {
+ } else {
tf.get(idxSettings2, null, tf.getName(), settings2).getSynonymFilter();
}
}
@@ -320,8 +348,10 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
public void testDisallowedTokenFilters() throws IOException {
Settings settings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
- VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT))
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
+ VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
+ )
.put("path.home", createTempDir().toString())
.putList("common_words", "a", "b")
.put("output_unigrams", "true")
@@ -329,29 +359,39 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
- String[] disallowedFactories = new String[]{
- "multiplexer", "cjk_bigram", "common_grams", "ngram", "edge_ngram",
- "word_delimiter", "word_delimiter_graph", "fingerprint"
- };
+ String[] disallowedFactories = new String[] {
+ "multiplexer",
+ "cjk_bigram",
+ "common_grams",
+ "ngram",
+ "edge_ngram",
+ "word_delimiter",
+ "word_delimiter_graph",
+ "fingerprint" };
for (String factory : disallowedFactories) {
TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings);
TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings);
SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings);
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
"Expected IllegalArgumentException for factory " + factory,
- () -> stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null));
+ () -> stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null)
+ );
- assertEquals(factory, "Token filter [" + factory
- + "] cannot be used to parse synonyms",
- e.getMessage());
+ assertEquals(factory, "Token filter [" + factory + "] cannot be used to parse synonyms", e.getMessage());
}
settings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
VersionUtils.randomVersionBetween(
- random(), LegacyESVersion.V_6_0_0, VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)))
+ random(),
+ LegacyESVersion.V_6_0_0,
+ VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)
+ )
+ )
.put("path.home", createTempDir().toString())
.putList("common_words", "a", "b")
.put("output_unigrams", "true")
@@ -365,16 +405,20 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings);
stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null);
- expectedWarnings.add("Token filter [" + factory
- + "] will not be usable to parse synonyms after v7.0");
+ expectedWarnings.add("Token filter [" + factory + "] will not be usable to parse synonyms after v7.0");
}
assertWarnings(expectedWarnings.toArray(new String[0]));
settings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
VersionUtils.randomVersionBetween(
- random(), LegacyESVersion.V_6_0_0, VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)))
+ random(),
+ LegacyESVersion.V_6_0_0,
+ VersionUtils.getPreviousVersion(LegacyESVersion.V_7_0_0)
+ )
+ )
.put("path.home", createTempDir().toString())
.put("preserve_original", "false")
.build();
@@ -383,11 +427,12 @@ public class SynonymsAnalysisTests extends OpenSearchTestCase {
TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings);
SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings);
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null));
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null)
+ );
- assertEquals("Token filter [multiplexer] cannot be used to parse synonyms unless [preserve_original] is [true]",
- e.getMessage());
+ assertEquals("Token filter [multiplexer] cannot be used to parse synonyms unless [preserve_original] is [true]", e.getMessage());
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/TrimTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/TrimTokenFilterTests.java
index 34ef77aebb3..3ea9c526052 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/TrimTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/TrimTokenFilterTests.java
@@ -54,7 +54,7 @@ public class TrimTokenFilterTests extends OpenSearchTokenStreamTestCase {
NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
assertNotNull(normalizer);
assertEquals("my_normalizer", normalizer.name());
- assertTokenStreamContents(normalizer.tokenStream("foo", " bar "), new String[] {"bar"});
+ assertTokenStreamContents(normalizer.tokenStream("foo", " bar "), new String[] { "bar" });
assertEquals(new BytesRef("bar"), normalizer.normalize("foo", " bar "));
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java
index 98b7348296c..d98e358621d 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java
@@ -54,8 +54,12 @@ public class WhitespaceTokenizerFactoryTests extends OpenSearchTestCase {
public void testSimpleWhiteSpaceTokenizer() throws IOException {
final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(new Index("test", "_na_"), indexSettings);
- WhitespaceTokenizer tokenizer = (WhitespaceTokenizer) new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen",
- Settings.EMPTY).create();
+ WhitespaceTokenizer tokenizer = (WhitespaceTokenizer) new WhitespaceTokenizerFactory(
+ indexProperties,
+ null,
+ "whitespace_maxlen",
+ Settings.EMPTY
+ ).create();
try (Reader reader = new StringReader("one, two, three")) {
tokenizer.setReader(reader);
@@ -67,8 +71,12 @@ public class WhitespaceTokenizerFactoryTests extends OpenSearchTestCase {
final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(new Index("test", "_na_"), indexSettings);
final Settings settings = Settings.builder().put(WhitespaceTokenizerFactory.MAX_TOKEN_LENGTH, 2).build();
- WhitespaceTokenizer tokenizer = (WhitespaceTokenizer) new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen",
- settings).create();
+ WhitespaceTokenizer tokenizer = (WhitespaceTokenizer) new WhitespaceTokenizerFactory(
+ indexProperties,
+ null,
+ "whitespace_maxlen",
+ settings
+ ).create();
try (Reader reader = new StringReader("one, two, three")) {
tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer, new String[] { "on", "e,", "tw", "o,", "th", "re", "e" });
@@ -76,7 +84,7 @@ public class WhitespaceTokenizerFactoryTests extends OpenSearchTestCase {
final Settings defaultSettings = Settings.EMPTY;
tokenizer = (WhitespaceTokenizer) new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen", defaultSettings)
- .create();
+ .create();
String veryLongToken = RandomStrings.randomAsciiAlphanumOfLength(random(), 256);
try (Reader reader = new StringReader(veryLongToken)) {
tokenizer.setReader(reader);
@@ -84,13 +92,17 @@ public class WhitespaceTokenizerFactoryTests extends OpenSearchTestCase {
}
final Settings tooLongSettings = Settings.builder().put(WhitespaceTokenizerFactory.MAX_TOKEN_LENGTH, 1024 * 1024 + 1).build();
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
- () -> new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen", tooLongSettings).create());
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen", tooLongSettings).create()
+ );
assertEquals("maxTokenLen must be greater than 0 and less than 1048576 passed: 1048577", e.getMessage());
final Settings negativeSettings = Settings.builder().put(WhitespaceTokenizerFactory.MAX_TOKEN_LENGTH, -1).build();
- e = expectThrows(IllegalArgumentException.class,
- () -> new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen", negativeSettings).create());
+ e = expectThrows(
+ IllegalArgumentException.class,
+ () -> new WhitespaceTokenizerFactory(indexProperties, null, "whitespace_maxlen", negativeSettings).create()
+ );
assertEquals("maxTokenLen must be greater than 0 and less than 1048576 passed: -1", e.getMessage());
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
index 851f2bfebaa..6129971a69e 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
@@ -53,35 +53,56 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.Collections;
-public class WordDelimiterGraphTokenFilterFactoryTests
- extends BaseWordDelimiterTokenFilterFactoryTestCase {
+public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase {
public WordDelimiterGraphTokenFilterFactoryTests() {
super("word_delimiter_graph");
}
public void testMultiTerms() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
- Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put("index.analysis.filter.my_word_delimiter.type", type)
- .put("index.analysis.filter.my_word_delimiter.catenate_all", "true")
- .put("index.analysis.filter.my_word_delimiter.preserve_original", "true")
- .build(),
- new CommonAnalysisPlugin());
+ Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put("index.analysis.filter.my_word_delimiter.type", type)
+ .put("index.analysis.filter.my_word_delimiter.catenate_all", "true")
+ .put("index.analysis.filter.my_word_delimiter.preserve_original", "true")
+ .build(),
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
- String[] expected = new String[] { "PowerShot", "PowerShot", "Power", "Shot", "500-42",
- "50042", "500", "42", "wi-fi", "wifi", "wi", "fi", "wi-fi-4000", "wifi4000", "wi",
- "fi", "4000", "j2se", "j2se", "j", "2", "se", "O'Neil's", "ONeil", "O", "Neil" };
+ String[] expected = new String[] {
+ "PowerShot",
+ "PowerShot",
+ "Power",
+ "Shot",
+ "500-42",
+ "50042",
+ "500",
+ "42",
+ "wi-fi",
+ "wifi",
+ "wi",
+ "fi",
+ "wi-fi-4000",
+ "wifi4000",
+ "wi",
+ "fi",
+ "4000",
+ "j2se",
+ "j2se",
+ "j",
+ "2",
+ "se",
+ "O'Neil's",
+ "ONeil",
+ "O",
+ "Neil" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
- int[] expectedIncr = new int[] { 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0,
- 1, 1, 1, 0, 0, 1 };
- int[] expectedPosLen = new int[] { 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 3, 3, 1, 1, 1, 3, 3,
- 1, 1, 1, 2, 2, 1, 1 };
- assertTokenStreamContents(tokenFilter.create(tokenizer), expected, null, null, null,
- expectedIncr, expectedPosLen, null);
+ int[] expectedIncr = new int[] { 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 };
+ int[] expectedPosLen = new int[] { 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 3, 3, 1, 1, 1, 3, 3, 1, 1, 1, 2, 2, 1, 1 };
+ assertTokenStreamContents(tokenFilter.create(tokenizer), expected, null, null, null, expectedIncr, expectedPosLen, null);
}
/**
@@ -89,24 +110,33 @@ public class WordDelimiterGraphTokenFilterFactoryTests
*/
public void testPartsAndCatenate() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
- Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put("index.analysis.filter.my_word_delimiter.type", type)
- .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
- .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
- .build(),
- new CommonAnalysisPlugin());
+ Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put("index.analysis.filter.my_word_delimiter.type", type)
+ .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
+ .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
+ .build(),
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
String source = "PowerShot";
- int[] expectedIncr = new int[]{1, 0, 1};
- int[] expectedPosLen = new int[]{2, 1, 1};
- int[] expectedStartOffsets = new int[]{0, 0, 5};
- int[] expectedEndOffsets = new int[]{9, 5, 9};
- String[] expected = new String[]{"PowerShot", "Power", "Shot" };
+ int[] expectedIncr = new int[] { 1, 0, 1 };
+ int[] expectedPosLen = new int[] { 2, 1, 1 };
+ int[] expectedStartOffsets = new int[] { 0, 0, 5 };
+ int[] expectedEndOffsets = new int[] { 9, 5, 9 };
+ String[] expected = new String[] { "PowerShot", "Power", "Shot" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
- assertTokenStreamContents(tokenFilter.create(tokenizer), expected, expectedStartOffsets, expectedEndOffsets, null,
- expectedIncr, expectedPosLen, null);
+ assertTokenStreamContents(
+ tokenFilter.create(tokenizer),
+ expected,
+ expectedStartOffsets,
+ expectedEndOffsets,
+ null,
+ expectedIncr,
+ expectedPosLen,
+ null
+ );
}
public void testAdjustingOffsets() throws IOException {
@@ -118,82 +148,94 @@ public class WordDelimiterGraphTokenFilterFactoryTests
.put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
.put("index.analysis.filter.my_word_delimiter.adjust_offsets", "false")
.build(),
- new CommonAnalysisPlugin());
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
String source = "PowerShot";
- int[] expectedIncr = new int[]{1, 0, 1};
- int[] expectedPosLen = new int[]{2, 1, 1};
- int[] expectedStartOffsets = new int[]{0, 0, 0};
- int[] expectedEndOffsets = new int[]{9, 9, 9};
- String[] expected = new String[]{"PowerShot", "Power", "Shot" };
+ int[] expectedIncr = new int[] { 1, 0, 1 };
+ int[] expectedPosLen = new int[] { 2, 1, 1 };
+ int[] expectedStartOffsets = new int[] { 0, 0, 0 };
+ int[] expectedEndOffsets = new int[] { 9, 9, 9 };
+ String[] expected = new String[] { "PowerShot", "Power", "Shot" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
- assertTokenStreamContents(tokenFilter.create(tokenizer), expected, expectedStartOffsets, expectedEndOffsets, null,
- expectedIncr, expectedPosLen, null);
+ assertTokenStreamContents(
+ tokenFilter.create(tokenizer),
+ expected,
+ expectedStartOffsets,
+ expectedEndOffsets,
+ null,
+ expectedIncr,
+ expectedPosLen,
+ null
+ );
}
public void testIgnoreKeywords() throws IOException {
- //test with keywords but ignore is false (default behavior)
+ // test with keywords but ignore is false (default behavior)
Settings settings = Settings.builder()
- .put("index.analysis.filter.my_word_delimiter.type", type)
- .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
- .put("index.analysis.filter.my_keyword.type", "keyword_marker")
- .put("index.analysis.filter.my_keyword.keywords", "PowerHungry")
- .put("index.analysis.analyzer.my_analyzer.type", "custom")
- .put("index.analysis.analyzer.my_analyzer.tokenizer", "whitespace")
- .put("index.analysis.analyzer.my_analyzer.filter", "my_keyword, my_word_delimiter")
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ .put("index.analysis.filter.my_word_delimiter.type", type)
+ .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
+ .put("index.analysis.filter.my_keyword.type", "keyword_marker")
+ .put("index.analysis.filter.my_keyword.keywords", "PowerHungry")
+ .put("index.analysis.analyzer.my_analyzer.type", "custom")
+ .put("index.analysis.analyzer.my_analyzer.tokenizer", "whitespace")
+ .put("index.analysis.analyzer.my_analyzer.filter", "my_keyword, my_word_delimiter")
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .build();
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
String source = "PowerShot PowerHungry";
- int[] expectedStartOffsets = new int[]{0, 5, 10, 15};
- int[] expectedEndOffsets = new int[]{5, 9, 15, 21};
- String[] expected = new String[]{"Power", "Shot", "Power", "Hungry"};
+ int[] expectedStartOffsets = new int[] { 0, 5, 10, 15 };
+ int[] expectedEndOffsets = new int[] { 5, 9, 15, 21 };
+ String[] expected = new String[] { "Power", "Shot", "Power", "Hungry" };
NamedAnalyzer analyzer = analysis.indexAnalyzers.get("my_analyzer");
assertAnalyzesTo(analyzer, source, expected, expectedStartOffsets, expectedEndOffsets);
- //test with keywords but ignore_keywords is set as true
- settings = Settings.builder().put(settings)
- .put("index.analysis.filter.my_word_delimiter.ignore_keywords", "true")
- .build();
+ // test with keywords but ignore_keywords is set as true
+ settings = Settings.builder().put(settings).put("index.analysis.filter.my_word_delimiter.ignore_keywords", "true").build();
analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
analyzer = analysis.indexAnalyzers.get("my_analyzer");
- expectedStartOffsets = new int[]{0, 5, 10};
- expectedEndOffsets = new int[]{5, 9, 21};
- expected = new String[]{"Power", "Shot", "PowerHungry"};
+ expectedStartOffsets = new int[] { 0, 5, 10 };
+ expectedEndOffsets = new int[] { 5, 9, 21 };
+ expected = new String[] { "Power", "Shot", "PowerHungry" };
assertAnalyzesTo(analyzer, source, expected, expectedStartOffsets, expectedEndOffsets);
}
public void testPreconfiguredFilter() throws IOException {
// Before 7.3 we don't adjust offsets
{
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
- .put(IndexMetadata.SETTING_VERSION_CREATED,
+ .put(
+ IndexMetadata.SETTING_VERSION_CREATED,
VersionUtils.randomVersionBetween(
- random(), LegacyESVersion.V_7_0_0, VersionUtils.getPreviousVersion(LegacyESVersion.V_7_3_0)))
+ random(),
+ LegacyESVersion.V_7_0_0,
+ VersionUtils.getPreviousVersion(LegacyESVersion.V_7_3_0)
+ )
+ )
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
- try (IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
- Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings)) {
+ try (
+ IndexAnalyzers indexAnalyzers = new AnalysisModule(
+ TestEnvironment.newEnvironment(settings),
+ Collections.singletonList(new CommonAnalysisPlugin())
+ ).getAnalysisRegistry().build(idxSettings)
+ ) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "h100", new String[]{"h", "100"}, new int[]{ 0, 0 }, new int[]{ 4, 4 });
+ assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 0 }, new int[] { 4, 4 });
}
}
// Afger 7.3 we do adjust offsets
{
- Settings settings = Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .build();
+ Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
@@ -201,12 +243,16 @@ public class WordDelimiterGraphTokenFilterFactoryTests
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
- try (IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
- Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings)) {
+ try (
+ IndexAnalyzers indexAnalyzers = new AnalysisModule(
+ TestEnvironment.newEnvironment(settings),
+ Collections.singletonList(new CommonAnalysisPlugin())
+ ).getAnalysisRegistry().build(idxSettings)
+ ) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
- assertAnalyzesTo(analyzer, "h100", new String[]{"h", "100"}, new int[]{ 0, 1 }, new int[]{ 1, 4 });
+ assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 1 }, new int[] { 1, 4 });
}
}
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java
index e5a5697e61a..ea37fd5ce95 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java
@@ -31,7 +31,6 @@
package org.opensearch.analysis.common;
-
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.opensearch.common.settings.Settings;
@@ -43,8 +42,7 @@ import org.opensearch.test.OpenSearchTestCase;
import java.io.IOException;
import java.io.StringReader;
-public class WordDelimiterTokenFilterFactoryTests
- extends BaseWordDelimiterTokenFilterFactoryTestCase {
+public class WordDelimiterTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase {
public WordDelimiterTokenFilterFactoryTests() {
super("word_delimiter");
}
@@ -54,16 +52,17 @@ public class WordDelimiterTokenFilterFactoryTests
*/
public void testPartsAndCatenate() throws IOException {
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
- Settings.builder()
- .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
- .put("index.analysis.filter.my_word_delimiter.type", type)
- .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
- .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
- .build(),
- new CommonAnalysisPlugin());
+ Settings.builder()
+ .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+ .put("index.analysis.filter.my_word_delimiter.type", type)
+ .put("index.analysis.filter.my_word_delimiter.catenate_words", "true")
+ .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true")
+ .build(),
+ new CommonAnalysisPlugin()
+ );
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
String source = "PowerShot";
- String[] expected = new String[]{"Power", "PowerShot", "Shot" };
+ String[] expected = new String[] { "Power", "PowerShot", "Shot" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
diff --git a/modules/analysis-common/src/yamlRestTest/java/org/opensearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java b/modules/analysis-common/src/yamlRestTest/java/org/opensearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java
index eea331fe19a..90bd7097171 100644
--- a/modules/analysis-common/src/yamlRestTest/java/org/opensearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java
+++ b/modules/analysis-common/src/yamlRestTest/java/org/opensearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java
@@ -38,7 +38,7 @@ import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
public class CommonAnalysisClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
- public CommonAnalysisClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) {
+ public CommonAnalysisClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
diff --git a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java
index 06cc1c7797b..babf024da01 100644
--- a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java
+++ b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java
@@ -79,9 +79,7 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
ctx.put("z", 0);
return null;
});
- pluginScripts.put("throwing_script", ctx -> {
- throw new RuntimeException("this script always fails");
- });
+ pluginScripts.put("throwing_script", ctx -> { throw new RuntimeException("this script always fails"); });
return pluginScripts;
}
}
@@ -90,30 +88,44 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
internalCluster().ensureAtLeastNumDataNodes(1);
internalCluster().startMasterOnlyNode();
final String pipelineId = "foo";
- client().admin().cluster().preparePutPipeline(pipelineId,
- new BytesArray("{\n" +
- " \"processors\" : [\n" +
- " {\"set\" : {\"field\": \"any_field\", \"value\": \"any_value\"}},\n" +
- " {\"set\" : {" + "" +
- " \"if\" : " + "{\"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"throwing_script\"}," +
- " \"field\": \"any_field2\"," +
- " \"value\": \"any_value2\"}" +
- " }\n" +
- " ]\n" +
- "}"), XContentType.JSON).get();
+ client().admin()
+ .cluster()
+ .preparePutPipeline(
+ pipelineId,
+ new BytesArray(
+ "{\n"
+ + " \"processors\" : [\n"
+ + " {\"set\" : {\"field\": \"any_field\", \"value\": \"any_value\"}},\n"
+ + " {\"set\" : {"
+ + ""
+ + " \"if\" : "
+ + "{\"lang\": \""
+ + MockScriptEngine.NAME
+ + "\", \"source\": \"throwing_script\"},"
+ + " \"field\": \"any_field2\","
+ + " \"value\": \"any_value2\"}"
+ + " }\n"
+ + " ]\n"
+ + "}"
+ ),
+ XContentType.JSON
+ )
+ .get();
Exception e = expectThrows(
Exception.class,
- () ->
- client().prepareIndex("index", "doc").setId("1")
- .setSource("x", 0)
- .setPipeline(pipelineId)
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get()
+ () -> client().prepareIndex("index", "doc")
+ .setId("1")
+ .setSource("x", 0)
+ .setPipeline(pipelineId)
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .get()
);
assertTrue(e.getMessage().contains("this script always fails"));
- NodesStatsResponse r = client().admin().cluster().prepareNodesStats(internalCluster().getNodeNames())
+ NodesStatsResponse r = client().admin()
+ .cluster()
+ .prepareNodesStats(internalCluster().getNodeNames())
.addMetric(INGEST.metricName())
.get();
int nodeCount = r.getNodes().size();
@@ -130,19 +142,23 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
String pipelineIdWithScript = pipelineIdWithoutScript + "_script";
internalCluster().startNode();
- BytesReference pipelineWithScript = new BytesArray("{\n" +
- " \"processors\" : [\n" +
- " {\"script\" : {\"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"my_script\"}}\n" +
- " ]\n" +
- "}");
- BytesReference pipelineWithoutScript = new BytesArray("{\n" +
- " \"processors\" : [\n" +
- " {\"set\" : {\"field\": \"y\", \"value\": 0}}\n" +
- " ]\n" +
- "}");
+ BytesReference pipelineWithScript = new BytesArray(
+ "{\n"
+ + " \"processors\" : [\n"
+ + " {\"script\" : {\"lang\": \""
+ + MockScriptEngine.NAME
+ + "\", \"source\": \"my_script\"}}\n"
+ + " ]\n"
+ + "}"
+ );
+ BytesReference pipelineWithoutScript = new BytesArray(
+ "{\n" + " \"processors\" : [\n" + " {\"set\" : {\"field\": \"y\", \"value\": 0}}\n" + " ]\n" + "}"
+ );
- Consumer checkPipelineExists = (id) -> assertThat(client().admin().cluster().prepareGetPipeline(id)
- .get().pipelines().get(0).getId(), equalTo(id));
+ Consumer checkPipelineExists = (id) -> assertThat(
+ client().admin().cluster().prepareGetPipeline(id).get().pipelines().get(0).getId(),
+ equalTo(id)
+ );
client().admin().cluster().preparePutPipeline(pipelineIdWithScript, pipelineWithScript, XContentType.JSON).get();
client().admin().cluster().preparePutPipeline(pipelineIdWithoutScript, pipelineWithoutScript, XContentType.JSON).get();
@@ -150,7 +166,6 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
checkPipelineExists.accept(pipelineIdWithScript);
checkPipelineExists.accept(pipelineIdWithoutScript);
-
internalCluster().restartNode(internalCluster().getMasterName(), new InternalTestCluster.RestartCallback() {
@Override
@@ -169,20 +184,30 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.get();
- IllegalStateException exception = expectThrows(IllegalStateException.class,
+ IllegalStateException exception = expectThrows(
+ IllegalStateException.class,
() -> client().prepareIndex("index", "doc", "2")
.setSource("x", 0)
.setPipeline(pipelineIdWithScript)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get());
- assertThat(exception.getMessage(),
- equalTo("pipeline with id [" + pipelineIdWithScript + "] could not be loaded, caused by " +
- "[OpenSearchParseException[Error updating pipeline with id [" + pipelineIdWithScript + "]]; " +
- "nested: OpenSearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; " +
- "nested: IllegalArgumentException[cannot execute [inline] scripts];; " +
- "OpenSearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; " +
- "nested: IllegalArgumentException[cannot execute [inline] scripts];; java.lang.IllegalArgumentException: " +
- "cannot execute [inline] scripts]"));
+ .get()
+ );
+ assertThat(
+ exception.getMessage(),
+ equalTo(
+ "pipeline with id ["
+ + pipelineIdWithScript
+ + "] could not be loaded, caused by "
+ + "[OpenSearchParseException[Error updating pipeline with id ["
+ + pipelineIdWithScript
+ + "]]; "
+ + "nested: OpenSearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; "
+ + "nested: IllegalArgumentException[cannot execute [inline] scripts];; "
+ + "OpenSearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; "
+ + "nested: IllegalArgumentException[cannot execute [inline] scripts];; java.lang.IllegalArgumentException: "
+ + "cannot execute [inline] scripts]"
+ )
+ );
Map source = client().prepareGet("index", "doc", "1").get().getSource();
assertThat(source.get("x"), equalTo(0));
@@ -192,24 +217,30 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exception {
internalCluster().startNode();
- client().admin().cluster().preparePutStoredScript()
- .setId("1")
- .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptEngine.NAME +
- "\", \"source\": \"my_script\"} }"), XContentType.JSON)
- .get();
- BytesReference pipeline = new BytesArray("{\n" +
- " \"processors\" : [\n" +
- " {\"set\" : {\"field\": \"y\", \"value\": 0}},\n" +
- " {\"script\" : {\"id\": \"1\"}}\n" +
- " ]\n" +
- "}");
+ client().admin()
+ .cluster()
+ .preparePutStoredScript()
+ .setId("1")
+ .setContent(
+ new BytesArray("{\"script\": {\"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"my_script\"} }"),
+ XContentType.JSON
+ )
+ .get();
+ BytesReference pipeline = new BytesArray(
+ "{\n"
+ + " \"processors\" : [\n"
+ + " {\"set\" : {\"field\": \"y\", \"value\": 0}},\n"
+ + " {\"script\" : {\"id\": \"1\"}}\n"
+ + " ]\n"
+ + "}"
+ );
client().admin().cluster().preparePutPipeline("_id", pipeline, XContentType.JSON).get();
client().prepareIndex("index", "doc", "1")
- .setSource("x", 0)
- .setPipeline("_id")
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get();
+ .setSource("x", 0)
+ .setPipeline("_id")
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .get();
Map source = client().prepareGet("index", "doc", "1").get().getSource();
assertThat(source.get("x"), equalTo(0));
@@ -224,10 +255,10 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
ensureYellow("index");
client().prepareIndex("index", "doc", "2")
- .setSource("x", 0)
- .setPipeline("_id")
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get();
+ .setSource("x", 0)
+ .setPipeline("_id")
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .get();
source = client().prepareGet("index", "doc", "2").get().getSource();
assertThat(source.get("x"), equalTo(0));
@@ -237,23 +268,18 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
public void testWithDedicatedIngestNode() throws Exception {
String node = internalCluster().startNode();
- String ingestNode = internalCluster().startNode(Settings.builder()
- .put("node.master", false)
- .put("node.data", false)
- );
+ String ingestNode = internalCluster().startNode(Settings.builder().put("node.master", false).put("node.data", false));
- BytesReference pipeline = new BytesArray("{\n" +
- " \"processors\" : [\n" +
- " {\"set\" : {\"field\": \"y\", \"value\": 0}}\n" +
- " ]\n" +
- "}");
+ BytesReference pipeline = new BytesArray(
+ "{\n" + " \"processors\" : [\n" + " {\"set\" : {\"field\": \"y\", \"value\": 0}}\n" + " ]\n" + "}"
+ );
client().admin().cluster().preparePutPipeline("_id", pipeline, XContentType.JSON).get();
client().prepareIndex("index", "doc", "1")
- .setSource("x", 0)
- .setPipeline("_id")
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get();
+ .setSource("x", 0)
+ .setPipeline("_id")
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .get();
Map source = client().prepareGet("index", "doc", "1").get().getSource();
assertThat(source.get("x"), equalTo(0));
@@ -263,10 +289,10 @@ public class IngestRestartIT extends OpenSearchIntegTestCase {
internalCluster().restartNode(node, new InternalTestCluster.RestartCallback());
client(ingestNode).prepareIndex("index", "doc", "2")
- .setSource("x", 0)
- .setPipeline("_id")
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .get();
+ .setSource("x", 0)
+ .setPipeline("_id")
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .get();
source = client(ingestNode).prepareGet("index", "doc", "2").get().getSource();
assertThat(source.get("x"), equalTo(0));
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AbstractStringProcessor.java
index b04d8801c91..48bab9c9edc 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AbstractStringProcessor.java
@@ -89,8 +89,17 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
if (value instanceof String) {
newList.add(process((String) value));
} else {
- throw new IllegalArgumentException("value [" + value + "] of type [" + value.getClass().getName() +
- "] in list field [" + field + "] cannot be cast to [" + String.class.getName() + "]");
+ throw new IllegalArgumentException(
+ "value ["
+ + value
+ + "] of type ["
+ + value.getClass().getName()
+ + "] in list field ["
+ + field
+ + "] cannot be cast to ["
+ + String.class.getName()
+ + "]"
+ );
}
}
newValue = newList;
@@ -98,8 +107,9 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
if (val instanceof String) {
newValue = process((String) val);
} else {
- throw new IllegalArgumentException("field [" + field + "] of type [" + val.getClass().getName() + "] cannot be cast to [" +
- String.class.getName() + "]");
+ throw new IllegalArgumentException(
+ "field [" + field + "] of type [" + val.getClass().getName() + "] cannot be cast to [" + String.class.getName() + "]"
+ );
}
}
@@ -118,8 +128,12 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
}
@Override
- public AbstractStringProcessor> create(Map registry, String tag,
- String description, Map config) throws Exception {
+ public AbstractStringProcessor> create(
+ Map registry,
+ String tag,
+ String description,
+ Map config
+ ) throws Exception {
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
String targetField = ConfigurationUtils.readStringProperty(processorType, tag, config, "target_field", field);
@@ -127,8 +141,13 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
return newProcessor(tag, description, config, field, ignoreMissing, targetField);
}
- protected abstract AbstractStringProcessor> newProcessor(String processorTag, String description,
- Map config, String field,
- boolean ignoreMissing, String targetField);
+ protected abstract AbstractStringProcessor> newProcessor(
+ String processorTag,
+ String description,
+ Map config,
+ String field,
+ boolean ignoreMissing,
+ String targetField
+ );
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AppendProcessor.java
index a9db05ee612..392746f09bd 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/AppendProcessor.java
@@ -90,15 +90,23 @@ public final class AppendProcessor extends AbstractProcessor {
}
@Override
- public AppendProcessor create(Map registry, String processorTag,
- String description, Map config) throws Exception {
+ public AppendProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
boolean allowDuplicates = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "allow_duplicates", true);
- TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
- "field", field, scriptService);
- return new AppendProcessor(processorTag, description, compiledTemplate, ValueSource.wrap(value, scriptService),
- allowDuplicates);
+ TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);
+ return new AppendProcessor(
+ processorTag,
+ description,
+ compiledTemplate,
+ ValueSource.wrap(value, scriptService),
+ allowDuplicates
+ );
}
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
index abcde8aa47d..3bd11379758 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
@@ -69,8 +69,14 @@ public final class BytesProcessor extends AbstractStringProcessor {
}
@Override
- protected BytesProcessor newProcessor(String tag, String description, Map config, String field,
- boolean ignoreMissing, String targetField) {
+ protected BytesProcessor newProcessor(
+ String tag,
+ String description,
+ Map config,
+ String field,
+ boolean ignoreMissing,
+ String targetField
+ ) {
return new BytesProcessor(tag, description, field, ignoreMissing, targetField);
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
index b038455fd49..2a81fa5f498 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
@@ -60,12 +60,13 @@ public final class ConvertProcessor extends AbstractProcessor {
return Integer.decode(strValue);
}
return Integer.parseInt(strValue);
- } catch(NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to integer", e);
}
}
- }, LONG {
+ },
+ LONG {
@Override
public Object convert(Object value) {
try {
@@ -74,29 +75,32 @@ public final class ConvertProcessor extends AbstractProcessor {
return Long.decode(strValue);
}
return Long.parseLong(strValue);
- } catch(NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to long", e);
}
}
- }, DOUBLE {
+ },
+ DOUBLE {
@Override
public Object convert(Object value) {
try {
return Double.parseDouble(value.toString());
- } catch(NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to double", e);
}
}
- }, FLOAT {
+ },
+ FLOAT {
@Override
public Object convert(Object value) {
try {
return Float.parseFloat(value.toString());
- } catch(NumberFormatException e) {
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to float", e);
}
}
- }, BOOLEAN {
+ },
+ BOOLEAN {
@Override
public Object convert(Object value) {
if (value.toString().equalsIgnoreCase("true")) {
@@ -107,16 +111,18 @@ public final class ConvertProcessor extends AbstractProcessor {
throw new IllegalArgumentException("[" + value + "] is not a boolean value, cannot convert to boolean");
}
}
- }, STRING {
+ },
+ STRING {
@Override
public Object convert(Object value) {
return value.toString();
}
- }, AUTO {
+ },
+ AUTO {
@Override
public Object convert(Object value) {
if (!(value instanceof String)) {
- return value;
+ return value;
}
try {
return BOOLEAN.convert(value);
@@ -147,9 +153,13 @@ public final class ConvertProcessor extends AbstractProcessor {
public static Type fromString(String processorTag, String propertyName, String type) {
try {
return Type.valueOf(type.toUpperCase(Locale.ROOT));
- } catch(IllegalArgumentException e) {
- throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type +
- "] not supported, cannot convert field.");
+ } catch (IllegalArgumentException e) {
+ throw newConfigurationException(
+ TYPE,
+ processorTag,
+ propertyName,
+ "type [" + type + "] not supported, cannot convert field."
+ );
}
}
}
@@ -217,8 +227,12 @@ public final class ConvertProcessor extends AbstractProcessor {
public static final class Factory implements Processor.Factory {
@Override
- public ConvertProcessor create(Map registry, String processorTag,
- String description, Map config) throws Exception {
+ public ConvertProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvParser.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvParser.java
index 726de87a225..66bc01c32e1 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvParser.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvParser.java
@@ -42,7 +42,10 @@ final class CsvParser {
private static final char TAB = '\t';
private enum State {
- START, UNQUOTED, QUOTED, QUOTED_END
+ START,
+ UNQUOTED,
+ QUOTED,
+ QUOTED_END
}
private final char quote;
@@ -94,7 +97,7 @@ final class CsvParser {
}
}
- //we've reached end of string, we need to handle last field
+ // we've reached end of string, we need to handle last field
switch (state) {
case UNQUOTED:
setField(length);
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvProcessor.java
index 4cc8b718667..6c487f2dee4 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/CsvProcessor.java
@@ -59,7 +59,7 @@ public final class CsvProcessor extends AbstractProcessor {
public static final String TYPE = "csv";
- //visible for testing
+ // visible for testing
final String field;
final String[] headers;
final boolean trim;
@@ -68,8 +68,17 @@ public final class CsvProcessor extends AbstractProcessor {
final boolean ignoreMissing;
final Object emptyValue;
- CsvProcessor(String tag, String description, String field, String[] headers, boolean trim, char separator, char quote,
- boolean ignoreMissing, Object emptyValue) {
+ CsvProcessor(
+ String tag,
+ String description,
+ String field,
+ String[] headers,
+ boolean trim,
+ char separator,
+ char quote,
+ boolean ignoreMissing,
+ Object emptyValue
+ ) {
super(tag, description);
this.field = field;
this.headers = headers;
@@ -103,8 +112,12 @@ public final class CsvProcessor extends AbstractProcessor {
public static final class Factory implements org.opensearch.ingest.Processor.Factory {
@Override
- public CsvProcessor create(Map registry, String processorTag,
- String description, Map config) {
+ public CsvProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\"");
if (quote.length() != 1) {
@@ -116,7 +129,7 @@ public final class CsvProcessor extends AbstractProcessor {
}
boolean trim = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trim", false);
Object emptyValue = null;
- if(config.containsKey("empty_value")){
+ if (config.containsKey("empty_value")) {
emptyValue = ConfigurationUtils.readObject(TYPE, processorTag, config, "empty_value");
}
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
@@ -124,8 +137,17 @@ public final class CsvProcessor extends AbstractProcessor {
if (targetFields.isEmpty()) {
throw newConfigurationException(TYPE, processorTag, "target_fields", "target fields list can't be empty");
}
- return new CsvProcessor(processorTag, description, field, targetFields.toArray(new String[0]), trim, separator.charAt(0),
- quote.charAt(0), ignoreMissing, emptyValue);
+ return new CsvProcessor(
+ processorTag,
+ description,
+ field,
+ targetFields.toArray(new String[0]),
+ trim,
+ separator.charAt(0),
+ quote.charAt(0),
+ ignoreMissing,
+ emptyValue
+ );
}
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateFormat.java
index 77bf0a6d54b..7af26a8cc4c 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateFormat.java
@@ -61,9 +61,8 @@ enum DateFormat {
Function getFunction(String format, ZoneId timezone, Locale locale) {
return (date) -> {
TemporalAccessor accessor = DateFormatter.forPattern("iso8601").parse(date);
- //even though locale could be set to en-us, Locale.ROOT (following iso8601 calendar data rules) should be used
- return DateFormatters.from(accessor, Locale.ROOT, timezone)
- .withZoneSameInstant(timezone);
+ // even though locale could be set to en-us, Locale.ROOT (following iso8601 calendar data rules) should be used
+ return DateFormatters.from(accessor, Locale.ROOT, timezone).withZoneSameInstant(timezone);
};
}
@@ -93,19 +92,24 @@ enum DateFormat {
long base = Long.parseLong(date.substring(1, 16), 16);
// 1356138046000
long rest = Long.parseLong(date.substring(16, 24), 16);
- return ((base * 1000) - 10000) + (rest/1000000);
+ return ((base * 1000) - 10000) + (rest / 1000000);
}
},
Java {
- private final List FIELDS =
- Arrays.asList(NANO_OF_SECOND, SECOND_OF_DAY, MINUTE_OF_DAY, HOUR_OF_DAY, DAY_OF_MONTH, MONTH_OF_YEAR);
+ private final List FIELDS = Arrays.asList(
+ NANO_OF_SECOND,
+ SECOND_OF_DAY,
+ MINUTE_OF_DAY,
+ HOUR_OF_DAY,
+ DAY_OF_MONTH,
+ MONTH_OF_YEAR
+ );
@Override
Function getFunction(String format, ZoneId zoneId, Locale locale) {
boolean isUtc = ZoneOffset.UTC.equals(zoneId);
- DateFormatter dateFormatter = DateFormatter.forPattern(format)
- .withLocale(locale);
+ DateFormatter dateFormatter = DateFormatter.forPattern(format).withLocale(locale);
// if UTC zone is set here, the time zone specified in the format will be ignored, leading to wrong dates
if (isUtc == false) {
dateFormatter = dateFormatter.withZone(zoneId);
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateIndexNameProcessor.java
index a801a02b28e..e925db879a4 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateIndexNameProcessor.java
@@ -63,9 +63,16 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
private final ZoneId timezone;
private final List> dateFormats;
- DateIndexNameProcessor(String tag, String description, String field, List> dateFormats,
- ZoneId timezone, TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
- TemplateScript.Factory indexNameFormatTemplate) {
+ DateIndexNameProcessor(
+ String tag,
+ String description,
+ String field,
+ List> dateFormats,
+ ZoneId timezone,
+ TemplateScript.Factory indexNamePrefixTemplate,
+ TemplateScript.Factory dateRoundingTemplate,
+ TemplateScript.Factory indexNameFormatTemplate
+ ) {
super(tag, description);
this.field = field;
this.timezone = timezone;
@@ -91,7 +98,7 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
try {
dateTime = dateParser.apply(date);
} catch (Exception e) {
- //try the next parser and keep track of the exceptions
+ // try the next parser and keep track of the exceptions
lastException = ExceptionsHelper.useOrSuppress(lastException, e);
}
}
@@ -106,15 +113,20 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
DateFormatter formatter = DateFormatter.forPattern(indexNameFormat);
// use UTC instead of Z is string representation of UTC, so behaviour is the same between 6.x and 7
String zone = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId();
- StringBuilder builder = new StringBuilder()
- .append('<')
- .append(indexNamePrefix)
- .append('{')
- .append(formatter.format(dateTime)).append("||/").append(dateRounding)
- .append('{').append(indexNameFormat).append('|').append(zone).append('}')
- .append('}')
- .append('>');
- String dynamicIndexName = builder.toString();
+ StringBuilder builder = new StringBuilder().append('<')
+ .append(indexNamePrefix)
+ .append('{')
+ .append(formatter.format(dateTime))
+ .append("||/")
+ .append(dateRounding)
+ .append('{')
+ .append(indexNameFormat)
+ .append('|')
+ .append(zone)
+ .append('}')
+ .append('}')
+ .append('>');
+ String dynamicIndexName = builder.toString();
ingestDocument.setFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), dynamicIndexName);
return ingestDocument;
}
@@ -157,8 +169,12 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
}
@Override
- public DateIndexNameProcessor create(Map registry, String tag,
- String description, Map config) throws Exception {
+ public DateIndexNameProcessor create(
+ Map registry,
+ String tag,
+ String description,
+ Map config
+ ) throws Exception {
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
ZoneId timezone = timezoneString == null ? ZoneOffset.UTC : ZoneId.of(timezoneString);
@@ -182,16 +198,39 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
- TemplateScript.Factory indexNamePrefixTemplate =
- ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
+ TemplateScript.Factory indexNamePrefixTemplate = ConfigurationUtils.compileTemplate(
+ TYPE,
+ tag,
+ "index_name_prefix",
+ indexNamePrefix,
+ scriptService
+ );
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
- TemplateScript.Factory dateRoundingTemplate =
- ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
+ TemplateScript.Factory dateRoundingTemplate = ConfigurationUtils.compileTemplate(
+ TYPE,
+ tag,
+ "date_rounding",
+ dateRounding,
+ scriptService
+ );
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
- TemplateScript.Factory indexNameFormatTemplate =
- ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
- return new DateIndexNameProcessor(tag, description, field, dateFormats, timezone, indexNamePrefixTemplate,
- dateRoundingTemplate, indexNameFormatTemplate);
+ TemplateScript.Factory indexNameFormatTemplate = ConfigurationUtils.compileTemplate(
+ TYPE,
+ tag,
+ "index_name_format",
+ indexNameFormat,
+ scriptService
+ );
+ return new DateIndexNameProcessor(
+ tag,
+ description,
+ field,
+ dateFormats,
+ timezone,
+ indexNamePrefixTemplate,
+ dateRoundingTemplate,
+ indexNameFormatTemplate
+ );
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateProcessor.java
index c29877e596d..4b90ace3563 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DateProcessor.java
@@ -67,13 +67,28 @@ public final class DateProcessor extends AbstractProcessor {
private final List, Function>> dateParsers;
private final String outputFormat;
- DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
- String field, List formats, String targetField) {
+ DateProcessor(
+ String tag,
+ String description,
+ @Nullable TemplateScript.Factory timezone,
+ @Nullable TemplateScript.Factory locale,
+ String field,
+ List formats,
+ String targetField
+ ) {
this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT);
}
- DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
- String field, List formats, String targetField, String outputFormat) {
+ DateProcessor(
+ String tag,
+ String description,
+ @Nullable TemplateScript.Factory timezone,
+ @Nullable TemplateScript.Factory locale,
+ String field,
+ List formats,
+ String targetField,
+ String outputFormat
+ ) {
super(tag, description);
this.timezone = timezone;
this.locale = locale;
@@ -112,7 +127,7 @@ public final class DateProcessor extends AbstractProcessor {
try {
dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value);
} catch (Exception e) {
- //try the next parser and keep track of the exceptions
+ // try the next parser and keep track of the exceptions
lastException = ExceptionsHelper.useOrSuppress(lastException, e);
}
}
@@ -162,33 +177,48 @@ public final class DateProcessor extends AbstractProcessor {
this.scriptService = scriptService;
}
- public DateProcessor create(Map registry, String processorTag,
- String description, Map config) throws Exception {
+ public DateProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
TemplateScript.Factory compiledTimezoneTemplate = null;
if (timezoneString != null) {
- compiledTimezoneTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
- "timezone", timezoneString, scriptService);
+ compiledTimezoneTemplate = ConfigurationUtils.compileTemplate(
+ TYPE,
+ processorTag,
+ "timezone",
+ timezoneString,
+ scriptService
+ );
}
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "locale");
TemplateScript.Factory compiledLocaleTemplate = null;
if (localeString != null) {
- compiledLocaleTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
- "locale", localeString, scriptService);
+ compiledLocaleTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "locale", localeString, scriptService);
}
List formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
- String outputFormat =
- ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
+ String outputFormat = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
try {
DateFormatter.forPattern(outputFormat);
} catch (Exception e) {
throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e);
}
- return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
- targetField, outputFormat);
+ return new DateProcessor(
+ processorTag,
+ description,
+ compiledTimezoneTemplate,
+ compiledLocaleTemplate,
+ field,
+ formats,
+ targetField,
+ outputFormat
+ );
}
}
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DissectProcessor.java
index 5a46855e6b9..408982a74fc 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DissectProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DissectProcessor.java
@@ -43,7 +43,7 @@ import java.util.Map;
public final class DissectProcessor extends AbstractProcessor {
public static final String TYPE = "dissect";
- //package private members for testing
+ // package private members for testing
final String field;
final boolean ignoreMissing;
final String pattern;
@@ -79,8 +79,12 @@ public final class DissectProcessor extends AbstractProcessor {
public static final class Factory implements Processor.Factory {
@Override
- public DissectProcessor create(Map registry, String processorTag, String description,
- Map config) {
+ public DissectProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern");
String appendSeparator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "append_separator", "");
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
index a79a64ed746..39c2d67ac0b 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
@@ -79,8 +79,9 @@ public final class DotExpanderProcessor extends AbstractProcessor {
if (ingestDocument.hasField(partialPath)) {
Object val = ingestDocument.getFieldValue(partialPath, Object.class);
if ((val instanceof Map) == false) {
- throw new IllegalArgumentException("cannot expend [" + path + "], because [" + partialPath +
- "] is not an object field, but a value field");
+ throw new IllegalArgumentException(
+ "cannot expend [" + path + "], because [" + partialPath + "] is not an object field, but a value field"
+ );
}
} else {
break;
@@ -109,22 +110,33 @@ public final class DotExpanderProcessor extends AbstractProcessor {
public static final class Factory implements Processor.Factory {
@Override
- public Processor create(Map processorFactories, String tag, String description,
- Map config) throws Exception {
+ public Processor create(
+ Map processorFactories,
+ String tag,
+ String description,
+ Map config
+ ) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
if (field.contains(".") == false) {
- throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
- "field does not contain a dot");
+ throw ConfigurationUtils.newConfigurationException(
+ ConfigurationUtils.TAG_KEY,
+ tag,
+ "field",
+ "field does not contain a dot"
+ );
}
if (field.indexOf('.') == 0 || field.lastIndexOf('.') == field.length() - 1) {
- throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
- "Field can't start or end with a dot");
+ throw ConfigurationUtils.newConfigurationException(
+ ConfigurationUtils.TAG_KEY,
+ tag,
+ "field",
+ "Field can't start or end with a dot"
+ );
}
int firstIndex = -1;
for (int index = field.indexOf('.'); index != -1; index = field.indexOf('.', index + 1)) {
if (index - firstIndex == 1) {
- throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
- "No space between dots");
+ throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field", "No space between dots");
}
firstIndex = index;
}
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessor.java
index 033981bafcf..2f2e4f719dc 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessor.java
@@ -79,13 +79,21 @@ public final class FailProcessor extends AbstractProcessor {
}
@Override
- public FailProcessor create(Map registry, String processorTag,
- String description, Map config) throws Exception {
+ public FailProcessor create(
+ Map registry,
+ String processorTag,
+ String description,
+ Map config
+ ) throws Exception {
String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
- TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
- "message", message, scriptService);
+ TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(
+ TYPE,
+ processorTag,
+ "message",
+ message,
+ scriptService
+ );
return new FailProcessor(processorTag, description, compiledTemplate);
}
}
}
-
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
index a02ab8d8699..37320c0e900 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
@@ -48,4 +48,3 @@ public class FailProcessorException extends RuntimeException {
super(message);
}
}
-
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
index 53b04f2829d..741a4fb29cf 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
@@ -92,8 +92,13 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
}
}
- void innerExecute(int index, List> values, List