honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ingest: date_index_name processor template resolution (#31841) 

This change adds support for template snippet (e.g. {{foo}}) resolution
in the date_index_name processor. The following configuration options
will now resolve a templated value if so configured:

* index_name_prefix  (e.g "index_name_prefix": "myindex-{{foo}}-")
* date_rounding (e.g. "date_rounding" : "{{bar}}")
* index_name_format (e.g."index_name_format": "{{baz}}")
This commit is contained in:
Jake Landis 2018-07-11 10:13:41 -05:00 committed by GitHub
parent d76293f990
commit 51bb27a991
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 83 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View File

@ -32,6 +32,8 @@ import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
@ -42,21 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
public static final String TYPE = "date_index_name";
private final String field;
private final String indexNamePrefix;
private final String dateRounding;
private final String indexNameFormat;
private final TemplateScript.Factory indexNamePrefixTemplate;
private final TemplateScript.Factory dateRoundingTemplate;
private final TemplateScript.Factory indexNameFormatTemplate;
private final DateTimeZone timezone;
private final List<Function<String, DateTime>> dateFormats;
DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
String indexNamePrefix, String dateRounding, String indexNameFormat) {
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
TemplateScript.Factory indexNameFormatTemplate) {
super(tag);
this.field = field;
this.timezone = timezone;
this.dateFormats = dateFormats;
this.indexNamePrefix = indexNamePrefix;
this.dateRounding = dateRounding;
this.indexNameFormat = indexNameFormat;
this.indexNamePrefixTemplate = indexNamePrefixTemplate;
this.dateRoundingTemplate = dateRoundingTemplate;
this.indexNameFormatTemplate = indexNameFormatTemplate;
}
@Override
@ -83,6 +86,9 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
if (dateTime == null) {
throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
}
String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate);
String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);
DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
StringBuilder builder = new StringBuilder()
@ -106,16 +112,16 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
return field;
}
String getIndexNamePrefix() {
return indexNamePrefix;
TemplateScript.Factory getIndexNamePrefixTemplate() {
return indexNamePrefixTemplate;
}
String getDateRounding() {
return dateRounding;
TemplateScript.Factory getDateRoundingTemplate() {
return dateRoundingTemplate;
}
String getIndexNameFormat() {
return indexNameFormat;
TemplateScript.Factory getIndexNameFormatTemplate() {
return indexNameFormatTemplate;
}
DateTimeZone getTimezone() {
@ -128,6 +134,12 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
public static final class Factory implements Processor.Factory {
private final ScriptService scriptService;
public Factory(ScriptService scriptService) {
this.scriptService = scriptService;
}
@Override
public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
@ -154,9 +166,16 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
TemplateScript.Factory indexNamePrefixTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
TemplateScript.Factory dateRoundingTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat);
TemplateScript.Factory indexNameFormatTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
dateRoundingTemplate, indexNameFormatTemplate);
}
}

6
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
View File

@ -73,7 +73,7 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl
processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService));
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService));
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters)));
processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
@ -97,12 +97,12 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl
Supplier<DiscoveryNodes> nodesInCluster) {
return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
}
@Override
public List<Setting<?>> getSettings() {
return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME);
}
private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) {
long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis();
long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis();

19
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
View File

@ -20,18 +20,20 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import org.joda.time.DateTimeZone;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
public class DateIndexNameFactoryTests extends ESTestCase {
public void testDefaults() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("date_rounding", "y");
@ -39,14 +41,14 @@ public class DateIndexNameFactoryTests extends ESTestCase {
DateIndexNameProcessor processor = factory.create(null, null, config);
assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
assertThat(processor.getField(), Matchers.equalTo("_field"));
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
}
public void testSpecifyOptionalSettings() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("index_name_prefix", "_prefix");
@ -63,7 +65,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("index_name_format", "yyyyMMdd");
processor = factory.create(null, null, config);
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));
config = new HashMap<>();
config.put("field", "_field");
@ -80,11 +82,11 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("date_rounding", "y");
processor = factory.create(null, null, config);
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
}
public void testRequiredFields() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("date_rounding", "y");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
@ -95,5 +97,4 @@ public class DateIndexNameFactoryTests extends ESTestCase {
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
}
}

45
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
View File

@ -19,11 +19,14 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;
@ -33,11 +36,8 @@ public class DateIndexNameProcessorTests extends ESTestCase {
public void testJodaPattern() throws Exception {
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
DateIndexNameProcessor processor = new DateIndexNameProcessor(
"_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
"events-", "y", "yyyyMMdd"
);
DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
processor.execute(document);
@ -46,7 +46,7 @@ public class DateIndexNameProcessorTests extends ESTestCase {
public void testTAI64N()throws Exception {
Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
@ -56,7 +56,7 @@ public class DateIndexNameProcessorTests extends ESTestCase {
public void testUnixMs()throws Exception {
Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "1000500"));
@ -71,7 +71,7 @@ public class DateIndexNameProcessorTests extends ESTestCase {
public void testUnix()throws Exception {
Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "1000.5"));
@ -79,4 +79,33 @@ public class DateIndexNameProcessorTests extends ESTestCase {
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
}
public void testTemplatedFields() throws Exception {
String indexNamePrefix = randomAlphaOfLength(10);
String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
String date = Integer.toString(randomInt());
Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = createProcessor("_field",
Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix,
dateRounding, indexNameFormat);
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", date));
dateProcessor.execute(document);
assertThat(document.getSourceAndMetadata().get("_index"),
equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat)
.print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>"));
}
private DateIndexNameProcessor createProcessor(String field, List<Function<String, DateTime>> dateFormats,
DateTimeZone timezone, String indexNamePrefix, String dateRounding,
String indexNameFormat) {
return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone,
new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix),
new TestTemplateService.MockTemplateScript.Factory(dateRounding),
new TestTemplateService.MockTemplateScript.Factory(indexNameFormat)
);
}
}