From eeeb355adfa67752e2dbd332b3870f1242251153 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 9 Sep 2020 11:11:02 -0500 Subject: [PATCH] Configurable output format for date processor (#61324) (#62175) --- .../ingest/common/DateProcessor.java | 27 +++++++++++-- .../common/DateProcessorFactoryTests.java | 38 +++++++++++++++++++ .../ingest/common/DateProcessorTests.java | 14 +++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 4fb95a3080c..30bae525056 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -43,17 +43,24 @@ public final class DateProcessor extends AbstractProcessor { public static final String TYPE = "date"; static final String DEFAULT_TARGET_FIELD = "@timestamp"; - private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + static final String DEFAULT_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"; + private final DateFormatter formatter; private final TemplateScript.Factory timezone; private final TemplateScript.Factory locale; private final String field; private final String targetField; private final List formats; private final List, Function>> dateParsers; + private final String outputFormat; DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale, String field, List formats, String targetField) { + this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT); + } + + DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale, + String field, List formats, String targetField, String outputFormat) { super(tag, description); this.timezone = timezone; this.locale = locale; @@ -65,6 +72,8 @@ public final class DateProcessor extends AbstractProcessor { DateFormat dateFormat = DateFormat.fromString(format); dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params))); } + this.outputFormat = outputFormat; + formatter = DateFormatter.forPattern(this.outputFormat); } private ZoneId newDateTimeZone(Map params) { @@ -99,7 +108,7 @@ public final class DateProcessor extends AbstractProcessor { throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException); } - ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime)); + ingestDocument.setFieldValue(targetField, formatter.format(dateTime)); return ingestDocument; } @@ -128,6 +137,10 @@ public final class DateProcessor extends AbstractProcessor { return formats; } + String getOutputFormat() { + return outputFormat; + } + public static final class Factory implements Processor.Factory { private final ScriptService scriptService; @@ -153,8 +166,16 @@ public final class DateProcessor extends AbstractProcessor { "locale", localeString, scriptService); } List formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats"); + String outputFormat = + ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT); + try { + DateFormatter.forPattern(outputFormat); + } catch (Exception e) { + throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e); + } + return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats, - targetField); + targetField, outputFormat); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java index 58336d8911e..fae5b219554 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java @@ -146,4 +146,42 @@ public class DateProcessorFactoryTests extends ESTestCase { DateProcessor processor = factory.create(null, null, null, config); assertThat(processor.getTargetField(), equalTo(targetField)); } + + public void testParseOutputFormat() throws Exception { + final String outputFormat = "dd:MM:yyyy"; + Map config = new HashMap<>(); + String sourceField = randomAlphaOfLengthBetween(1, 10); + String targetField = randomAlphaOfLengthBetween(1, 10); + config.put("field", sourceField); + config.put("target_field", targetField); + config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")); + config.put("output_format", outputFormat); + DateProcessor processor = factory.create(null, null, null, config); + assertThat(processor.getOutputFormat(), equalTo(outputFormat)); + } + + public void testDefaultOutputFormat() throws Exception { + Map config = new HashMap<>(); + String sourceField = randomAlphaOfLengthBetween(1, 10); + String targetField = randomAlphaOfLengthBetween(1, 10); + config.put("field", sourceField); + config.put("target_field", targetField); + config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")); + DateProcessor processor = factory.create(null, null, null, config); + assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT)); + } + + public void testInvalidOutputFormatRejected() throws Exception { + final String outputFormat = "invalid_date_format"; + Map config = new HashMap<>(); + String sourceField = randomAlphaOfLengthBetween(1, 10); + String targetField = randomAlphaOfLengthBetween(1, 10); + config.put("field", sourceField); + config.put("target_field", targetField); + config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")); + config.put("output_format", outputFormat); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config)); + assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]")); + } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 8f3fd31ab4a..9a8e105eba5 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.script.TemplateScript; import org.elasticsearch.test.ESTestCase; +import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -224,4 +225,17 @@ public class DateProcessorTests extends ESTestCase { assertThat(e.getMessage(), equalTo("unable to parse date [2010]")); assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid")); } + + public void testOutputFormat() { + long nanosAfterEpoch = randomLongBetween(1, 999999); + DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), null, null, null, + "date_as_string", Collections.singletonList("iso8601"), "date_as_date", "HH:mm:ss.SSSSSSSSS"); + Map document = new HashMap<>(); + document.put("date_as_string", Instant.EPOCH.plusNanos(nanosAfterEpoch).toString()); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + // output format is time only with nanosecond precision + String expectedDate = "00:00:00." + String.format(Locale.ROOT, "%09d", nanosAfterEpoch); + assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate)); + } }