honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Configurable output format for date processor (#61324) (#62175)

This commit is contained in:
Dan Hermann 2020-09-09 11:11:02 -05:00 committed by GitHub
parent e181e24d48
commit eeeb355adf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 76 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
View File

@ -43,17 +43,24 @@ public final class DateProcessor extends AbstractProcessor {
public static final String TYPE = "date";
static final String DEFAULT_TARGET_FIELD = "@timestamp";
private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
static final String DEFAULT_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
private final DateFormatter formatter;
private final TemplateScript.Factory timezone;
private final TemplateScript.Factory locale;
private final String field;
private final String targetField;
private final List<String> formats;
private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
private final String outputFormat;
DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField) {
this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT);
}
DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField, String outputFormat) {
super(tag, description);
this.timezone = timezone;
this.locale = locale;
@ -65,6 +72,8 @@ public final class DateProcessor extends AbstractProcessor {
DateFormat dateFormat = DateFormat.fromString(format);
dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
}
this.outputFormat = outputFormat;
formatter = DateFormatter.forPattern(this.outputFormat);
}
private ZoneId newDateTimeZone(Map<String, Object> params) {
@ -99,7 +108,7 @@ public final class DateProcessor extends AbstractProcessor {
throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
}
ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime));
ingestDocument.setFieldValue(targetField, formatter.format(dateTime));
return ingestDocument;
}
@ -128,6 +137,10 @@ public final class DateProcessor extends AbstractProcessor {
return formats;
}
String getOutputFormat() {
return outputFormat;
}
public static final class Factory implements Processor.Factory {
private final ScriptService scriptService;
@ -153,8 +166,16 @@ public final class DateProcessor extends AbstractProcessor {
"locale", localeString, scriptService);
}
List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
String outputFormat =
ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
try {
DateFormatter.forPattern(outputFormat);
} catch (Exception e) {
throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e);
}
return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
targetField);
targetField, outputFormat);
}
}
}

38
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
View File

@ -146,4 +146,42 @@ public class DateProcessorFactoryTests extends ESTestCase {
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getTargetField(), equalTo(targetField));
}
public void testParseOutputFormat() throws Exception {
final String outputFormat = "dd:MM:yyyy";
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getOutputFormat(), equalTo(outputFormat));
}
public void testDefaultOutputFormat() throws Exception {
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
}
public void testInvalidOutputFormatRejected() throws Exception {
final String outputFormat = "invalid_date_format";
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
}
}

14
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.test.ESTestCase;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
@ -224,4 +225,17 @@ public class DateProcessorTests extends ESTestCase {
assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid"));
}
public void testOutputFormat() {
long nanosAfterEpoch = randomLongBetween(1, 999999);
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), null, null, null,
"date_as_string", Collections.singletonList("iso8601"), "date_as_date", "HH:mm:ss.SSSSSSSSS");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", Instant.EPOCH.plusNanos(nanosAfterEpoch).toString());
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
// output format is time only with nanosecond precision
String expectedDate = "00:00:00." + String.format(Locale.ROOT, "%09d", nanosAfterEpoch);
assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate));
}
}