honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Date processor: simplify switch to identify the specified date format

This commit is contained in:
javanna 2016-01-22 16:28:04 +01:00 committed by Luca Cavanna
parent 17a36aecff
commit 38ea9c7d13
4 changed files with 57 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
core/src/main/java/org/elasticsearch/ingest/processor/DateFormat.java
View File

@ -25,36 +25,30 @@ import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.ISODateTimeFormat; import org.joda.time.format.ISODateTimeFormat;
import java.util.Locale; import java.util.Locale;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
enum DateFormat { enum DateFormat {
Iso8601 { Iso8601 {
@Override @Override
Function<String, DateTime> getFunction(DateTimeZone timezone) { Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return ISODateTimeFormat.dateTimeParser().withZone(timezone)::parseDateTime; return ISODateTimeFormat.dateTimeParser().withZone(timezone)::parseDateTime;
} }
}, },
Unix { Unix {
@Override @Override
Function<String, DateTime> getFunction(DateTimeZone timezone) { Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime((long)(Float.parseFloat(date) * 1000), timezone); return (date) -> new DateTime((long)(Float.parseFloat(date) * 1000), timezone);
} }
}, },
UnixMs { UnixMs {
@Override @Override
Function<String, DateTime> getFunction(DateTimeZone timezone) { Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime(Long.parseLong(date), timezone); return (date) -> new DateTime(Long.parseLong(date), timezone);
} }
@Override
public String toString() {
return "UNIX_MS";
}
}, },
Tai64n { Tai64n {
@Override @Override
Function<String, DateTime> getFunction(DateTimeZone timezone) { Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime(parseMillis(date), timezone); return (date) -> new DateTime(parseMillis(date), timezone);
} }
@ -67,33 +61,30 @@ enum DateFormat {
long rest = Long.parseLong(date.substring(16, 24), 16); long rest = Long.parseLong(date.substring(16, 24), 16);
return ((base * 1000) - 10000) + (rest/1000000); return ((base * 1000) - 10000) + (rest/1000000);
} }
}; },
Joda {
abstract Function<String, DateTime> getFunction(DateTimeZone timezone); @Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
static Optional<DateFormat> fromString(String format) { return DateTimeFormat.forPattern(format)
switch (format) {
case "ISO8601":
return Optional.of(Iso8601);
case "UNIX":
return Optional.of(Unix);
case "UNIX_MS":
return Optional.of(UnixMs);
case "TAI64N":
return Optional.of(Tai64n);
default:
return Optional.empty();
}
}
static Function<String, DateTime> getJodaFunction(String matchFormat, DateTimeZone timezone, Locale locale) {
return DateTimeFormat.forPattern(matchFormat)
.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()) .withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear())
.withZone(timezone).withLocale(locale)::parseDateTime; .withZone(timezone).withLocale(locale)::parseDateTime;
} }
};
@Override abstract Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale);
public String toString() {
return name().toUpperCase(Locale.ROOT); static DateFormat fromString(String format) {
switch (format) {
case "ISO8601":
return Iso8601;
case "UNIX":
return Unix;
case "UNIX_MS":
return UnixMs;
case "TAI64N":
return Tai64n;
default:
return Joda;
}
} }
} }

14
core/src/main/java/org/elasticsearch/ingest/processor/DateProcessor.java
View File

@ -22,8 +22,8 @@ package org.elasticsearch.ingest.processor;
import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessor;
import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.ingest.core.IngestDocument;
import org.elasticsearch.ingest.core.ConfigurationUtils; import org.elasticsearch.ingest.core.ConfigurationUtils;
import org.elasticsearch.ingest.core.IngestDocument;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
import org.joda.time.format.ISODateTimeFormat; import org.joda.time.format.ISODateTimeFormat;
@ -33,7 +33,6 @@ import java.util.IllformedLocaleException;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
public final class DateProcessor extends AbstractProcessor { public final class DateProcessor extends AbstractProcessor {
@ -57,21 +56,14 @@ public final class DateProcessor extends AbstractProcessor {
this.matchFormats = matchFormats; this.matchFormats = matchFormats;
this.dateParsers = new ArrayList<>(); this.dateParsers = new ArrayList<>();
for (String matchFormat : matchFormats) { for (String matchFormat : matchFormats) {
Optional<DateFormat> dateFormat = DateFormat.fromString(matchFormat); DateFormat dateFormat = DateFormat.fromString(matchFormat);
Function<String, DateTime> stringToDateFunction; dateParsers.add(dateFormat.getFunction(matchFormat, timezone, locale));
if (dateFormat.isPresent()) {
stringToDateFunction = dateFormat.get().getFunction(timezone);
} else {
stringToDateFunction = DateFormat.getJodaFunction(matchFormat, timezone, locale);
}
dateParsers.add(stringToDateFunction);
} }
} }
@Override @Override
public void execute(IngestDocument ingestDocument) { public void execute(IngestDocument ingestDocument) {
String value = ingestDocument.getFieldValue(matchField, String.class); String value = ingestDocument.getFieldValue(matchField, String.class);
// TODO(talevy): handle custom timestamp fields
DateTime dateTime = null; DateTime dateTime = null;
Exception lastException = null; Exception lastException = null;

32
core/src/test/java/org/elasticsearch/ingest/processor/DateFormatTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.ingest.processor; package org.elasticsearch.ingest.processor;
import org.elasticsearch.ingest.processor.DateFormat;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
@ -28,7 +27,6 @@ import java.time.Instant;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Locale; import java.util.Locale;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsEqual.equalTo;
@ -36,7 +34,7 @@ import static org.hamcrest.core.IsEqual.equalTo;
public class DateFormatTests extends ESTestCase { public class DateFormatTests extends ESTestCase {
public void testParseJoda() { public void testParseJoda() {
Function<String, DateTime> jodaFunction = DateFormat.getJodaFunction("MMM dd HH:mm:ss Z", DateTimeZone.forOffsetHours(-8), Locale.ENGLISH); Function<String, DateTime> jodaFunction = DateFormat.Joda.getFunction("MMM dd HH:mm:ss Z", DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis()) assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis())
.atZone(ZoneId.of("GMT-8")) .atZone(ZoneId.of("GMT-8"))
.format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)), .format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)),
@ -44,19 +42,19 @@ public class DateFormatTests extends ESTestCase {
} }
public void testParseUnixMs() { public void testParseUnixMs() {
assertThat(DateFormat.UnixMs.getFunction(DateTimeZone.UTC).apply("1000500").getMillis(), equalTo(1000500L)); assertThat(DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null).apply("1000500").getMillis(), equalTo(1000500L));
} }
public void testParseUnix() { public void testParseUnix() {
assertThat(DateFormat.Unix.getFunction(DateTimeZone.UTC).apply("1000.5").getMillis(), equalTo(1000500L)); assertThat(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null).apply("1000.5").getMillis(), equalTo(1000500L));
} }
public void testParseISO8601() { public void testParseISO8601() {
assertThat(DateFormat.Iso8601.getFunction(DateTimeZone.UTC).apply("2001-01-01T00:00:00-0800").getMillis(), equalTo(978336000000L)); assertThat(DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null).apply("2001-01-01T00:00:00-0800").getMillis(), equalTo(978336000000L));
} }
public void testParseISO8601Failure() { public void testParseISO8601Failure() {
Function<String, DateTime> function = DateFormat.Iso8601.getFunction(DateTimeZone.UTC); Function<String, DateTime> function = DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null);
try { try {
function.apply("2001-01-0:00-0800"); function.apply("2001-01-0:00-0800");
fail("parse should have failed"); fail("parse should have failed");
@ -68,18 +66,18 @@ public class DateFormatTests extends ESTestCase {
public void testTAI64NParse() { public void testTAI64NParse() {
String input = "4000000050d506482dbdf024"; String input = "4000000050d506482dbdf024";
String expected = "2012-12-22T03:00:46.767+02:00"; String expected = "2012-12-22T03:00:46.767+02:00";
assertThat(DateFormat.Tai64n.getFunction(DateTimeZone.forOffsetHours(2)).apply((randomBoolean() ? "@" : "") + input).toString(), equalTo(expected)); assertThat(DateFormat.Tai64n.getFunction(null, DateTimeZone.forOffsetHours(2), null).apply((randomBoolean() ? "@" : "") + input).toString(), equalTo(expected));
} }
public void testFromString() { public void testFromString() {
assertThat(DateFormat.fromString("UNIX_MS"), equalTo(Optional.of(DateFormat.UnixMs))); assertThat(DateFormat.fromString("UNIX_MS"), equalTo(DateFormat.UnixMs));
assertThat(DateFormat.fromString("unix_ms"), equalTo(Optional.empty())); assertThat(DateFormat.fromString("unix_ms"), equalTo(DateFormat.Joda));
assertThat(DateFormat.fromString("UNIX"), equalTo(Optional.of(DateFormat.Unix))); assertThat(DateFormat.fromString("UNIX"), equalTo(DateFormat.Unix));
assertThat(DateFormat.fromString("unix"), equalTo(Optional.empty())); assertThat(DateFormat.fromString("unix"), equalTo(DateFormat.Joda));
assertThat(DateFormat.fromString("ISO8601"), equalTo(Optional.of(DateFormat.Iso8601))); assertThat(DateFormat.fromString("ISO8601"), equalTo(DateFormat.Iso8601));
assertThat(DateFormat.fromString("iso8601"), equalTo(Optional.empty())); assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Joda));
assertThat(DateFormat.fromString("TAI64N"), equalTo(Optional.of(DateFormat.Tai64n))); assertThat(DateFormat.fromString("TAI64N"), equalTo(DateFormat.Tai64n));
assertThat(DateFormat.fromString("tai64n"), equalTo(Optional.empty())); assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Joda));
assertThat(DateFormat.fromString("prefix-" + randomAsciiOfLengthBetween(1, 10)), equalTo(Optional.empty())); assertThat(DateFormat.fromString("prefix-" + randomAsciiOfLengthBetween(1, 10)), equalTo(DateFormat.Joda));
} }
} }

16
core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorTests.java
View File

@ -84,6 +84,16 @@ public class DateProcessorTests extends ESTestCase {
} }
} }
public void testInvalidJodaPattern() {
try {
new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
"date_as_string", Collections.singletonList("invalid pattern"), "date_as_date");
fail("date processor initialization should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("Illegal pattern component: i"));
}
}
public void testJodaPatternLocale() { public void testJodaPatternLocale() {
DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ITALIAN, DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ITALIAN,
"date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date"); "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
@ -106,7 +116,7 @@ public class DateProcessorTests extends ESTestCase {
public void testTAI64N() { public void testTAI64N() {
DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forOffsetHours(2), randomLocale(random()), DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forOffsetHours(2), randomLocale(random()),
"date_as_string", Collections.singletonList(DateFormat.Tai64n.toString()), "date_as_date"); "date_as_string", Collections.singletonList("TAI64N"), "date_as_date");
Map<String, Object> document = new HashMap<>(); Map<String, Object> document = new HashMap<>();
String dateAsString = (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"; String dateAsString = (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024";
document.put("date_as_string", dateAsString); document.put("date_as_string", dateAsString);
@ -117,7 +127,7 @@ public class DateProcessorTests extends ESTestCase {
public void testUnixMs() { public void testUnixMs() {
DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()), DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
"date_as_string", Collections.singletonList(DateFormat.UnixMs.toString()), "date_as_date"); "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
Map<String, Object> document = new HashMap<>(); Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "1000500"); document.put("date_as_string", "1000500");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@ -127,7 +137,7 @@ public class DateProcessorTests extends ESTestCase {
public void testUnix() { public void testUnix() {
DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()), DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.UTC, randomLocale(random()),
"date_as_string", Collections.singletonList(DateFormat.Unix.toString()), "date_as_date"); "date_as_string", Collections.singletonList("UNIX"), "date_as_date");
Map<String, Object> document = new HashMap<>(); Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "1000.5"); document.put("date_as_string", "1000.5");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);