From d775347b06b70d3a25d28d61f4b8d74f4df0a22b Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 11 Aug 2017 13:02:42 -0700 Subject: [PATCH] TimestampSpec: Have "auto" detect timestamps in almost-iso format. (#4682) Fixes #4082. --- .../druid/data/input/impl/TimestampSpec.java | 2 -- .../util/common/parsers/TimestampParser.java | 31 +++++++++++++++++-- .../common/parsers/TimestampParserTest.java | 28 +++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/io/druid/data/input/impl/TimestampSpec.java b/api/src/main/java/io/druid/data/input/impl/TimestampSpec.java index aad2ea16871..8f83d25b714 100644 --- a/api/src/main/java/io/druid/data/input/impl/TimestampSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/TimestampSpec.java @@ -22,9 +22,7 @@ package io.druid.data.input.impl; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; - import io.druid.java.util.common.parsers.TimestampParser; - import org.joda.time.DateTime; import java.util.List; diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/TimestampParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/TimestampParser.java index f7179002d88..fdff821e288 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/TimestampParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/TimestampParser.java @@ -25,6 +25,9 @@ import io.druid.java.util.common.IAE; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.DateTimeFormatterBuilder; +import org.joda.time.format.DateTimeParser; +import org.joda.time.format.ISODateTimeFormat; public class TimestampParser { @@ -34,16 +37,16 @@ public class TimestampParser { if (format.equalsIgnoreCase("auto")) { // Could be iso or millis + final DateTimeFormatter parser = createAutoParser(); return new Function() { @Override public DateTime apply(String input) { Preconditions.checkArgument(input != null && !input.isEmpty(), "null timestamp"); - for (int i = 0; i < input.length(); i++) { if (input.charAt(i) < '0' || input.charAt(i) > '9') { - return new DateTime(ParserUtils.stripQuotes(input)); + return parser.parseDateTime(ParserUtils.stripQuotes(input)); } } @@ -161,4 +164,28 @@ public class TimestampParser } }; } + + private static DateTimeFormatter createAutoParser() + { + final DateTimeFormatter offsetElement = new DateTimeFormatterBuilder() + .appendTimeZoneOffset("Z", true, 2, 4) + .toFormatter(); + + DateTimeParser timeOrOffset = new DateTimeFormatterBuilder() + .append( + null, + new DateTimeParser[]{ + new DateTimeFormatterBuilder().appendLiteral('T').toParser(), + new DateTimeFormatterBuilder().appendLiteral(' ').toParser() + } + ) + .appendOptional(ISODateTimeFormat.timeElementParser().getParser()) + .appendOptional(offsetElement.getParser()) + .toParser(); + + return new DateTimeFormatterBuilder() + .append(ISODateTimeFormat.dateElementParser()) + .appendOptional(timeOrOffset) + .toFormatter(); + } } diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/TimestampParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/TimestampParserTest.java index 04fb245c3e1..35ce86e1c9d 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/TimestampParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/TimestampParserTest.java @@ -22,10 +22,14 @@ package io.druid.java.util.common.parsers; import com.google.common.base.Function; import org.joda.time.DateTime; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class TimestampParserTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); @Test public void testStripQuotes() throws Exception @@ -40,9 +44,33 @@ public class TimestampParserTest final Function parser = TimestampParser.createObjectTimestampParser("auto"); Assert.assertEquals(new DateTime("2009-02-13T23:31:30Z"), parser.apply("1234567890000")); Assert.assertEquals(new DateTime("2009-02-13T23:31:30Z"), parser.apply("2009-02-13T23:31:30Z")); + Assert.assertEquals(new DateTime("2009-02-13T23:31:30-08:00"), parser.apply("2009-02-13T23:31:30-08:00")); + Assert.assertEquals(new DateTime("2009-02-13T23:31:30Z"), parser.apply("2009-02-13 23:31:30Z")); + Assert.assertEquals(new DateTime("2009-02-13T23:31:30-08:00"), parser.apply("2009-02-13 23:31:30-08:00")); + Assert.assertEquals(new DateTime("2009-02-13T00:00:00Z"), parser.apply("2009-02-13")); + Assert.assertEquals(new DateTime("2009-02-13T00:00:00Z"), parser.apply("\"2009-02-13\"")); + Assert.assertEquals(new DateTime("2009-02-13T23:31:30Z"), parser.apply("2009-02-13 23:31:30")); Assert.assertEquals(new DateTime("2009-02-13T23:31:30Z"), parser.apply(1234567890000L)); } + @Test + public void testAutoNull() throws Exception + { + final Function parser = TimestampParser.createObjectTimestampParser("auto"); + + expectedException.expect(IllegalArgumentException.class); + parser.apply(null); + } + + @Test + public void testAutoInvalid() throws Exception + { + final Function parser = TimestampParser.createObjectTimestampParser("auto"); + + expectedException.expect(IllegalArgumentException.class); + parser.apply("asdf"); + } + @Test public void testRuby() throws Exception {