Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add "asMillis" option to "timeFormat" extractionFn. (#3733) 

This is useful for chaining extractionFns that all want to treat time as millis,
such as having a javascript extractionFn after a timeFormat.
This commit is contained in:
Gian Merlino 2016-12-02 13:45:16 -08:00 committed by Fangjin Yang
parent 102375d9bb
commit 353fee79dd
6 changed files with 91 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/content/querying/dimensionspecs.md
View File

@ -195,13 +195,15 @@ For a regular dimension, it assumes the string is formatted in
* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](http://www.oracle.com/technetwork/java/javase/java8locales-2095355.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc.
* `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone)
* `granularity` : [granularity](granularities.html) to apply before formatting, or omit to not apply any granularity.
* `asMillis` : boolean value, set to true to treat input strings as millis rather than ISO8601 strings. Additionally, if `format` is null or not specified, output will be in millis rather than ISO8601.
```json
{ "type" : "timeFormat",
"format" : <output_format> (optional),
"timeZone" : <time_zone> (optional),
"locale" : <locale> (optional),
"granularity" : <granularity> (optional) }
"granularity" : <granularity> (optional) },
"asMillis" : <true or false> (optional) }
```
For example, the following dimension spec returns the day of the week for Montréal in French:

57
processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java
View File

@ -20,6 +20,8 @@
package io.druid.query.extraction;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import io.druid.common.guava.GuavaUtils;
import io.druid.granularity.QueryGranularities;
import io.druid.granularity.QueryGranularity;
import io.druid.java.util.common.StringUtils;
@ -38,22 +40,33 @@ public class TimeFormatExtractionFn implements ExtractionFn
private final DateTimeZone tz;
private final Locale locale;
private final QueryGranularity granularity;
private final boolean asMillis;
private final DateTimeFormatter formatter;
public TimeFormatExtractionFn(
@JsonProperty("format") String format,
@JsonProperty("timeZone") DateTimeZone tz,
@JsonProperty("locale") String localeString,
@JsonProperty("granularity") QueryGranularity granularity
@JsonProperty("granularity") QueryGranularity granularity,
@JsonProperty("asMillis") boolean asMillis
)
{
this.format = format;
this.tz = tz;
this.locale = localeString == null ? null : Locale.forLanguageTag(localeString);
this.granularity = granularity == null ? QueryGranularities.NONE : granularity;
this.formatter = (format == null ? ISODateTimeFormat.dateTime() : DateTimeFormat.forPattern(format))
.withZone(tz == null ? DateTimeZone.UTC : tz)
.withLocale(locale);
if (asMillis && format == null) {
Preconditions.checkArgument(tz == null, "timeZone requires a format");
Preconditions.checkArgument(localeString == null, "locale requires a format");
this.formatter = null;
} else {
this.formatter = (format == null ? ISODateTimeFormat.dateTime() : DateTimeFormat.forPattern(format))
.withZone(tz == null ? DateTimeZone.UTC : tz)
.withLocale(locale);
}
this.asMillis = asMillis;
}
@JsonProperty
@ -84,29 +97,43 @@ public class TimeFormatExtractionFn implements ExtractionFn
return granularity;
}
@JsonProperty
public boolean isAsMillis()
{
return asMillis;
}
@Override
public byte[] getCacheKey()
{
final byte[] exprBytes = StringUtils.toUtf8(format + "\u0001" + tz.getID() + "\u0001" + locale.toLanguageTag());
final byte[] granularityCacheKey = granularity.cacheKey();
return ByteBuffer.allocate(2 + exprBytes.length + granularityCacheKey.length)
return ByteBuffer.allocate(4 + exprBytes.length + granularityCacheKey.length)
.put(ExtractionCacheHelper.CACHE_TYPE_ID_TIME_FORMAT)
.put(exprBytes)
.put((byte) 0xFF)
.put(granularityCacheKey)
.put((byte) 0xFF)
.put(asMillis ? (byte) 1 : (byte) 0)
.array();
}
@Override
public String apply(long value)
{
return formatter.print(granularity.truncate(value));
final long truncated = granularity.truncate(value);
return formatter == null ? String.valueOf(truncated) : formatter.print(truncated);
}
@Override
public String apply(Object value)
{
return apply(new DateTime(value).getMillis());
if (asMillis && value instanceof String) {
final Long theLong = GuavaUtils.tryParseLong((String) value);
return theLong == null ? apply(new DateTime(value).getMillis()) : apply(theLong.longValue());
} else {
return apply(new DateTime(value).getMillis());
}
}
@Override
@ -139,32 +166,36 @@ public class TimeFormatExtractionFn implements ExtractionFn
TimeFormatExtractionFn that = (TimeFormatExtractionFn) o;
if (tz != null ? !tz.equals(that.tz) : that.tz != null) {
if (asMillis != that.asMillis) {
return false;
}
if (format != null ? !format.equals(that.format) : that.format != null) {
return false;
}
if (tz != null ? !tz.equals(that.tz) : that.tz != null) {
return false;
}
if (locale != null ? !locale.equals(that.locale) : that.locale != null) {
return false;
}
return granularity.equals(that.granularity);
return granularity != null ? granularity.equals(that.granularity) : that.granularity == null;
}
@Override
public int hashCode()
{
int result = tz != null ? tz.hashCode() : 0;
result = 31 * result + (format != null ? format.hashCode() : 0);
int result = format != null ? format.hashCode() : 0;
result = 31 * result + (tz != null ? tz.hashCode() : 0);
result = 31 * result + (locale != null ? locale.hashCode() : 0);
result = 31 * result + granularity.hashCode();
result = 31 * result + (granularity != null ? granularity.hashCode() : 0);
result = 31 * result + (asMillis ? 1 : 0);
return result;
}
@Override
public String toString()
{
return String.format("timeFormat(\"%s\", %s, %s, %s)", format, tz, locale, granularity);
return String.format("timeFormat(\"%s\", %s, %s, %s, %s)", format, tz, locale, granularity, asMillis);
}
}

42
processing/src/test/java/io/druid/query/extraction/TimeFormatExtractionFnTest.java
View File

@ -28,6 +28,8 @@ import org.joda.time.DateTimeZone;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
public class TimeFormatExtractionFnTest
{
@ -43,7 +45,7 @@ public class TimeFormatExtractionFnTest
@Test
public void testDayOfWeekExtraction() throws Exception
{
TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, null, null);
TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, null, null, false);
Assert.assertEquals("Thursday", fn.apply(timestamps[0]));
Assert.assertEquals("Friday", fn.apply(timestamps[1]));
Assert.assertEquals("Tuesday", fn.apply(timestamps[2]));
@ -57,7 +59,7 @@ public class TimeFormatExtractionFnTest
@Test
public void testLocalizedExtraction() throws Exception
{
TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, "is", null);
TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, "is", null, false);
Assert.assertEquals("fimmtudagur", fn.apply(timestamps[0]));
Assert.assertEquals("föstudagur", fn.apply(timestamps[1]));
Assert.assertEquals("þriðjudagur", fn.apply(timestamps[2]));
@ -71,7 +73,7 @@ public class TimeFormatExtractionFnTest
@Test
public void testGranularExtractionWithNullPattern() throws Exception
{
TimeFormatExtractionFn fn = new TimeFormatExtractionFn(null, null, null, QueryGranularities.DAY);
TimeFormatExtractionFn fn = new TimeFormatExtractionFn(null, null, null, QueryGranularities.DAY, false);
Assert.assertEquals("2015-01-01T00:00:00.000Z", fn.apply(timestamps[0]));
Assert.assertEquals("2015-01-02T00:00:00.000Z", fn.apply(timestamps[1]));
Assert.assertEquals("2015-03-03T00:00:00.000Z", fn.apply(timestamps[2]));
@ -89,7 +91,8 @@ public class TimeFormatExtractionFnTest
"'In Berlin ist es schon 'EEEE",
DateTimeZone.forID("Europe/Berlin"),
"de",
null
null,
false
);
Assert.assertEquals("In Berlin ist es schon Freitag", fn.apply(timestamps[0]));
Assert.assertEquals("In Berlin ist es schon Samstag", fn.apply(timestamps[1]));
@ -140,4 +143,35 @@ public class TimeFormatExtractionFnTest
)
);
}
@Test
public void testCacheKey()
{
TimeFormatExtractionFn fn = new TimeFormatExtractionFn(
"'In Berlin ist es schon 'EEEE",
DateTimeZone.forID("Europe/Berlin"),
"de",
null,
false
);
TimeFormatExtractionFn fn2 = new TimeFormatExtractionFn(
"'In Berlin ist es schon 'EEEE",
DateTimeZone.forID("Europe/Berlin"),
"de",
null,
true
);
TimeFormatExtractionFn fn3 = new TimeFormatExtractionFn(
"'In Berlin ist es schon 'EEEE",
DateTimeZone.forID("Europe/Berlin"),
"de",
null,
true
);
Assert.assertFalse(Arrays.equals(fn.getCacheKey(), fn2.getCacheKey()));
Assert.assertTrue(Arrays.equals(fn2.getCacheKey(), fn3.getCacheKey()));
}
}

8
processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java
View File

@ -4120,7 +4120,7 @@ public class GroupByQueryRunnerTest
new ExtractionDimensionSpec(
Column.TIME_COLUMN_NAME,
Column.TIME_COLUMN_NAME,
new TimeFormatExtractionFn("EEEE", null, null, null),
new TimeFormatExtractionFn("EEEE", null, null, null, false),
null
)
)
@ -4994,8 +4994,8 @@ public class GroupByQueryRunnerTest
.setGranularity(QueryRunnerTestHelper.dayGran)
.build();
final DimFilter fridayFilter = new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null));
final DimFilter firstDaysFilter = new InDimFilter(Column.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"), new TimeFormatExtractionFn("d", null, null, null));
final DimFilter fridayFilter = new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null, false));
final DimFilter firstDaysFilter = new InDimFilter(Column.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"), new TimeFormatExtractionFn("d", null, null, null, false));
final GroupByQuery query = GroupByQuery
.builder()
.setDataSource(subquery)
@ -5452,7 +5452,7 @@ public class GroupByQueryRunnerTest
new ExtractionDimensionSpec(
Column.TIME_COLUMN_NAME,
"dayOfWeek",
new TimeFormatExtractionFn("EEEE", null, null, null),
new TimeFormatExtractionFn("EEEE", null, null, null, false),
null
)
)

2
processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java
View File

@ -3063,7 +3063,7 @@ public class TopNQueryRunnerTest
new ExtractionDimensionSpec(
Column.TIME_COLUMN_NAME,
"dayOfWeek",
new TimeFormatExtractionFn("EEEE", null, null, null),
new TimeFormatExtractionFn("EEEE", null, null, null, false),
null
)
)

2
processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java
View File

@ -225,7 +225,7 @@ public class TimeFilteringTest extends BaseFilterTest
@Test
public void testTimeFilterWithTimeFormatExtractionFn()
{
ExtractionFn exfn = new TimeFormatExtractionFn("EEEE", DateTimeZone.forID("America/New_York"), "en", null);
ExtractionFn exfn = new TimeFormatExtractionFn("EEEE", DateTimeZone.forID("America/New_York"), "en", null, false);
assertFilterMatches(
new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Wednesday", exfn),
ImmutableList.<String>of("0", "1", "2", "3", "4", "5")