Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Granularity: Introduce primitive-typed bucketStart, increment methods. (#10904) 

* Granularity: Introduce primitive-typed bucketStart, increment methods.

Saves creation of unnecessary DateTime objects in timestamp_floor and
timestamp_ceil expressions.

* Fix style.

* Amp up the test coverage.
This commit is contained in:
Gian Merlino 2021-02-25 07:59:20 -08:00 committed by GitHub
parent 67eff4110d
commit 07902f607b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 339 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
core/src/main/java/org/apache/druid/java/util/common/granularity/AllGranularity.java
View File

@ -44,12 +44,24 @@ public class AllGranularity extends Granularity
throw new UnsupportedOperationException("This method should not be invoked for this granularity type");
}
@Override
public long increment(long time)
{
return DateTimes.MAX.getMillis();
}
@Override
public DateTime increment(DateTime time)
{
return DateTimes.MAX;
}
@Override
public long bucketStart(long time)
{
return DateTimes.MIN.getMillis();
}
@Override
public DateTime bucketStart(DateTime time)
{

21
core/src/main/java/org/apache/druid/java/util/common/granularity/DurationGranularity.java
View File

@ -72,21 +72,32 @@ public class DurationGranularity extends Granularity
}
@Override
public DateTime increment(DateTime time)
public long increment(long time)
{
return time.plus(getDuration());
return time + duration;
}
@Override
public DateTime bucketStart(DateTime time)
public DateTime increment(DateTime time)
{
return time.plus(duration);
}
@Override
public long bucketStart(long t)
{
long t = time.getMillis();
final long duration = getDurationMillis();
long offset = t % duration - origin;
if (offset < 0) {
offset += duration;
}
return new DateTime(t - offset, time.getChronology());
return t - offset;
}
@Override
public DateTime bucketStart(DateTime time)
{
return new DateTime(bucketStart(time.getMillis()), time.getChronology());
}
@Override

4
core/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java
View File

@ -132,8 +132,12 @@ public abstract class Granularity implements Cacheable
public abstract DateTimeFormatter getFormatter(Formatter type);
public abstract long increment(long time);
public abstract DateTime increment(DateTime time);
public abstract long bucketStart(long time);
public abstract DateTime bucketStart(DateTime time);
public abstract DateTime toDate(String filePath, Formatter formatter);

12
core/src/main/java/org/apache/druid/java/util/common/granularity/NoneGranularity.java
View File

@ -42,12 +42,24 @@ public class NoneGranularity extends Granularity
throw new UnsupportedOperationException("This method should not be invoked for this granularity type");
}
@Override
public long increment(long time)
{
return time + 1;
}
@Override
public DateTime increment(DateTime time)
{
return time.plus(1);
}
@Override
public long bucketStart(long time)
{
return time;
}
@Override
public DateTime bucketStart(DateTime time)
{

17
core/src/main/java/org/apache/druid/java/util/common/granularity/PeriodGranularity.java
View File

@ -109,6 +109,18 @@ public class PeriodGranularity extends Granularity implements JsonSerializable
}
}
@Override
public long bucketStart(long time)
{
return truncate(time);
}
@Override
public long increment(long t)
{
return chronology.add(period, t, 1);
}
@Override
public DateTime increment(DateTime time)
{
@ -219,11 +231,6 @@ public class PeriodGranularity extends Granularity implements JsonSerializable
return false;
}
private long increment(long t)
{
return chronology.add(period, t, 1);
}
private long truncate(long t)
{
if (isCompound) {

51
core/src/test/java/org/apache/druid/java/util/common/GranularityTest.java
View File

@ -682,6 +682,7 @@ public class GranularityTest
{
DateTime dt = DateTimes.of("2011-02-03T04:05:06.100");
Assert.assertEquals(Intervals.ETERNITY, ALL.bucket(dt));
Assert.assertEquals(Intervals.of("2011-01-01/2012-01-01"), YEAR.bucket(dt));
Assert.assertEquals(Intervals.of("2011-02-01/2011-03-01"), MONTH.bucket(dt));
Assert.assertEquals(Intervals.of("2011-01-31/2011-02-07"), WEEK.bucket(dt));
@ -689,15 +690,17 @@ public class GranularityTest
Assert.assertEquals(Intervals.of("2011-02-03T04/2011-02-03T05"), HOUR.bucket(dt));
Assert.assertEquals(Intervals.of("2011-02-03T04:05:00/2011-02-03T04:06:00"), MINUTE.bucket(dt));
Assert.assertEquals(Intervals.of("2011-02-03T04:05:06/2011-02-03T04:05:07"), SECOND.bucket(dt));
Assert.assertEquals(Intervals.of("2011-02-03T04:05:06.100/2011-02-03T04:05:06.101"), NONE.bucket(dt));
// Test with aligned DateTime
Assert.assertEquals(Intervals.of("2011-01-01/2011-01-02"), DAY.bucket(DateTimes.of("2011-01-01")));
}
@Test
public void testTruncate()
public void testBucketStart()
{
DateTime date = DateTimes.of("2011-03-15T22:42:23.898");
Assert.assertEquals(DateTimes.MIN, ALL.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-01-01T00:00:00.000"), YEAR.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-03-01T00:00:00.000"), MONTH.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-03-14T00:00:00.000"), WEEK.bucketStart(date));
@ -705,6 +708,52 @@ public class GranularityTest
Assert.assertEquals(DateTimes.of("2011-03-15T22:00:00.000"), HOUR.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:00.000"), MINUTE.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.000"), SECOND.bucketStart(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.898"), NONE.bucketStart(date));
}
@Test
public void testBucketStartOnMillis()
{
DateTime date = DateTimes.of("2011-03-15T22:42:23.898");
Assert.assertEquals(DateTimes.MIN.getMillis(), ALL.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-01-01T00:00:00.000").getMillis(), YEAR.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-01T00:00:00.000").getMillis(), MONTH.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-14T00:00:00.000").getMillis(), WEEK.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T00:00:00.000").getMillis(), DAY.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:00:00.000").getMillis(), HOUR.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:00.000").getMillis(), MINUTE.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.000").getMillis(), SECOND.bucketStart(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.898").getMillis(), NONE.bucketStart(date.getMillis()));
}
@Test
public void testIncrement()
{
DateTime date = DateTimes.of("2011-03-15T22:42:23.898");
Assert.assertEquals(DateTimes.MIN, ALL.bucketStart(date));
Assert.assertEquals(DateTimes.of("2012-03-15T22:42:23.898"), YEAR.increment(date));
Assert.assertEquals(DateTimes.of("2011-04-15T22:42:23.898"), MONTH.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-22T22:42:23.898"), WEEK.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-16T22:42:23.898"), DAY.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-15T23:42:23.898"), HOUR.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:43:23.898"), MINUTE.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:24.898"), SECOND.increment(date));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.899"), NONE.increment(date));
}
@Test
public void testIncrementOnMillis()
{
DateTime date = DateTimes.of("2011-03-15T22:42:23.898");
Assert.assertEquals(DateTimes.MIN, ALL.bucketStart(date));
Assert.assertEquals(DateTimes.of("2012-03-15T22:42:23.898").getMillis(), YEAR.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-04-15T22:42:23.898").getMillis(), MONTH.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-22T22:42:23.898").getMillis(), WEEK.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-16T22:42:23.898").getMillis(), DAY.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T23:42:23.898").getMillis(), HOUR.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:43:23.898").getMillis(), MINUTE.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:24.898").getMillis(), SECOND.increment(date.getMillis()));
Assert.assertEquals(DateTimes.of("2011-03-15T22:42:23.899").getMillis(), NONE.increment(date.getMillis()));
}
@Test

2
indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java
View File

@ -754,7 +754,7 @@ public class IndexTask extends AbstractBatchIndexTask implements ChatHandler
hllCollectors.computeIfAbsent(interval, intv -> Optional.of(HyperLogLogCollector.makeLatestCollector()));
List<Object> groupKey = Rows.toGroupKey(
queryGranularity.bucketStart(inputRow.getTimestamp()).getMillis(),
queryGranularity.bucketStart(inputRow.getTimestampFromEpoch()),
inputRow
);
hllCollectors.get(interval).get()

4
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionCardinalityTask.java
View File

@ -198,9 +198,9 @@ public class PartialDimensionCardinalityTask extends PerfectRollupWorkerTask
DateTime timestamp = inputRow.getTimestamp();
final Interval interval;
if (granularitySpec.inputIntervals().isEmpty()) {
interval = granularitySpec.getSegmentGranularity().bucket(inputRow.getTimestamp());
interval = granularitySpec.getSegmentGranularity().bucket(timestamp);
} else {
final Optional<Interval> optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp());
final Optional<Interval> optInterval = granularitySpec.bucketInterval(timestamp);
// this interval must exist since it passed the rowFilter
assert optInterval.isPresent();
interval = optInterval.get();

5
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionTask.java
View File

@ -50,7 +50,6 @@ import org.apache.druid.segment.incremental.ParseExceptionHandler;
import org.apache.druid.segment.incremental.RowIngestionMeters;
import org.apache.druid.segment.indexing.DataSchema;
import org.apache.druid.segment.indexing.granularity.GranularitySpec;
import org.joda.time.DateTime;
import org.joda.time.Interval;
import javax.annotation.Nullable;
@ -369,8 +368,8 @@ public class PartialDimensionDistributionTask extends PerfectRollupWorkerTask
private long getBucketTimestamp(InputRow inputRow)
{
DateTime timestamp = inputRow.getTimestamp();
return queryGranularity.bucketStart(timestamp).getMillis();
final long timestamp = inputRow.getTimestampFromEpoch();
return queryGranularity.bucketStart(timestamp);
}
@Override

4
processing/src/main/java/org/apache/druid/query/ResultGranularTimestampComparator.java
View File

@ -42,8 +42,8 @@ public class ResultGranularTimestampComparator<T> implements Comparator<Result<T
public int compare(Result<T> r1, Result<T> r2)
{
return Longs.compare(
gran.bucketStart(r1.getTimestamp()).getMillis(),
gran.bucketStart(r2.getTimestamp()).getMillis()
gran.bucketStart(r1.getTimestamp().getMillis()),
gran.bucketStart(r2.getTimestamp().getMillis())
);
}

22
processing/src/main/java/org/apache/druid/query/expression/TimestampCeilExprMacro.java
View File

@ -20,7 +20,6 @@
package org.apache.druid.query.expression;
import com.google.common.annotations.VisibleForTesting;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.granularity.PeriodGranularity;
@ -28,7 +27,6 @@ import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.math.expr.ExprType;
import org.joda.time.DateTime;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -80,12 +78,12 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
// Return null if the argument if null.
return ExprEval.of(null);
}
DateTime argTime = DateTimes.utc(eval.asLong());
DateTime bucketStartTime = granularity.bucketStart(argTime);
if (argTime.equals(bucketStartTime)) {
return ExprEval.of(bucketStartTime.getMillis());
long argTime = eval.asLong();
long bucketStartTime = granularity.bucketStart(argTime);
if (argTime == bucketStartTime) {
return ExprEval.of(bucketStartTime);
}
return ExprEval.of(granularity.increment(bucketStartTime).getMillis());
return ExprEval.of(granularity.increment(bucketStartTime));
}
@Override
@ -148,12 +146,12 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
public ExprEval eval(final ObjectBinding bindings)
{
final PeriodGranularity granularity = getGranularity(args, bindings);
DateTime argTime = DateTimes.utc(args.get(0).eval(bindings).asLong());
DateTime bucketStartTime = granularity.bucketStart(argTime);
if (argTime.equals(bucketStartTime)) {
return ExprEval.of(bucketStartTime.getMillis());
long argTime = args.get(0).eval(bindings).asLong();
long bucketStartTime = granularity.bucketStart(argTime);
if (argTime == bucketStartTime) {
return ExprEval.of(bucketStartTime);
}
return ExprEval.of(granularity.increment(bucketStartTime).getMillis());
return ExprEval.of(granularity.increment(bucketStartTime));
}
@Override

7
processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.query.expression;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.math.expr.Expr;
@ -105,7 +104,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
// Return null if the argument if null.
return ExprEval.of(null);
}
return ExprEval.of(granularity.bucketStart(DateTimes.utc(eval.asLong())).getMillis());
return ExprEval.of(granularity.bucketStart(eval.asLong()));
}
@Override
@ -141,7 +140,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
@Override
public long apply(long input)
{
return granularity.bucketStart(DateTimes.utc(input)).getMillis();
return granularity.bucketStart(input);
}
};
@ -183,7 +182,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
public ExprEval eval(final ObjectBinding bindings)
{
final PeriodGranularity granularity = computeGranularity(args, bindings);
return ExprEval.of(granularity.bucketStart(DateTimes.utc(args.get(0).eval(bindings).asLong())).getMillis());
return ExprEval.of(granularity.bucketStart(args.get(0).eval(bindings).asLong()));
}
@Override

2
processing/src/main/java/org/apache/druid/query/extraction/TimeFormatExtractionFn.java
View File

@ -126,7 +126,7 @@ public class TimeFormatExtractionFn implements ExtractionFn
@Override
public String apply(long value)
{
final long truncated = granularity.bucketStart(DateTimes.utc(value)).getMillis();
final long truncated = granularity.bucketStart(value);
return formatter == null ? String.valueOf(truncated) : formatter.print(truncated);
}

4
processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
View File

@ -675,8 +675,8 @@ public class GroupByQuery extends BaseQuery<ResultRow>
if (granular) {
return (lhs, rhs) -> Longs.compare(
getGranularity().bucketStart(DateTimes.utc(lhs.getLong(0))).getMillis(),
getGranularity().bucketStart(DateTimes.utc(rhs.getLong(0))).getMillis()
getGranularity().bucketStart(lhs.getLong(0)),
getGranularity().bucketStart(rhs.getLong(0))
);
} else {
return NON_GRANULAR_TIME_COMP;

7
processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryHelper.java
View File

@ -45,7 +45,6 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.incremental.IndexSizeExceededException;
import org.apache.druid.segment.incremental.OffheapIncrementalIndex;
import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
import org.joda.time.DateTime;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
@ -67,10 +66,10 @@ public class GroupByQueryHelper
{
final GroupByQueryConfig querySpecificConfig = config.withOverrides(query);
final Granularity gran = query.getGranularity();
final DateTime timeStart = query.getIntervals().get(0).getStart();
final long timeStart = query.getIntervals().get(0).getStartMillis();
final boolean combine = subquery == null;
DateTime granTimeStart = timeStart;
long granTimeStart = timeStart;
if (!(Granularities.ALL.equals(gran))) {
granTimeStart = gran.bucketStart(timeStart);
}
@ -117,7 +116,7 @@ public class GroupByQueryHelper
.withDimensionsSpec(new DimensionsSpec(dimensionSchemas, null, null))
.withMetrics(aggs.toArray(new AggregatorFactory[0]))
.withQueryGranularity(gran)
.withMinTimestamp(granTimeStart.getMillis())
.withMinTimestamp(granTimeStart)
.build();

3
processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByBinaryFnV2.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.query.groupby.epinephelinae;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.AllGranularity;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.dimension.DimensionSpec;
@ -88,7 +87,7 @@ public class GroupByBinaryFnV2 implements BinaryOperator<ResultRow>
if (query.getGranularity() instanceof AllGranularity) {
return row.getLong(0);
} else {
return query.getGranularity().bucketStart(DateTimes.utc(row.getLong(0))).getMillis();
return query.getGranularity().bucketStart(row.getLong(0));
}
}
}

3
processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
View File

@ -34,7 +34,6 @@ import org.apache.druid.collections.ReferenceCountingResourceHolder;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.common.guava.SettableSupplier;
import org.apache.druid.common.utils.IntArrayUtils;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Pair;
@ -448,7 +447,7 @@ public class RowBasedGrouperHelper
if (query.getGranularity() instanceof AllGranularity) {
return row -> query.getIntervals().get(0).getStartMillis();
} else {
return row -> query.getGranularity().bucketStart(DateTimes.utc(row.getLong(0))).getMillis();
return row -> query.getGranularity().bucketStart(row.getLong(0));
}
}
} else {

2
processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java
View File

@ -127,7 +127,7 @@ public class QueryableIndexCursorSequenceBuilder
final long timeStart = Math.max(interval.getStartMillis(), inputInterval.getStartMillis());
final long timeEnd = Math.min(
interval.getEndMillis(),
gran.increment(inputInterval.getStart()).getMillis()
gran.increment(inputInterval.getStartMillis())
);
if (descending) {

2
processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java
View File

@ -590,7 +590,7 @@ public abstract class IncrementalIndex<AggregatorType> extends AbstractIndex imp
long truncated = 0;
if (row.getTimestamp() != null) {
truncated = gran.bucketStart(row.getTimestamp()).getMillis();
truncated = gran.bucketStart(row.getTimestampFromEpoch());
}
IncrementalIndexRow incrementalIndexRow = IncrementalIndexRow.createTimeAndDimswithDimsKeySize(
Math.max(truncated, minTimestamp),

2
processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
View File

@ -333,7 +333,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
cursorIterable = index.getFacts().timeRangeIterable(
descending,
timeStart,
Math.min(actualInterval.getEndMillis(), gran.increment(interval.getStart()).getMillis())
Math.min(actualInterval.getEndMillis(), gran.increment(interval.getStartMillis()))
);
emptyRange = !cursorIterable.iterator().hasNext();
time = gran.toDateTime(interval.getStartMillis());

150
processing/src/test/java/org/apache/druid/granularity/QueryGranularityTest.java
View File

@ -582,7 +582,7 @@ public class QueryGranularityTest
}
@Test
public void testDurationTruncate()
public void testDurationBucketStart()
{
{
final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00");
@ -590,6 +590,7 @@ public class QueryGranularityTest
new Period("PT12H5M").toStandardDuration().getMillis(),
origin
);
assertSameDateTime(
Lists.newArrayList(
DateTimes.of("2012-01-01T04:50:00.000-08:00"),
@ -604,6 +605,82 @@ public class QueryGranularityTest
gran.bucketStart(DateTimes.of("2012-02-03T22:25:00.000-08:00"))
)
);
// Same as above, but using the millis form of the method.
Assert.assertEquals(
DateTimes.of("2012-01-01T04:50:00.000-08:00").getMillis(),
gran.bucketStart(DateTimes.of("2012-01-01T05:00:04.123-08:00").getMillis())
);
Assert.assertEquals(
DateTimes.of("2012-01-02T05:00:00.000-08:00").getMillis(),
gran.bucketStart(DateTimes.of("2012-01-02T07:00:04.123-08:00").getMillis())
);
Assert.assertEquals(
DateTimes.of("2012-01-02T17:05:00.000-08:00").getMillis(),
gran.bucketStart(DateTimes.of("2012-01-03T00:20:04.123-08:00").getMillis())
);
Assert.assertEquals(
DateTimes.of("2012-02-03T22:25:00.000-08:00").getMillis(),
gran.bucketStart(DateTimes.of("2012-02-03T22:25:00.000-08:00").getMillis())
);
}
}
@Test
public void testDurationIncrement()
{
{
final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00");
Granularity gran = new DurationGranularity(
new Period("PT12H5M").toStandardDuration().getMillis(),
origin
);
Assert.assertEquals(
DateTimes.of("2012-01-01T17:05:04.123-08:00"),
gran.increment(DateTimes.of("2012-01-01T05:00:04.123-08:00"))
);
Assert.assertEquals(
DateTimes.of("2012-01-02T19:05:04.123-08:00"),
gran.increment(DateTimes.of("2012-01-02T07:00:04.123-08:00"))
);
Assert.assertEquals(
DateTimes.of("2012-01-03T12:25:04.123-08:00"),
gran.increment(DateTimes.of("2012-01-03T00:20:04.123-08:00"))
);
Assert.assertEquals(
DateTimes.of("2012-02-04T10:30:00.000-08:00"),
gran.increment(DateTimes.of("2012-02-03T22:25:00.000-08:00"))
);
}
}
@Test
public void testDurationIncrementOnMillis()
{
{
final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00");
Granularity gran = new DurationGranularity(
new Period("PT12H5M").toStandardDuration().getMillis(),
origin
);
Assert.assertEquals(
DateTimes.of("2012-01-01T17:05:04.123-08:00").getMillis(),
gran.increment(DateTimes.of("2012-01-01T05:00:04.123-08:00").getMillis())
);
Assert.assertEquals(
DateTimes.of("2012-01-02T19:05:04.123-08:00").getMillis(),
gran.increment(DateTimes.of("2012-01-02T07:00:04.123-08:00")).getMillis()
);
Assert.assertEquals(
DateTimes.of("2012-01-03T12:25:04.123-08:00").getMillis(),
gran.increment(DateTimes.of("2012-01-03T00:20:04.123-08:00")).getMillis()
);
Assert.assertEquals(
DateTimes.of("2012-02-04T10:30:00.000-08:00").getMillis(),
gran.increment(DateTimes.of("2012-02-03T22:25:00.000-08:00")).getMillis()
);
}
}
@ -810,21 +887,25 @@ public class QueryGranularityTest
final PeriodGranularity hour = new PeriodGranularity(new Period("PT1H"), null, tz);
final PeriodGranularity twoHour = new PeriodGranularity(new Period("PT2H"), null, tz);
Assert.assertEquals(
new DateTime("2011-01-01T00:00:00.000+05:45", tz),
year.toDateTime(year.bucketStart(date).getMillis())
);
Assert.assertEquals(
new DateTime("2011-03-15T21:00:00.000+05:45", tz),
hour.toDateTime(hour.bucketStart(date).getMillis())
assertBucketStart(
year,
date,
new DateTime("2011-01-01T00:00:00.000+05:45", tz)
);
Assert.assertEquals(
new DateTime("2011-03-15T20:00:00.000+05:45", tz),
twoHour.toDateTime(twoHour.bucketStart(date).getMillis())
assertBucketStart(
hour,
date,
new DateTime("2011-03-15T21:00:00.000+05:45", tz)
);
assertBucketStart(
twoHour,
date,
new DateTime("2011-03-15T20:00:00.000+05:45", tz)
);
}
@Test
public void testTruncateDhaka()
{
@ -834,20 +915,43 @@ public class QueryGranularityTest
final PeriodGranularity hour = new PeriodGranularity(new Period("PT1H"), null, tz);
final PeriodGranularity twoHour = new PeriodGranularity(new Period("PT2H"), null, tz);
Assert.assertEquals(
new DateTime("2011-01-01T00:00:00.000+06:00", tz),
year.toDateTime(year.bucketStart(date).getMillis())
);
Assert.assertEquals(
new DateTime("2011-03-15T21:00:00.000+06:00", tz),
hour.toDateTime(hour.bucketStart(date).getMillis())
assertBucketStart(
year,
date,
new DateTime("2011-01-01T00:00:00.000+06:00", tz)
);
Assert.assertEquals(
new DateTime("2011-03-15T20:00:00.000+06:00", tz),
twoHour.toDateTime(twoHour.bucketStart(date).getMillis())
assertBucketStart(
hour,
date,
new DateTime("2011-03-15T21:00:00.000+06:00", tz)
);
assertBucketStart(
twoHour,
date,
new DateTime("2011-03-15T20:00:00.000+06:00", tz)
);
}
private void assertBucketStart(final Granularity granularity, final DateTime in, final DateTime expectedInProperTz)
{
Assert.assertEquals(
StringUtils.format("Granularity [%s] toDateTime(bucketStart(DateTime))", granularity),
expectedInProperTz,
granularity.toDateTime(granularity.bucketStart(in).getMillis())
);
Assert.assertEquals(
StringUtils.format("Granularity [%s] bucketStart(DateTime)", granularity),
expectedInProperTz.withZone(in.getZone()),
granularity.bucketStart(in)
);
Assert.assertEquals(
StringUtils.format("Granularity [%s] bucketStart(long)", granularity),
expectedInProperTz.getMillis(),
granularity.bucketStart(in.getMillis())
);
}
}

24
server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java → processing/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java
View File

@ -73,20 +73,6 @@ public class ExprMacroTest
assertExpr("like(x, '')", 0L);
}
@Test
public void testLookup()
{
assertExpr("lookup(x, 'lookyloo')", "xfoo");
}
@Test
public void testLookupNotFound()
{
expectedException.expect(IllegalStateException.class);
expectedException.expectMessage("Lookup [lookylook] not found");
assertExpr("lookup(x, 'lookylook')", null);
}
@Test
public void testRegexpExtract()
{
@ -98,6 +84,7 @@ public class ExprMacroTest
@Test
public void testTimestampCeil()
{
assertExpr("timestamp_ceil(null, 'P1M')", null);
assertExpr("timestamp_ceil(t, 'P1M')", DateTimes.of("2000-03-01").getMillis());
assertExpr("timestamp_ceil(t, 'P1D',null,'America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis());
assertExpr("timestamp_ceil(t, 'P1D',null,CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis());
@ -108,6 +95,7 @@ public class ExprMacroTest
@Test
public void testTimestampFloor()
{
assertExpr("timestamp_floor(null, 'P1M')", null);
assertExpr("timestamp_floor(t, 'P1M')", DateTimes.of("2000-02-01").getMillis());
assertExpr("timestamp_floor(t, 'P1D',null,'America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis());
assertExpr("timestamp_floor(t, 'P1D',null,CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis());
@ -231,15 +219,15 @@ public class ExprMacroTest
private void assertExpr(final String expression, final Object expectedResult)
{
final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE);
final Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);
Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value());
final Expr exprNotFlattened = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE, false);
final Expr exprNotFlattened = Parser.parse(expression, TestExprMacroTable.INSTANCE, false);
final Expr roundTripNotFlattened =
Parser.parse(exprNotFlattened.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Parser.parse(exprNotFlattened.stringify(), TestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTripNotFlattened.eval(BINDINGS).value());
final Expr roundTrip = Parser.parse(expr.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
final Expr roundTrip = Parser.parse(expr.stringify(), TestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTrip.eval(BINDINGS).value());
}
}

69
server/src/test/java/org/apache/druid/query/expression/LookupExprMacroTest.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.expression;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class LookupExprMacroTest extends InitializedNullHandlingTest
{
private static final Expr.ObjectBinding BINDINGS = Parser.withMap(
ImmutableMap.<String, Object>builder()
.put("x", "foo")
.build()
);
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testLookup()
{
assertExpr("lookup(x, 'lookyloo')", "xfoo");
}
@Test
public void testLookupNotFound()
{
expectedException.expect(IllegalStateException.class);
expectedException.expectMessage("Lookup [lookylook] not found");
assertExpr("lookup(x, 'lookylook')", null);
}
private void assertExpr(final String expression, final Object expectedResult)
{
final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value());
final Expr exprNotFlattened = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE, false);
final Expr roundTripNotFlattened =
Parser.parse(exprNotFlattened.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTripNotFlattened.eval(BINDINGS).value());
final Expr roundTrip = Parser.parse(expr.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTrip.eval(BINDINGS).value());
}
}