Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-17 07:25:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add finalizeAsBase64Binary option to FixedBucketsHistogramAggregatorFactory (#7784)

Browse Source 
* Add finalizeAsBase64Binary option to FixedBucketsHistogramAggregatorFactory

* Add finalizeAsBase64Binary option to ApproximateHistogramFactory

* Update approx histogram doc
This commit is contained in:
Jonathan Wei 2019-06-21 18:00:19 -07:00 committed by Clint Wylie
parent 6c8f9482c7
commit 35601bb7a0
14 changed files with 326 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/content/development/extensions-core/approximate-histograms.md
View File

@ -99,6 +99,7 @@ query.
|`resolution` |Number of centroids (data points) to store. The higher the resolution, the more accurate results are, but the slower the computation will be.|50|
|`numBuckets` |Number of output buckets for the resulting histogram. Bucket intervals are dynamic, based on the range of the underlying data. Use a post-aggregator to have finer control over the bucketing scheme|7|
|`lowerLimit`/`upperLimit`|Restrict the approximation to the given range. The values outside this range will be aggregated into two centroids. Counts of values outside this range are still maintained. |-INF/+INF|
|`finalizeAsBase64Binary` |If true, the finalized aggregator value will be a Base64-encoded byte array containing the serialized form of the histogram. If false, the finalized aggregator value will be a JSON representation of the histogram.|false|
## Fixed Buckets Histogram
@ -124,6 +125,7 @@ For general histogram and quantile use cases, the [DataSketches Quantiles Sketch
|`upperLimit`|Upper limit of the histogram. |No default, must be specified|
|`numBuckets`|Number of buckets for the histogram. The range [lowerLimit, upperLimit] will be divided into `numBuckets` intervals of equal size.|10|
|`outlierHandlingMode`|Specifies how values outside of [lowerLimit, upperLimit] will be handled. Supported modes are "ignore", "overflow", and "clip". See [outlier handling modes](#outlier-handling-modes) for more details.|No default, must be specified|
|`finalizeAsBase64Binary`|If true, the finalized aggregator value will be a Base64-encoded byte array containing the [serialized form](#serialization-formats) of the histogram. If false, the finalized aggregator value will be a JSON representation of the histogram.|false|
An example aggregator spec is shown below:

119
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java
View File

@ -32,6 +32,7 @@ import org.apache.druid.query.aggregation.AggregatorFactoryNotMergeableException
import org.apache.druid.query.aggregation.AggregatorUtil;
import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.aggregation.ObjectAggregateCombiner;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
@ -40,6 +41,7 @@ import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
@JsonTypeName("approxHistogram")
public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@ -53,6 +55,8 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
protected final float lowerLimit;
protected final float upperLimit;
protected final boolean finalizeAsBase64Binary;
@JsonCreator
public ApproximateHistogramAggregatorFactory(
@JsonProperty("name") String name,
@ -60,7 +64,8 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@JsonProperty("resolution") Integer resolution,
@JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("lowerLimit") Float lowerLimit,
@JsonProperty("upperLimit") Float upperLimit
@JsonProperty("upperLimit") Float upperLimit,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
)
{
@ -70,6 +75,7 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
this.numBuckets = numBuckets == null ? ApproximateHistogram.DEFAULT_BUCKET_SIZE : numBuckets;
this.lowerLimit = lowerLimit == null ? Float.NEGATIVE_INFINITY : lowerLimit;
this.upperLimit = upperLimit == null ? Float.POSITIVE_INFINITY : upperLimit;
this.finalizeAsBase64Binary = finalizeAsBase64Binary == null ? false : finalizeAsBase64Binary;
Preconditions.checkArgument(this.resolution > 0, "resolution must be greater than 1");
Preconditions.checkArgument(this.numBuckets > 0, "numBuckets must be greater than 1");
@ -149,7 +155,15 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@Override
public AggregatorFactory getCombiningFactory()
{
return new ApproximateHistogramFoldingAggregatorFactory(name, name, resolution, numBuckets, lowerLimit, upperLimit);
return new ApproximateHistogramFoldingAggregatorFactory(
name,
name,
resolution,
numBuckets,
lowerLimit,
upperLimit,
finalizeAsBase64Binary
);
}
@Override
@ -164,7 +178,8 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
Math.max(resolution, castedOther.resolution),
numBuckets,
Math.min(lowerLimit, castedOther.lowerLimit),
Math.max(upperLimit, castedOther.upperLimit)
Math.max(upperLimit, castedOther.upperLimit),
finalizeAsBase64Binary
);
} else {
@ -182,7 +197,8 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
resolution,
numBuckets,
lowerLimit,
upperLimit
upperLimit,
finalizeAsBase64Binary
)
);
}
@ -218,7 +234,11 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@Override
public Object finalizeComputation(@Nullable Object object)
{
return object == null ? null : ((ApproximateHistogram) object).toHistogram(numBuckets);
if (finalizeAsBase64Binary) {
return object;
} else {
return object == null ? null : ((ApproximateHistogram) object).toHistogram(numBuckets);
}
}
@JsonProperty
@ -267,14 +287,15 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = StringUtils.toUtf8(fieldName);
return ByteBuffer.allocate(1 + fieldNameBytes.length + Integer.BYTES * 2 + Float.BYTES * 2)
.put(AggregatorUtil.APPROX_HIST_CACHE_TYPE_ID)
.put(fieldNameBytes)
.putInt(resolution)
.putInt(numBuckets)
.putFloat(lowerLimit)
.putFloat(upperLimit).array();
CacheKeyBuilder builder = new CacheKeyBuilder(AggregatorUtil.APPROX_HIST_CACHE_TYPE_ID)
.appendString(fieldName)
.appendInt(resolution)
.appendInt(numBuckets)
.appendFloat(lowerLimit)
.appendFloat(upperLimit)
.appendBoolean(finalizeAsBase64Binary);
return builder.build();
}
@Override
@ -289,52 +310,6 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
return new ApproximateHistogram(resolution).getMaxStorageSize();
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ApproximateHistogramAggregatorFactory that = (ApproximateHistogramAggregatorFactory) o;
if (Float.compare(that.lowerLimit, lowerLimit) != 0) {
return false;
}
if (numBuckets != that.numBuckets) {
return false;
}
if (resolution != that.resolution) {
return false;
}
if (Float.compare(that.upperLimit, upperLimit) != 0) {
return false;
}
if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) {
return false;
}
if (name != null ? !name.equals(that.name) : that.name != null) {
return false;
}
return true;
}
@Override
public int hashCode()
{
int result = name != null ? name.hashCode() : 0;
result = 31 * result + (fieldName != null ? fieldName.hashCode() : 0);
result = 31 * result + resolution;
result = 31 * result + numBuckets;
result = 31 * result + (lowerLimit != +0.0f ? Float.floatToIntBits(lowerLimit) : 0);
result = 31 * result + (upperLimit != +0.0f ? Float.floatToIntBits(upperLimit) : 0);
return result;
}
@Override
public String toString()
{
@ -345,6 +320,32 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
", numBuckets=" + numBuckets +
", lowerLimit=" + lowerLimit +
", upperLimit=" + upperLimit +
", finalizeAsBase64Binary=" + finalizeAsBase64Binary +
'}';
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ApproximateHistogramAggregatorFactory that = (ApproximateHistogramAggregatorFactory) o;
return resolution == that.resolution &&
numBuckets == that.numBuckets &&
Float.compare(that.lowerLimit, lowerLimit) == 0 &&
Float.compare(that.upperLimit, upperLimit) == 0 &&
finalizeAsBase64Binary == that.finalizeAsBase64Binary &&
Objects.equals(name, that.name) &&
Objects.equals(fieldName, that.fieldName);
}
@Override
public int hashCode()
{
return Objects.hash(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary);
}
}

100
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java
View File

@ -23,15 +23,16 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.aggregation.Aggregator;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.AggregatorUtil;
import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import java.nio.ByteBuffer;
import javax.annotation.Nullable;
import java.util.Objects;
@JsonTypeName("approxHistogramFold")
public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHistogramAggregatorFactory
@ -44,10 +45,11 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis
@JsonProperty("resolution") Integer resolution,
@JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("lowerLimit") Float lowerLimit,
@JsonProperty("upperLimit") Float upperLimit
@JsonProperty("upperLimit") Float upperLimit,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
)
{
super(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit);
super(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary);
}
@Override
@ -94,67 +96,21 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis
@Override
public AggregatorFactory getCombiningFactory()
{
return new ApproximateHistogramFoldingAggregatorFactory(name, name, resolution, numBuckets, lowerLimit, upperLimit);
return new ApproximateHistogramFoldingAggregatorFactory(name, name, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary);
}
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = StringUtils.toUtf8(fieldName);
return ByteBuffer.allocate(1 + fieldNameBytes.length + Integer.BYTES * 2 + Float.BYTES * 2)
.put(AggregatorUtil.APPROX_HIST_FOLDING_CACHE_TYPE_ID)
.put(fieldNameBytes)
.putInt(resolution)
.putInt(numBuckets)
.putFloat(lowerLimit)
.putFloat(upperLimit)
.array();
}
CacheKeyBuilder builder = new CacheKeyBuilder(AggregatorUtil.APPROX_HIST_FOLDING_CACHE_TYPE_ID)
.appendString(fieldName)
.appendInt(resolution)
.appendInt(numBuckets)
.appendFloat(lowerLimit)
.appendFloat(upperLimit)
.appendBoolean(finalizeAsBase64Binary);
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ApproximateHistogramAggregatorFactory that = (ApproximateHistogramAggregatorFactory) o;
if (Float.compare(that.lowerLimit, lowerLimit) != 0) {
return false;
}
if (numBuckets != that.numBuckets) {
return false;
}
if (resolution != that.resolution) {
return false;
}
if (Float.compare(that.upperLimit, upperLimit) != 0) {
return false;
}
if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) {
return false;
}
if (name != null ? !name.equals(that.name) : that.name != null) {
return false;
}
return true;
}
@Override
public int hashCode()
{
int result = name != null ? name.hashCode() : 0;
result = 31 * result + (fieldName != null ? fieldName.hashCode() : 0);
result = 31 * result + resolution;
result = 31 * result + numBuckets;
result = 31 * result + (lowerLimit != +0.0f ? Float.floatToIntBits(lowerLimit) : 0);
result = 31 * result + (upperLimit != +0.0f ? Float.floatToIntBits(upperLimit) : 0);
return result;
return builder.build();
}
@Override
@ -167,7 +123,33 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis
", numBuckets=" + numBuckets +
", lowerLimit=" + lowerLimit +
", upperLimit=" + upperLimit +
", finalizeAsBase64Binary=" + finalizeAsBase64Binary +
'}';
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ApproximateHistogramAggregatorFactory that = (ApproximateHistogramAggregatorFactory) o;
return resolution == that.resolution &&
numBuckets == that.numBuckets &&
Float.compare(that.lowerLimit, lowerLimit) == 0 &&
Float.compare(that.upperLimit, upperLimit) == 0 &&
finalizeAsBase64Binary == that.finalizeAsBase64Binary &&
Objects.equals(name, that.name) &&
Objects.equals(fieldName, that.fieldName);
}
@Override
public int hashCode()
{
return Objects.hash(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary);
}
}

57
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java
View File

@ -29,11 +29,11 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.AggregatorUtil;
import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.aggregation.ObjectAggregateCombiner;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@ -53,6 +53,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
private FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode;
private boolean finalizeAsBase64Binary;
@JsonCreator
public FixedBucketsHistogramAggregatorFactory(
@JsonProperty("name") String name,
@ -60,7 +62,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
@Nullable @JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("lowerLimit") double lowerLimit,
@JsonProperty("upperLimit") double upperLimit,
@JsonProperty("outlierHandlingMode") FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode
@JsonProperty("outlierHandlingMode") FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
)
{
this.name = name;
@ -69,6 +72,7 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
this.lowerLimit = lowerLimit;
this.upperLimit = upperLimit;
this.outlierHandlingMode = outlierHandlingMode;
this.finalizeAsBase64Binary = finalizeAsBase64Binary == null ? false : finalizeAsBase64Binary;
}
@Override
@ -166,7 +170,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
numBuckets,
lowerLimit,
upperLimit,
outlierHandlingMode
outlierHandlingMode,
finalizeAsBase64Binary
);
}
@ -179,7 +184,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
numBuckets,
lowerLimit,
upperLimit,
outlierHandlingMode
outlierHandlingMode,
finalizeAsBase64Binary
);
}
@ -193,7 +199,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
numBuckets,
lowerLimit,
upperLimit,
outlierHandlingMode
outlierHandlingMode,
finalizeAsBase64Binary
)
);
}
@ -214,7 +221,15 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
@Override
public Object finalizeComputation(@Nullable Object object)
{
return object;
if (object == null) {
return null;
}
if (finalizeAsBase64Binary) {
return object;
} else {
return object.toString();
}
}
@JsonProperty
@ -245,14 +260,15 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = StringUtils.toUtf8(fieldName);
return ByteBuffer.allocate(1 + fieldNameBytes.length + Integer.BYTES * 2 + Double.BYTES * 2)
.put(AggregatorUtil.FIXED_BUCKET_HIST_CACHE_TYPE_ID)
.put(fieldNameBytes)
.putInt(outlierHandlingMode.ordinal())
.putInt(numBuckets)
.putDouble(lowerLimit)
.putDouble(upperLimit).array();
final CacheKeyBuilder builder = new CacheKeyBuilder(AggregatorUtil.FIXED_BUCKET_HIST_CACHE_TYPE_ID)
.appendString(fieldName)
.appendInt(outlierHandlingMode.ordinal())
.appendInt(numBuckets)
.appendDouble(lowerLimit)
.appendDouble(upperLimit)
.appendBoolean(finalizeAsBase64Binary);
return builder.build();
}
@JsonProperty
@ -285,6 +301,12 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
return outlierHandlingMode;
}
@JsonProperty
public boolean isFinalizeAsBase64Binary()
{
return finalizeAsBase64Binary;
}
@Override
public boolean equals(Object o)
{
@ -300,7 +322,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
getNumBuckets() == that.getNumBuckets() &&
Objects.equals(getName(), that.getName()) &&
Objects.equals(getFieldName(), that.getFieldName()) &&
getOutlierHandlingMode() == that.getOutlierHandlingMode();
getOutlierHandlingMode() == that.getOutlierHandlingMode() &&
isFinalizeAsBase64Binary() == that.isFinalizeAsBase64Binary();
}
@Override
@ -312,7 +335,8 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
getLowerLimit(),
getUpperLimit(),
getNumBuckets(),
getOutlierHandlingMode()
getOutlierHandlingMode(),
isFinalizeAsBase64Binary()
);
}
@ -326,6 +350,7 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
", upperLimit=" + upperLimit +
", numBuckets=" + numBuckets +
", outlierHandlingMode=" + outlierHandlingMode +
", finalizeAsBase64Binary=" + finalizeAsBase64Binary +
'}';
}
}

6
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregator.java
View File

@ -231,7 +231,8 @@ public class FixedBucketsHistogramQuantileSqlAggregator implements SqlAggregator
numBuckets,
lowerLimit,
upperLimit,
outlierHandlingMode
outlierHandlingMode,
false
);
} else {
VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
@ -246,7 +247,8 @@ public class FixedBucketsHistogramQuantileSqlAggregator implements SqlAggregator
numBuckets,
lowerLimit,
upperLimit,
outlierHandlingMode
outlierHandlingMode,
false
);
}

9
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java
View File

@ -182,7 +182,8 @@ public class QuantileSqlAggregator implements SqlAggregator
resolution,
numBuckets,
lowerLimit,
upperLimit
upperLimit,
false
);
} else {
aggregatorFactory = new ApproximateHistogramAggregatorFactory(
@ -191,7 +192,8 @@ public class QuantileSqlAggregator implements SqlAggregator
resolution,
numBuckets,
lowerLimit,
upperLimit
upperLimit,
false
);
}
} else {
@ -204,7 +206,8 @@ public class QuantileSqlAggregator implements SqlAggregator
resolution,
numBuckets,
lowerLimit,
upperLimit
upperLimit,
false
);
}

40
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.query.aggregation.histogram;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.aggregation.TestFloatColumnSelector;
import org.junit.Assert;
@ -44,7 +45,7 @@ public class ApproximateHistogramAggregatorTest
final TestFloatColumnSelector selector = new TestFloatColumnSelector(values);
ApproximateHistogramAggregatorFactory factory = new ApproximateHistogramAggregatorFactory(
"billy", "billy", resolution, numBuckets, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY
"billy", "billy", resolution, numBuckets, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, false
);
ApproximateHistogramBufferAggregator agg = new ApproximateHistogramBufferAggregator(selector, resolution);
@ -74,4 +75,41 @@ public class ApproximateHistogramAggregatorTest
Assert.assertEquals("bin count doesn't match expected bin count", 5, h.binCount());
}
@Test
public void testFinalize() throws Exception
{
DefaultObjectMapper objectMapper = new DefaultObjectMapper();
final float[] values = {23, 19, 10, 16, 36, 2, 9, 32, 30, 45};
final int resolution = 5;
final int numBuckets = 5;
final TestFloatColumnSelector selector = new TestFloatColumnSelector(values);
ApproximateHistogramAggregatorFactory humanReadableFactory = new ApproximateHistogramAggregatorFactory(
"billy", "billy", resolution, numBuckets, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, false
);
ApproximateHistogramAggregatorFactory binaryFactory = new ApproximateHistogramAggregatorFactory(
"billy", "billy", resolution, numBuckets, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, true
);
ApproximateHistogramAggregator agg = new ApproximateHistogramAggregator(selector, resolution, 0, 100);
agg.aggregate();
Object finalizedObjectHumanReadable = humanReadableFactory.finalizeComputation(agg.get());
String finalStringHumanReadable = objectMapper.writeValueAsString(finalizedObjectHumanReadable);
Assert.assertEquals(
"{\"breaks\":[23.0,23.0,23.0,23.0,23.0,23.0],\"counts\":[0.0,0.0,0.0,0.0,0.0]}",
finalStringHumanReadable
);
Object finalizedObjectBinary = binaryFactory.finalizeComputation(agg.get());
String finalStringBinary = objectMapper.writeValueAsString(finalizedObjectBinary);
Assert.assertEquals(
"\"//sBQbgAAA==\"",
finalStringBinary
);
}
}

6
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java
View File

@ -163,7 +163,8 @@ public class ApproximateHistogramGroupByQueryTest
10,
5,
Float.NEGATIVE_INFINITY,
Float.POSITIVE_INFINITY
Float.POSITIVE_INFINITY,
false
);
GroupByQuery query = new GroupByQuery.Builder()
@ -222,7 +223,8 @@ public class ApproximateHistogramGroupByQueryTest
10,
5,
Float.NEGATIVE_INFINITY,
Float.POSITIVE_INFINITY
Float.POSITIVE_INFINITY,
false
);
GroupByQuery query = new GroupByQuery.Builder()

3
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java
View File

@ -118,7 +118,8 @@ public class ApproximateHistogramTopNQueryTest
10,
5,
Float.NEGATIVE_INFINITY,
Float.POSITIVE_INFINITY
Float.POSITIVE_INFINITY,
false
);
TopNQuery query = new TopNQueryBuilder()

57
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramBufferAggregatorTest.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.query.aggregation.histogram;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.aggregation.TestFloatColumnSelector;
import org.junit.Assert;
@ -47,7 +48,8 @@ public class FixedBucketsHistogramBufferAggregatorTest
5,
0,
50,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
false
);
FixedBucketsHistogramBufferAggregator agg = new FixedBucketsHistogramBufferAggregator(
@ -79,4 +81,57 @@ public class FixedBucketsHistogramBufferAggregatorTest
Assert.assertEquals("count doesn't match expected count", 10, h.getCount());
}
@Test
public void testFinalize() throws Exception
{
DefaultObjectMapper objectMapper = new DefaultObjectMapper();
final float[] values = {23, 19, 10, 16, 36, 2, 9, 32, 30, 45};
final TestFloatColumnSelector selector = new TestFloatColumnSelector(values);
FixedBucketsHistogramAggregatorFactory humanReadableFactory = new FixedBucketsHistogramAggregatorFactory(
"billy",
"billy",
5,
0,
50,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
false
);
FixedBucketsHistogramAggregatorFactory binaryFactory = new FixedBucketsHistogramAggregatorFactory(
"billy",
"billy",
5,
0,
50,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
true
);
FixedBucketsHistogramAggregator agg = new FixedBucketsHistogramAggregator(
selector,
0,
50,
5,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW
);
agg.aggregate();
Object finalizedObjectHumanReadable = humanReadableFactory.finalizeComputation(agg.get());
String finalStringHumanReadable = objectMapper.writeValueAsString(finalizedObjectHumanReadable);
Assert.assertEquals(
"\"{lowerLimit=0.0, upperLimit=50.0, numBuckets=5, upperOutlierCount=0, lowerOutlierCount=0, missingValueCount=0, histogram=[0, 0, 1, 0, 0], outlierHandlingMode=overflow, count=1, max=23.0, min=23.0}\"",
finalStringHumanReadable
);
Object finalizedObjectBinary = binaryFactory.finalizeComputation(agg.get());
String finalStringBinary = objectMapper.writeValueAsString(finalizedObjectBinary);
Assert.assertEquals(
"\"AQIAAAAAAAAAAEBJAAAAAAAAAAAABQEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEA3AAAAAAAAQDcAAAAAAAAAAAABAAAAAgAAAAAAAAAB\"",
finalStringBinary
);
}
}

8
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramGroupByQueryTest.java
View File

@ -164,7 +164,8 @@ public class FixedBucketsHistogramGroupByQueryTest
10,
0,
2000,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
false
);
GroupByQuery query = new GroupByQuery.Builder()
@ -206,7 +207,7 @@ public class FixedBucketsHistogramGroupByQueryTest
0,
0,
0
)
).toString()
)
);
@ -223,7 +224,8 @@ public class FixedBucketsHistogramGroupByQueryTest
10,
0,
2000,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
false
);
GroupByQuery query = new GroupByQuery.Builder()

9
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramTopNQueryTest.java
View File

@ -118,7 +118,8 @@ public class FixedBucketsHistogramTopNQueryTest
10,
0,
2000,
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW
FixedBucketsHistogram.OutlierHandlingMode.OVERFLOW,
false
);
TopNQuery query = new TopNQueryBuilder()
@ -178,7 +179,7 @@ public class FixedBucketsHistogramTopNQueryTest
0,
0,
0
)
).toString()
)
.build(),
ImmutableMap.<String, Object>builder()
@ -205,7 +206,7 @@ public class FixedBucketsHistogramTopNQueryTest
0,
0,
0
)
).toString()
)
.build(),
ImmutableMap.<String, Object>builder()
@ -232,7 +233,7 @@ public class FixedBucketsHistogramTopNQueryTest
0,
0,
0
)
).toString()
)
.build()
)

58
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java
View File

@ -140,7 +140,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
20,
0,
10,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
)
)
.withRollup(false)
@ -239,25 +240,55 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
)
.aggregators(ImmutableList.of(
new FixedBucketsHistogramAggregatorFactory(
"a0:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE
"a0:agg",
"m1",
20,
0.0d,
10.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new FixedBucketsHistogramAggregatorFactory(
"a4:agg", "v0", 40, 0.0d, 20.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE
"a4:agg",
"v0",
40,
0.0d,
20.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new FilteredAggregatorFactory(
new FixedBucketsHistogramAggregatorFactory(
"a5:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE
"a5:agg",
"m1",
20,
0.0d,
10.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new FixedBucketsHistogramAggregatorFactory(
"a6:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE
"a6:agg",
"m1",
20,
0.0d,
10.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
),
new FixedBucketsHistogramAggregatorFactory(
"a8:agg", "cnt", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE
"a8:agg",
"cnt",
20,
0.0d,
10.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
)
))
.postAggregators(
@ -325,7 +356,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
20,
0.0,
10.0,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new FixedBucketsHistogramAggregatorFactory(
"a2:agg",
@ -333,7 +365,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
30,
0.0,
10.0,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new FilteredAggregatorFactory(
new FixedBucketsHistogramAggregatorFactory(
@ -342,7 +375,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
20,
0.0,
10.0,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new SelectorDimFilter("dim1", "abc", null)
),
@ -353,7 +387,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
20,
0.0,
10.0,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
)
@ -427,7 +462,8 @@ public class FixedBucketsHistogramQuantileSqlAggregatorTest extends CalciteTestB
100,
0,
100.0d,
FixedBucketsHistogram.OutlierHandlingMode.IGNORE
FixedBucketsHistogram.OutlierHandlingMode.IGNORE,
false
)
)
.setPostAggregatorSpecs(

26
extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java
View File

@ -139,7 +139,8 @@ public class QuantileSqlAggregatorTest extends CalciteTestBase
null,
null,
null,
null
null,
false
)
)
.withRollup(false)
@ -239,18 +240,18 @@ public class QuantileSqlAggregatorTest extends CalciteTestBase
)
)
.aggregators(ImmutableList.of(
new ApproximateHistogramAggregatorFactory("a0:agg", "m1", null, null, null, null),
new ApproximateHistogramAggregatorFactory("a2:agg", "m1", 200, null, null, null),
new ApproximateHistogramAggregatorFactory("a4:agg", "v0", null, null, null, null),
new ApproximateHistogramAggregatorFactory("a0:agg", "m1", null, null, null, null, false),
new ApproximateHistogramAggregatorFactory("a2:agg", "m1", 200, null, null, null, false),
new ApproximateHistogramAggregatorFactory("a4:agg", "v0", null, null, null, null, false),
new FilteredAggregatorFactory(
new ApproximateHistogramAggregatorFactory("a5:agg", "m1", null, null, null, null),
new ApproximateHistogramAggregatorFactory("a5:agg", "m1", null, null, null, null, false),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new ApproximateHistogramAggregatorFactory("a6:agg", "m1", null, null, null, null),
new ApproximateHistogramAggregatorFactory("a6:agg", "m1", null, null, null, null, false),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
),
new ApproximateHistogramAggregatorFactory("a8:agg", "cnt", null, null, null, null)
new ApproximateHistogramAggregatorFactory("a8:agg", "cnt", null, null, null, null, false)
))
.postAggregators(
new QuantilePostAggregator("a0", "a0:agg", 0.01f),
@ -300,14 +301,14 @@ public class QuantileSqlAggregatorTest extends CalciteTestBase
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
new ApproximateHistogramFoldingAggregatorFactory("a0:agg", "hist_m1", null, null, null, null),
new ApproximateHistogramFoldingAggregatorFactory("a2:agg", "hist_m1", 200, null, null, null),
new ApproximateHistogramFoldingAggregatorFactory("a0:agg", "hist_m1", null, null, null, null, false),
new ApproximateHistogramFoldingAggregatorFactory("a2:agg", "hist_m1", 200, null, null, null, false),
new FilteredAggregatorFactory(
new ApproximateHistogramFoldingAggregatorFactory("a4:agg", "hist_m1", null, null, null, null),
new ApproximateHistogramFoldingAggregatorFactory("a4:agg", "hist_m1", null, null, null, null, false),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new ApproximateHistogramFoldingAggregatorFactory("a5:agg", "hist_m1", null, null, null, null),
new ApproximateHistogramFoldingAggregatorFactory("a5:agg", "hist_m1", null, null, null, null, false),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
)
))
@ -376,7 +377,8 @@ public class QuantileSqlAggregatorTest extends CalciteTestBase
null,
null,
null,
null
null,
false
)
)
.setPostAggregatorSpecs(