Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Attempt to coerce COMPLEX to number in numeric aggregators. (#16564) 

* Coerce COMPLEX to number in numeric aggregators.

PR #15371 eliminated ObjectColumnSelector's built-in implementations of
numeric methods, which had been marked deprecated.

However, some complex types, like SpectatorHistogram, can be successfully coerced
to number. The documentation for spectator histograms encourages taking advantage of
this by aggregating complex columns with doubleSum and longSum. Currently, this
doesn't work properly for IncrementalIndex, where the behavior relied on those
deprecated ObjectColumnSelector methods.

This patch fixes the behavior by making two changes:

1) SimpleXYZAggregatorFactory (XYZ = type; base class for simple numeric aggregators;
   all of these extend NullableNumericAggregatorFactory) use getObject for STRING
   and COMPLEX. Previously, getObject was only used for STRING.

2) NullableNumericAggregatorFactory (base class for simple numeric aggregators)
   has a new protected method "useGetObject". This allows the base class to
   correctly check for null (using getObject or isNull).

The patch also adds a test for SpectatorHistogram + doubleSum + IncrementalIndex.

* Fix tests.

* Remove the special ColumnValueSelector.

* Add test.
This commit is contained in:
Gian Merlino 2024-07-25 08:45:29 -07:00 committed by GitHub
parent b5f117bca2
commit b2a88da200
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 236 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
extensions-contrib/spectator-histogram/pom.xml
View File

@ -92,6 +92,11 @@
<artifactId>error_prone_annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-sql</artifactId>
@ -137,5 +142,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

69
extensions-contrib/spectator-histogram/src/main/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumn.java
View File

@ -19,18 +19,12 @@
package org.apache.druid.spectator.histogram;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ComplexColumn;
import org.apache.druid.segment.data.ReadableOffset;
import javax.annotation.Nullable;
public class SpectatorHistogramIndexBasedComplexColumn implements ComplexColumn
{
private final SpectatorHistogramIndexed index;
private final String typeName;
private static final Number ZERO = 0;
public SpectatorHistogramIndexBasedComplexColumn(String typeName, SpectatorHistogramIndexed index)
{
@ -59,72 +53,11 @@ public class SpectatorHistogramIndexBasedComplexColumn implements ComplexColumn
@Override
public int getLength()
{
return index.size();
return -1;
}
@Override
public void close()
{
}
@Override
public ColumnValueSelector<SpectatorHistogram> makeColumnValueSelector(ReadableOffset offset)
{
// Use ColumnValueSelector directly so that we support being queried as a Number using
// longSum or doubleSum aggregators, the NullableNumericBufferAggregator will call isNull.
// This allows us to behave as a Number or SpectatorHistogram object.
// When queried as a Number, we're returning the count of entries in the histogram.
// As such, we can safely return 0 where the histogram is null.
return new ColumnValueSelector<SpectatorHistogram>()
{
@Override
public boolean isNull()
{
return getObject() == null;
}
private Number getOrZero()
{
SpectatorHistogram histogram = getObject();
return histogram != null ? histogram : ZERO;
}
@Override
public long getLong()
{
return getOrZero().longValue();
}
@Override
public float getFloat()
{
return getOrZero().floatValue();
}
@Override
public double getDouble()
{
return getOrZero().doubleValue();
}
@Nullable
@Override
public SpectatorHistogram getObject()
{
return (SpectatorHistogram) getRowValue(offset.getOffset());
}
@Override
public Class classOfObject()
{
return getClazz();
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector)
{
inspector.visit("column", SpectatorHistogramIndexBasedComplexColumn.this);
}
};
}
}

67
extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramAggregatorTest.java
View File

@ -20,8 +20,14 @@
package org.apache.druid.spectator.histogram;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.netflix.spectator.api.histogram.PercentileBuckets;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.MapBasedInputRow;
import org.apache.druid.data.input.impl.NoopInputRowParser;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.query.Druids;
@ -32,6 +38,9 @@ import org.apache.druid.query.Result;
import org.apache.druid.query.aggregation.AggregationTestHelper;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.AggregatorUtil;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.groupby.GroupByQuery;
import org.apache.druid.query.groupby.GroupByQueryConfig;
import org.apache.druid.query.groupby.GroupByQueryRunnerTest;
import org.apache.druid.query.groupby.ResultRow;
@ -42,13 +51,17 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis;
import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
import org.apache.druid.query.timeseries.TimeseriesResultValue;
import org.apache.druid.segment.IncrementalIndexSegment;
import org.apache.druid.segment.IndexIO;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexSegment;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.incremental.IncrementalIndex;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.SegmentId;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@ -59,6 +72,7 @@ import org.junit.runners.Parameterized;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -716,6 +730,59 @@ public class SpectatorHistogramAggregatorTest extends InitializedNullHandlingTes
}
}
@Test
public void testBuildingAndCountingHistogramsIncrementalIndex() throws Exception
{
List<String> dimensions = Collections.singletonList("d");
int n = 10;
DateTime startOfDay = DateTimes.of("2000-01-01");
List<InputRow> inputRows = new ArrayList<>(n);
for (int i = 1; i <= n; i++) {
String val = String.valueOf(i * 1.0d);
inputRows.add(new MapBasedInputRow(
startOfDay.plusMinutes(i),
dimensions,
ImmutableMap.of("x", i, "d", val)
));
}
IncrementalIndex index = AggregationTestHelper.createIncrementalIndex(
inputRows.iterator(),
new NoopInputRowParser(null),
new AggregatorFactory[]{
new CountAggregatorFactory("count"),
new SpectatorHistogramAggregatorFactory("histogram", "x")
},
0,
Granularities.NONE,
100,
false
);
ImmutableList<Segment> segments = ImmutableList.of(
new IncrementalIndexSegment(index, SegmentId.dummy("test")),
helper.persistIncrementalIndex(index, null)
);
GroupByQuery query = new GroupByQuery.Builder()
.setDataSource("test")
.setGranularity(Granularities.HOUR)
.setInterval("1970/2050")
.setAggregatorSpecs(
new DoubleSumAggregatorFactory("doubleSum", "histogram")
).build();
Sequence<ResultRow> seq = helper.runQueryOnSegmentsObjs(segments, query);
List<ResultRow> results = seq.toList();
Assert.assertEquals(1, results.size());
// Check timestamp
Assert.assertEquals(startOfDay.getMillis(), results.get(0).get(0));
// Check doubleSum
Assert.assertEquals(n * segments.size(), (Double) results.get(0).get(1), 0.001);
}
private static void assertResultsMatch(List<ResultRow> results, int rowNum, String expectedProduct)
{
ResultRow row = results.get(rowNum);

42
extensions-contrib/spectator-histogram/src/test/java/org/apache/druid/spectator/histogram/SpectatorHistogramIndexBasedComplexColumnTest.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.spectator.histogram;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Test;
public class SpectatorHistogramIndexBasedComplexColumnTest
{
@Test
public void testComplexColumn()
{
final SpectatorHistogramIndexed mockIndexed = EasyMock.createMock(SpectatorHistogramIndexed.class);
EasyMock.replay(mockIndexed);
final String typeName = "type";
final SpectatorHistogramIndexBasedComplexColumn column =
new SpectatorHistogramIndexBasedComplexColumn("type", mockIndexed);
Assert.assertEquals(typeName, column.getTypeName());
Assert.assertEquals(-1, column.getLength());
EasyMock.verify(mockIndexed);
}
}

26
processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java
View File

@ -28,6 +28,8 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.BaseLongColumnValueSelector;
import org.apache.druid.segment.BaseObjectColumnValueSelector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
@ -35,6 +37,8 @@ import org.apache.druid.segment.DoubleColumnSelector;
import org.apache.druid.segment.FloatColumnSelector;
import org.apache.druid.segment.LongColumnSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.Types;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorValueSelector;
import org.apache.druid.segment.virtual.ExpressionSelectors;
@ -428,4 +432,26 @@ public class AggregatorUtil
.array();
});
}
/**
* Whether a simple numeric aggregator should use {@link BaseObjectColumnValueSelector#getObject()}, and coerce the
* result to number, rather than using a primitive method like {@link BaseLongColumnValueSelector#getLong()}.
*
* @param fieldName field name, or null if the aggregator is expression-based
* @param columnSelectorFactory column selector factory
*/
public static boolean shouldUseObjectColumnAggregatorWrapper(
@Nullable final String fieldName,
final ColumnSelectorFactory columnSelectorFactory
)
{
if (fieldName != null) {
ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName);
// STRING can be coerced to a number. COMPLEX types can be subclasses of Number (or subclasses of some type
// that is coercible to a number.)
return Types.is(capabilities, ValueType.STRING) || Types.is(capabilities, ValueType.COMPLEX);
}
return false;
}
}

2
processing/src/main/java/org/apache/druid/query/aggregation/DelegatingAggregator.java
View File

@ -23,7 +23,7 @@ import javax.annotation.Nullable;
/**
* An Aggregator that delegates everything. It is used by Aggregator wrappers e.g.
* {@link StringColumnDoubleAggregatorWrapper} that modify some behavior of a delegate.
* {@link ObjectColumnDoubleAggregatorWrapper} that modify some behavior of a delegate.
*/
public abstract class DelegatingAggregator implements Aggregator
{

2
processing/src/main/java/org/apache/druid/query/aggregation/DelegatingBufferAggregator.java
View File

@ -26,7 +26,7 @@ import java.nio.ByteBuffer;
/**
* A BufferAggregator that delegates everything. It is used by BufferAggregator wrappers e.g.
* {@link StringColumnDoubleBufferAggregatorWrapper} that modify some behavior of a delegate.
* {@link ObjectColumnDoubleBufferAggregatorWrapper} that modify some behavior of a delegate.
*/
public abstract class DelegatingBufferAggregator implements BufferAggregator
{

46
processing/src/main/java/org/apache/druid/query/aggregation/NullableNumericAggregatorFactory.java
View File

@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.guice.annotations.ExtensionPoint;
import org.apache.druid.segment.BaseNullableColumnValueSelector;
import org.apache.druid.segment.BaseObjectColumnValueSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ColumnType;
@ -38,8 +39,9 @@ import org.apache.druid.segment.vector.VectorValueSelector;
* values to be aggregated are null values, or if no values are aggregated at all. If any of the values are non-null,
* the result will be the aggregated value of the non-null values.
*
* This superclass should only be extended by aggregators that read primitive numbers. It implements logic that is
* not valid for non-numeric selector methods such as {@link ColumnValueSelector#getObject()}.
* Aggregators that use {@link ColumnValueSelector#getObject()} must override
* {@link #useGetObject(ColumnSelectorFactory)}. Otherwise, the logic in this class is not correct for
* non-numeric selectors.
*
* @see BaseNullableColumnValueSelector#isNull() for why this only works in the numeric case
*/
@ -51,16 +53,18 @@ public abstract class NullableNumericAggregatorFactory<T extends BaseNullableCol
public final Aggregator factorize(ColumnSelectorFactory columnSelectorFactory)
{
T selector = selector(columnSelectorFactory);
BaseNullableColumnValueSelector nullSelector = makeNullSelector(selector, columnSelectorFactory);
Aggregator aggregator = factorize(columnSelectorFactory, selector);
return NullHandling.replaceWithDefault() ? aggregator : new NullableNumericAggregator(aggregator, selector);
return NullHandling.sqlCompatible() ? new NullableNumericAggregator(aggregator, nullSelector) : aggregator;
}
@Override
public final BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory)
{
T selector = selector(columnSelectorFactory);
BaseNullableColumnValueSelector nullSelector = makeNullSelector(selector, columnSelectorFactory);
BufferAggregator aggregator = factorizeBuffered(columnSelectorFactory, selector);
return NullHandling.replaceWithDefault() ? aggregator : new NullableNumericBufferAggregator(aggregator, selector);
return NullHandling.sqlCompatible() ? new NullableNumericBufferAggregator(aggregator, nullSelector) : aggregator;
}
@Override
@ -69,14 +73,14 @@ public abstract class NullableNumericAggregatorFactory<T extends BaseNullableCol
Preconditions.checkState(canVectorize(columnSelectorFactory), "Cannot vectorize");
VectorValueSelector selector = vectorSelector(columnSelectorFactory);
VectorAggregator aggregator = factorizeVector(columnSelectorFactory, selector);
return NullHandling.replaceWithDefault() ? aggregator : new NullableNumericVectorAggregator(aggregator, selector);
return NullHandling.sqlCompatible() ? new NullableNumericVectorAggregator(aggregator, selector) : aggregator;
}
@Override
public final AggregateCombiner makeNullableAggregateCombiner()
{
AggregateCombiner combiner = makeAggregateCombiner();
return NullHandling.replaceWithDefault() ? combiner : new NullableNumericAggregateCombiner(combiner);
AggregateCombiner<?> combiner = makeAggregateCombiner();
return NullHandling.sqlCompatible() ? new NullableNumericAggregateCombiner<>(combiner) : combiner;
}
@Override
@ -85,6 +89,23 @@ public abstract class NullableNumericAggregatorFactory<T extends BaseNullableCol
return getMaxIntermediateSize() + (NullHandling.replaceWithDefault() ? 0 : Byte.BYTES);
}
/**
* Returns the selector that should be used by {@link NullableNumericAggregator} and
* {@link NullableNumericBufferAggregator} to determine if the current value is null.
*/
private BaseNullableColumnValueSelector makeNullSelector(
final T selector,
final ColumnSelectorFactory columnSelectorFactory
)
{
if (useGetObject(columnSelectorFactory)) {
final BaseObjectColumnValueSelector<?> objectSelector = (BaseObjectColumnValueSelector<?>) selector;
return () -> objectSelector.getObject() == null;
} else {
return selector;
}
}
// ---- ABSTRACT METHODS BELOW ------
/**
@ -94,6 +115,17 @@ public abstract class NullableNumericAggregatorFactory<T extends BaseNullableCol
*/
protected abstract T selector(ColumnSelectorFactory columnSelectorFactory);
/**
* Returns whether the selector created by {@link #selector(ColumnSelectorFactory)} for the given
* {@link ColumnSelectorFactory} prefers {@link BaseObjectColumnValueSelector#getObject()}.
*
* For backwards compatibilty with older extensions, this is a non-abstract method.
*/
protected boolean useGetObject(ColumnSelectorFactory columnSelectorFactory)
{
return false;
}
/**
* Creates a {@link VectorValueSelector} for the aggregated column.
*

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnDoubleAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnDoubleAggregatorWrapper.java
View File

@ -28,15 +28,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Double Aggregator that consume double type columns to handle String type.
* This class can be used to wrap Double Aggregator that consume double type columns to handle Object type.
*/
public class StringColumnDoubleAggregatorWrapper extends DelegatingAggregator
public class ObjectColumnDoubleAggregatorWrapper extends DelegatingAggregator
{
private final BaseObjectColumnValueSelector selector;
private final double nullValue;
private final SettableValueDoubleColumnValueSelector doubleSelector;
public StringColumnDoubleAggregatorWrapper(
public ObjectColumnDoubleAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseDoubleColumnValueSelector, Aggregator> delegateBuilder,
double nullValue

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnDoubleBufferAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnDoubleBufferAggregatorWrapper.java
View File

@ -29,15 +29,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Double BufferAggregator that consume double type columns to handle String type.
* This class can be used to wrap Double BufferAggregator that consume double type columns to handle Object type.
*/
public class StringColumnDoubleBufferAggregatorWrapper extends DelegatingBufferAggregator
public class ObjectColumnDoubleBufferAggregatorWrapper extends DelegatingBufferAggregator
{
private final BaseObjectColumnValueSelector selector;
private final double nullValue;
private final SettableValueDoubleColumnValueSelector doubleSelector;
public StringColumnDoubleBufferAggregatorWrapper(
public ObjectColumnDoubleBufferAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseDoubleColumnValueSelector, BufferAggregator> delegateBuilder,
double nullValue

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnFloatAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnFloatAggregatorWrapper.java
View File

@ -28,15 +28,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Float Aggregator that consume float type columns to handle String type.
* This class can be used to wrap Float Aggregator that consume float type columns to handle Object type.
*/
public class StringColumnFloatAggregatorWrapper extends DelegatingAggregator
public class ObjectColumnFloatAggregatorWrapper extends DelegatingAggregator
{
private final BaseObjectColumnValueSelector selector;
private final float nullValue;
private final SettableValueFloatColumnValueSelector floatSelector;
public StringColumnFloatAggregatorWrapper(
public ObjectColumnFloatAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseFloatColumnValueSelector, Aggregator> delegateBuilder,
float nullValue

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnFloatBufferAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnFloatBufferAggregatorWrapper.java
View File

@ -29,15 +29,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Float BufferAggregator that consume float type columns to handle String type.
* This class can be used to wrap Float BufferAggregator that consume float type columns to handle Object type.
*/
public class StringColumnFloatBufferAggregatorWrapper extends DelegatingBufferAggregator
public class ObjectColumnFloatBufferAggregatorWrapper extends DelegatingBufferAggregator
{
private final BaseObjectColumnValueSelector selector;
private final float nullValue;
private final SettableValueFloatColumnValueSelector floatSelector;
public StringColumnFloatBufferAggregatorWrapper(
public ObjectColumnFloatBufferAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseFloatColumnValueSelector, BufferAggregator> delegateBuilder,
float nullValue

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnLongAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnLongAggregatorWrapper.java
View File

@ -28,15 +28,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Long Aggregator that consume long type columns to handle String type.
* This class can be used to wrap Long Aggregator that consume long type columns to handle Object type.
*/
public class StringColumnLongAggregatorWrapper extends DelegatingAggregator
public class ObjectColumnLongAggregatorWrapper extends DelegatingAggregator
{
private final BaseObjectColumnValueSelector selector;
private final long nullValue;
private final SettableValueLongColumnValueSelector longSelector;
public StringColumnLongAggregatorWrapper(
public ObjectColumnLongAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseLongColumnValueSelector, Aggregator> delegateBuilder,
long nullValue

6
processing/src/main/java/org/apache/druid/query/aggregation/StringColumnLongBufferAggregatorWrapper.java → processing/src/main/java/org/apache/druid/query/aggregation/ObjectColumnLongBufferAggregatorWrapper.java
View File

@ -29,15 +29,15 @@ import java.util.List;
import java.util.function.Function;
/**
* This class can be used to wrap Long BufferAggregator that consume long type columns to handle String type.
* This class can be used to wrap Long BufferAggregator that consume long type columns to handle Object type.
*/
public class StringColumnLongBufferAggregatorWrapper extends DelegatingBufferAggregator
public class ObjectColumnLongBufferAggregatorWrapper extends DelegatingBufferAggregator
{
private final BaseObjectColumnValueSelector selector;
private final long nullValue;
private final SettableValueLongColumnValueSelector longSelector;
public StringColumnLongBufferAggregatorWrapper(
public ObjectColumnLongBufferAggregatorWrapper(
BaseObjectColumnValueSelector selector,
Function<BaseLongColumnValueSelector, BufferAggregator> delegateBuilder,
long nullValue

20
processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java
View File

@ -31,11 +31,8 @@ import org.apache.druid.segment.BaseDoubleColumnValueSelector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.Types;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorValueSelector;
@ -86,8 +83,8 @@ public abstract class SimpleDoubleAggregatorFactory extends NullableNumericAggre
@Override
protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnDoubleAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnDoubleAggregatorWrapper(
selector,
SimpleDoubleAggregatorFactory.this::buildAggregator,
nullValue()
@ -103,8 +100,8 @@ public abstract class SimpleDoubleAggregatorFactory extends NullableNumericAggre
ColumnValueSelector selector
)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnDoubleBufferAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnDoubleBufferAggregatorWrapper(
selector,
SimpleDoubleAggregatorFactory.this::buildBufferAggregator,
nullValue()
@ -131,13 +128,10 @@ public abstract class SimpleDoubleAggregatorFactory extends NullableNumericAggre
return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression);
}
private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory)
@Override
protected boolean useGetObject(ColumnSelectorFactory columnSelectorFactory)
{
if (fieldName != null) {
ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName);
return Types.is(capabilities, ValueType.STRING);
}
return false;
return AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, columnSelectorFactory);
}
@Override

26
processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java
View File

@ -31,10 +31,7 @@ import org.apache.druid.segment.BaseFloatColumnValueSelector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.Types;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorValueSelector;
@ -76,8 +73,8 @@ public abstract class SimpleFloatAggregatorFactory extends NullableNumericAggreg
@Override
protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnFloatAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnFloatAggregatorWrapper(
selector,
SimpleFloatAggregatorFactory.this::buildAggregator,
nullValue()
@ -93,8 +90,8 @@ public abstract class SimpleFloatAggregatorFactory extends NullableNumericAggreg
ColumnValueSelector selector
)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnFloatBufferAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnFloatBufferAggregatorWrapper(
selector,
SimpleFloatAggregatorFactory.this::buildBufferAggregator,
nullValue()
@ -121,6 +118,12 @@ public abstract class SimpleFloatAggregatorFactory extends NullableNumericAggreg
return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression);
}
@Override
protected boolean useGetObject(ColumnSelectorFactory columnSelectorFactory)
{
return AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, columnSelectorFactory);
}
@Override
public Object deserialize(Object object)
{
@ -233,15 +236,6 @@ public abstract class SimpleFloatAggregatorFactory extends NullableNumericAggreg
return AggregatorUtil.canVectorize(columnInspector, fieldName, expression, fieldExpression);
}
private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory)
{
if (fieldName != null) {
ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName);
return Types.is(capabilities, ValueType.STRING);
}
return false;
}
protected abstract float nullValue();
protected abstract Aggregator buildAggregator(BaseFloatColumnValueSelector selector);

26
processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java
View File

@ -31,10 +31,7 @@ import org.apache.druid.segment.BaseLongColumnValueSelector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.Types;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorValueSelector;
@ -82,8 +79,8 @@ public abstract class SimpleLongAggregatorFactory extends NullableNumericAggrega
@Override
protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnLongAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnLongAggregatorWrapper(
selector,
SimpleLongAggregatorFactory.this::buildAggregator,
nullValue()
@ -99,8 +96,8 @@ public abstract class SimpleLongAggregatorFactory extends NullableNumericAggrega
ColumnValueSelector selector
)
{
if (shouldUseStringColumnAggregatorWrapper(metricFactory)) {
return new StringColumnLongBufferAggregatorWrapper(
if (AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, metricFactory)) {
return new ObjectColumnLongBufferAggregatorWrapper(
selector,
SimpleLongAggregatorFactory.this::buildBufferAggregator,
nullValue()
@ -127,6 +124,12 @@ public abstract class SimpleLongAggregatorFactory extends NullableNumericAggrega
return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression);
}
@Override
protected boolean useGetObject(ColumnSelectorFactory columnSelectorFactory)
{
return AggregatorUtil.shouldUseObjectColumnAggregatorWrapper(fieldName, columnSelectorFactory);
}
@Override
public Object deserialize(Object object)
{
@ -236,15 +239,6 @@ public abstract class SimpleLongAggregatorFactory extends NullableNumericAggrega
return AggregatorUtil.canVectorize(columnInspector, fieldName, expression, fieldExpression);
}
private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory)
{
if (fieldName != null) {
ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName);
return Types.is(capabilities, ValueType.STRING);
}
return false;
}
protected abstract long nullValue();
protected abstract Aggregator buildAggregator(BaseLongColumnValueSelector selector);

2
processing/src/test/java/org/apache/druid/query/aggregation/DoubleMaxAggregationTest.java
View File

@ -63,7 +63,7 @@ public class DoubleMaxAggregationTest
selector = new TestDoubleColumnSelectorImpl(values);
colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class);
EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(selector);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null).anyTimes();
EasyMock.replay(colSelectorFactory);
VectorValueSelector vectorValueSelector = EasyMock.createMock(VectorValueSelector.class);

2
processing/src/test/java/org/apache/druid/query/aggregation/DoubleMinAggregationTest.java
View File

@ -63,7 +63,7 @@ public class DoubleMinAggregationTest
selector = new TestDoubleColumnSelectorImpl(values);
colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class);
EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(selector);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null).anyTimes();
EasyMock.replay(colSelectorFactory);

2
processing/src/test/java/org/apache/druid/query/aggregation/LongMaxAggregationTest.java
View File

@ -62,7 +62,7 @@ public class LongMaxAggregationTest
selector = new TestLongColumnSelector(values);
colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class);
EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(selector);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null).anyTimes();
EasyMock.replay(colSelectorFactory);

2
processing/src/test/java/org/apache/druid/query/aggregation/LongMinAggregationTest.java
View File

@ -62,7 +62,7 @@ public class LongMinAggregationTest
selector = new TestLongColumnSelector(values);
colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class);
EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(selector);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null);
EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(null).anyTimes();
EasyMock.replay(colSelectorFactory);
VectorValueSelector vectorValueSelector = EasyMock.createMock(VectorValueSelector.class);