mirror of https://github.com/apache/druid.git
Fix post-aggregator computation when used with subtotals (#10653)
* Fix post-aggregator computation * remove commented code * Fix numeric null handling * Add test when subquery returns null long
This commit is contained in:
Abhishek Agarwal
GitHub
parent
6ae8059c09
commit
796c25532e
|
|
@ -712,13 +712,13 @@ public class RowBasedGrouperHelper
|
|||
return new StringInputRawSupplierColumnSelectorStrategy();
|
||||
case LONG:
|
||||
return (InputRawSupplierColumnSelectorStrategy<BaseLongColumnValueSelector>)
|
||||
columnSelector -> columnSelector::getLong;
|
||||
columnSelector -> () -> columnSelector.isNull() ? null : columnSelector.getLong();
|
||||
case FLOAT:
|
||||
return (InputRawSupplierColumnSelectorStrategy<BaseFloatColumnValueSelector>)
|
||||
columnSelector -> columnSelector::getFloat;
|
||||
columnSelector -> () -> columnSelector.isNull() ? null : columnSelector.getFloat();
|
||||
case DOUBLE:
|
||||
return (InputRawSupplierColumnSelectorStrategy<BaseDoubleColumnValueSelector>)
|
||||
columnSelector -> columnSelector::getDouble;
|
||||
columnSelector -> () -> columnSelector.isNull() ? null : columnSelector.getDouble();
|
||||
default:
|
||||
throw new IAE("Cannot create query type helper from invalid type [%s]", type);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -408,27 +408,10 @@ public class GroupByStrategyV2 implements GroupByStrategy
|
|||
// Dimension spec including dimension name and output name
|
||||
final List<DimensionSpec> subTotalDimensionSpec = new ArrayList<>(dimsInSubtotalSpec.size());
|
||||
final List<DimensionSpec> dimensions = query.getDimensions();
|
||||
final List<DimensionSpec> newDimensions = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < dimensions.size(); i++) {
|
||||
DimensionSpec dimensionSpec = dimensions.get(i);
|
||||
for (DimensionSpec dimensionSpec : dimensions) {
|
||||
if (dimsInSubtotalSpec.contains(dimensionSpec.getOutputName())) {
|
||||
newDimensions.add(
|
||||
new DefaultDimensionSpec(
|
||||
dimensionSpec.getOutputName(),
|
||||
dimensionSpec.getOutputName(),
|
||||
dimensionSpec.getOutputType()
|
||||
)
|
||||
);
|
||||
subTotalDimensionSpec.add(dimensionSpec);
|
||||
} else {
|
||||
// Insert dummy dimension so all subtotals queries have ResultRows with the same shape.
|
||||
// Use a field name that does not appear in the main query result, to assure the result will be null.
|
||||
String dimName = "_" + i;
|
||||
while (query.getResultRowSignature().indexOf(dimName) >= 0) {
|
||||
dimName = "_" + dimName;
|
||||
}
|
||||
newDimensions.add(DefaultDimensionSpec.of(dimName));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -442,8 +425,7 @@ public class GroupByStrategyV2 implements GroupByStrategy
|
|||
}
|
||||
|
||||
GroupByQuery subtotalQuery = baseSubtotalQuery
|
||||
.withLimitSpec(subtotalQueryLimitSpec)
|
||||
.withDimensionSpecs(newDimensions);
|
||||
.withLimitSpec(subtotalQueryLimitSpec);
|
||||
|
||||
final GroupByRowProcessor.ResultSupplier resultSupplierOneFinal = resultSupplierOne;
|
||||
if (Utils.isPrefix(subtotalSpec, queryDimNames)) {
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ import org.apache.druid.query.extraction.ExtractionFn;
|
|||
import org.apache.druid.query.extraction.JavaScriptExtractionFn;
|
||||
import org.apache.druid.query.extraction.MapLookupExtractor;
|
||||
import org.apache.druid.query.extraction.RegexDimExtractionFn;
|
||||
import org.apache.druid.query.extraction.SearchQuerySpecDimExtractionFn;
|
||||
import org.apache.druid.query.extraction.StringFormatExtractionFn;
|
||||
import org.apache.druid.query.extraction.StrlenExtractionFn;
|
||||
import org.apache.druid.query.extraction.TimeFormatExtractionFn;
|
||||
|
|
@ -9563,6 +9564,92 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
|
|||
TestHelper.assertExpectedObjects(expectedResults, results, "numerics");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupByNestedWithInnerQueryOutputNullNumerics()
|
||||
{
|
||||
cannotVectorize();
|
||||
|
||||
if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
|
||||
expectedException.expect(UnsupportedOperationException.class);
|
||||
expectedException.expectMessage("GroupBy v1 only supports dimensions with an outputType of STRING.");
|
||||
}
|
||||
|
||||
// Following extractionFn will generate null value for one kind of quality
|
||||
ExtractionFn extractionFn = new SearchQuerySpecDimExtractionFn(new ContainsSearchQuerySpec("1200", false));
|
||||
GroupByQuery subquery = makeQueryBuilder()
|
||||
.setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
|
||||
.setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
|
||||
.setDimensions(
|
||||
new DefaultDimensionSpec("quality", "alias"),
|
||||
new ExtractionDimensionSpec("qualityLong", "ql_alias", ValueType.LONG, extractionFn),
|
||||
new ExtractionDimensionSpec("qualityFloat", "qf_alias", ValueType.FLOAT, extractionFn),
|
||||
new ExtractionDimensionSpec("qualityDouble", "qd_alias", ValueType.DOUBLE, extractionFn)
|
||||
)
|
||||
.setDimFilter(
|
||||
new InDimFilter(
|
||||
"quality",
|
||||
Arrays.asList("entertainment", "business"),
|
||||
null
|
||||
)
|
||||
).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index"))
|
||||
.setGranularity(QueryRunnerTestHelper.DAY_GRAN)
|
||||
.build();
|
||||
|
||||
GroupByQuery outerQuery = makeQueryBuilder()
|
||||
.setDataSource(subquery)
|
||||
.setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
|
||||
.setDimensions(
|
||||
new DefaultDimensionSpec("ql_alias", "quallong", ValueType.LONG),
|
||||
new DefaultDimensionSpec("qf_alias", "qualfloat", ValueType.FLOAT),
|
||||
new DefaultDimensionSpec("qd_alias", "qualdouble", ValueType.DOUBLE)
|
||||
)
|
||||
.setAggregatorSpecs(
|
||||
new LongSumAggregatorFactory("ql_alias_sum", "ql_alias"),
|
||||
new DoubleSumAggregatorFactory("qf_alias_sum", "qf_alias"),
|
||||
new DoubleSumAggregatorFactory("qd_alias_sum", "qd_alias")
|
||||
)
|
||||
.setGranularity(QueryRunnerTestHelper.ALL_GRAN)
|
||||
.build();
|
||||
|
||||
List<ResultRow> expectedResults = Arrays.asList(
|
||||
makeRow(
|
||||
outerQuery,
|
||||
"2011-04-01",
|
||||
"quallong",
|
||||
NullHandling.defaultLongValue(),
|
||||
"qualfloat",
|
||||
NullHandling.defaultFloatValue(),
|
||||
"qualdouble",
|
||||
NullHandling.defaultDoubleValue(),
|
||||
"ql_alias_sum",
|
||||
NullHandling.defaultLongValue(),
|
||||
"qf_alias_sum",
|
||||
NullHandling.defaultFloatValue(),
|
||||
"qd_alias_sum",
|
||||
NullHandling.defaultDoubleValue()
|
||||
),
|
||||
makeRow(
|
||||
outerQuery,
|
||||
"2011-04-01",
|
||||
"quallong",
|
||||
1200L,
|
||||
"qualfloat",
|
||||
12000.0,
|
||||
"qualdouble",
|
||||
12000.0,
|
||||
"ql_alias_sum",
|
||||
2400L,
|
||||
"qf_alias_sum",
|
||||
24000.0,
|
||||
"qd_alias_sum",
|
||||
24000.0
|
||||
)
|
||||
);
|
||||
|
||||
Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, outerQuery);
|
||||
TestHelper.assertExpectedObjects(expectedResults, results, "numerics");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupByNestedWithInnerQueryNumericsWithLongTime()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -12159,23 +12159,23 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
|
|||
List<Object[]> resultList;
|
||||
if (NullHandling.sqlCompatible()) {
|
||||
resultList = ImmutableList.of(
|
||||
new Object[]{NULL_STRING, 2L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"", 1L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"a", 2L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"abc", 1L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{NULL_STRING, 2L, 0L, NULL_STRING},
|
||||
new Object[]{"", 1L, 0L, ""},
|
||||
new Object[]{"a", 2L, 0L, "a"},
|
||||
new Object[]{"abc", 1L, 0L, "abc"},
|
||||
new Object[]{NULL_STRING, 6L, 1L, "ALL"}
|
||||
);
|
||||
} else {
|
||||
resultList = ImmutableList.of(
|
||||
new Object[]{"", 3L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"a", 2L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"abc", 1L, 0L, "INDIVIDUAL"},
|
||||
new Object[]{"", 3L, 0L, ""},
|
||||
new Object[]{"a", 2L, 0L, "a"},
|
||||
new Object[]{"abc", 1L, 0L, "abc"},
|
||||
new Object[]{NULL_STRING, 6L, 1L, "ALL"}
|
||||
);
|
||||
}
|
||||
testQuery(
|
||||
"SELECT dim2, SUM(cnt), GROUPING(dim2), \n"
|
||||
+ "CASE WHEN GROUPING(dim2) = 1 THEN 'ALL' ELSE 'INDIVIDUAL' END\n"
|
||||
+ "CASE WHEN GROUPING(dim2) = 1 THEN 'ALL' ELSE dim2 END\n"
|
||||
+ "FROM druid.foo\n"
|
||||
+ "GROUP BY GROUPING SETS ( (dim2), () )",
|
||||
ImmutableList.of(
|
||||
|
|
@ -12200,7 +12200,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
|
|||
)
|
||||
.setPostAggregatorSpecs(Collections.singletonList(new ExpressionPostAggregator(
|
||||
"p0",
|
||||
"case_searched((\"a1\" == 1),'ALL','INDIVIDUAL')",
|
||||
"case_searched((\"a1\" == 1),'ALL',\"d0\")",
|
||||
null,
|
||||
ExprMacroTable.nil()
|
||||
)))
|
||||
|
|
|
|||
Loading…
Reference in New Issue