From 339876b69d5aa9aed67e6e67c9f13f9cbb32c217 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 7 May 2020 13:48:33 -0700 Subject: [PATCH] fill out missing test coverage for druid-stats, druid-momentsketch, druid-tdigestsketch postaggs (#9740) * postagg test coverage for druid-stats, druid-momentsketch, druid-tdigestsketch and fixes * style fixes * fix comparator for TDigestQuantilePostAggregator --- extensions-contrib/momentsketch/pom.xml | 5 ++ .../MomentSketchMaxPostAggregator.java | 6 +- .../MomentSketchMinPostAggregator.java | 6 +- .../MomentSketchMaxPostAggregatorTest.java | 68 ++++++++++++++ .../MomentSketchMinPostAggregatorTest.java | 68 ++++++++++++++ ...omentSketchQuantilePostAggregatorTest.java | 83 +++++++++++++++++ extensions-contrib/tdigestsketch/pom.xml | 5 ++ ...TDigestSketchToQuantilePostAggregator.java | 7 +- ...estSketchToQuantilePostAggregatorTest.java | 73 +++++++++++++++ ...stSketchToQuantilesPostAggregatorTest.java | 90 +++++++++++++++++++ extensions-core/stats/pom.xml | 5 ++ .../PvaluefromZscorePostAggregator.java | 10 ++- .../teststats/ZtestPostAggregator.java | 15 +++- .../PvaluefromZscorePostAggregatorTest.java | 59 +++++++++--- .../teststats/ZtestPostAggregatorTest.java | 86 ++++++++++++------ .../StandardDeviationPostAggregatorTest.java | 67 ++++++++++++++ .../variance/VarianceGroupByQueryTest.java | 82 +++++++++++++++++ 17 files changed, 678 insertions(+), 57 deletions(-) create mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregatorTest.java create mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregatorTest.java create mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregatorTest.java create mode 100644 extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregatorTest.java create mode 100644 extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregatorTest.java create mode 100644 extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregatorTest.java diff --git a/extensions-contrib/momentsketch/pom.xml b/extensions-contrib/momentsketch/pom.xml index 2980d6fb37b..626d14b352e 100644 --- a/extensions-contrib/momentsketch/pom.xml +++ b/extensions-contrib/momentsketch/pom.xml @@ -94,6 +94,11 @@ easymock test + + nl.jqno.equalsverifier + equalsverifier + test + org.apache.druid druid-core diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java index 38755a6b702..b7fe1f81270 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java @@ -22,7 +22,7 @@ package org.apache.druid.query.aggregation.momentsketch.aggregator; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.IAE; +import com.google.common.primitives.Doubles; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper; @@ -71,9 +71,9 @@ public class MomentSketchMaxPostAggregator implements PostAggregator } @Override - public Comparator getComparator() + public Comparator getComparator() { - throw new IAE("Comparing arrays of quantiles is not supported"); + return Doubles::compare; } @Override diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java index b244243b505..2c454405768 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java @@ -22,7 +22,7 @@ package org.apache.druid.query.aggregation.momentsketch.aggregator; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.IAE; +import com.google.common.primitives.Doubles; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper; @@ -70,9 +70,9 @@ public class MomentSketchMinPostAggregator implements PostAggregator } @Override - public Comparator getComparator() + public Comparator getComparator() { - throw new IAE("Comparing arrays of quantiles is not supported"); + return Doubles::compare; } @Override diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregatorTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregatorTest.java new file mode 100644 index 00000000000..83e1aab0d4f --- /dev/null +++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregatorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.momentsketch.aggregator; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.junit.Assert; +import org.junit.Test; + +public class MomentSketchMaxPostAggregatorTest +{ + @Test + public void testSerde() throws Exception + { + MomentSketchMaxPostAggregator there = + new MomentSketchMaxPostAggregator("post", new ConstantPostAggregator("", 100)); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + MomentSketchMaxPostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + MomentSketchMaxPostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = + new MomentSketchMaxPostAggregator("post", new ConstantPostAggregator("", 100)); + + Assert.assertEquals( + "MomentSketchMaxPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}}", + postAgg.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(MomentSketchMaxPostAggregator.class) + .withNonnullFields("name", "field") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregatorTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregatorTest.java new file mode 100644 index 00000000000..6613d5f2aef --- /dev/null +++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregatorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.momentsketch.aggregator; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.junit.Assert; +import org.junit.Test; + +public class MomentSketchMinPostAggregatorTest +{ + @Test + public void testSerde() throws Exception + { + MomentSketchMinPostAggregator there = + new MomentSketchMinPostAggregator("post", new ConstantPostAggregator("", 100)); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + MomentSketchMinPostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + MomentSketchMinPostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = + new MomentSketchMinPostAggregator("post", new ConstantPostAggregator("", 100)); + + Assert.assertEquals( + "MomentSketchMinPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}}", + postAgg.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(MomentSketchMinPostAggregator.class) + .withNonnullFields("name", "field") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregatorTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregatorTest.java new file mode 100644 index 00000000000..bc2179d55d5 --- /dev/null +++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregatorTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.momentsketch.aggregator; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class MomentSketchQuantilePostAggregatorTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testSerde() throws Exception + { + MomentSketchQuantilePostAggregator there = + new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75}); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + MomentSketchQuantilePostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + MomentSketchQuantilePostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = + new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75}); + + Assert.assertEquals( + "MomentSketchQuantilePostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fractions=[0.25, 0.75]}", + postAgg.toString() + ); + } + + @Test + public void testComparator() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("Comparing arrays of quantiles is not supported"); + PostAggregator postAgg = + new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75}); + postAgg.getComparator(); + } + @Test + public void testEquals() + { + EqualsVerifier.forClass(MomentSketchQuantilePostAggregator.class) + .withNonnullFields("name", "field", "fractions") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-contrib/tdigestsketch/pom.xml b/extensions-contrib/tdigestsketch/pom.xml index e43061a5155..4cd72e532e0 100644 --- a/extensions-contrib/tdigestsketch/pom.xml +++ b/extensions-contrib/tdigestsketch/pom.xml @@ -147,6 +147,11 @@ easymock test + + nl.jqno.equalsverifier + equalsverifier + test + org.apache.druid druid-core diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java index d54b90b4ac8..e41cb3ac3b8 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java @@ -22,8 +22,8 @@ package org.apache.druid.query.aggregation.tdigestsketch; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.primitives.Doubles; import com.tdunning.math.stats.MergingDigest; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; @@ -40,7 +40,6 @@ import java.util.Set; */ public class TDigestSketchToQuantilePostAggregator implements PostAggregator { - private final String name; private final PostAggregator field; @@ -87,9 +86,9 @@ public class TDigestSketchToQuantilePostAggregator implements PostAggregator } @Override - public Comparator getComparator() + public Comparator getComparator() { - throw new IAE("Comparing arrays of quantiles is not supported"); + return Doubles::compare; } @Override diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregatorTest.java new file mode 100644 index 00000000000..db85a4cbe29 --- /dev/null +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregatorTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.tdigestsketch; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class TDigestSketchToQuantilePostAggregatorTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testSerde() throws Exception + { + TDigestSketchToQuantilePostAggregator there = + new TDigestSketchToQuantilePostAggregator("post", new ConstantPostAggregator("", 100), 0.5); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + TDigestSketchToQuantilePostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + TDigestSketchToQuantilePostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = + new TDigestSketchToQuantilePostAggregator("post", new ConstantPostAggregator("", 100), 0.5); + + Assert.assertEquals( + "TDigestSketchToQuantilePostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fraction=0.5}", + postAgg.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(TDigestSketchToQuantilePostAggregator.class) + .withNonnullFields("name", "field", "fraction") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregatorTest.java new file mode 100644 index 00000000000..7660a3b1012 --- /dev/null +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregatorTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.tdigestsketch; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class TDigestSketchToQuantilesPostAggregatorTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testSerde() throws Exception + { + TDigestSketchToQuantilesPostAggregator there = + new TDigestSketchToQuantilesPostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75}); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + TDigestSketchToQuantilesPostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + TDigestSketchToQuantilesPostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = new TDigestSketchToQuantilesPostAggregator( + "post", + new ConstantPostAggregator("", 100), + new double[]{0.25, 0.75} + ); + + Assert.assertEquals( + "TDigestSketchToQuantilesPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fractions=[0.25, 0.75]}", + postAgg.toString() + ); + } + + @Test + public void testComparator() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("Comparing arrays of quantiles is not supported"); + PostAggregator postAgg = new TDigestSketchToQuantilesPostAggregator( + "post", + new ConstantPostAggregator("", 100), + new double[]{0.25, 0.75} + ); + postAgg.getComparator(); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(TDigestSketchToQuantilesPostAggregator.class) + .withNonnullFields("name", "field", "fractions") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index b71bdf5ad58..9123c3fa309 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -141,6 +141,11 @@ easymock test + + nl.jqno.equalsverifier + equalsverifier + test + diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java index cf9ff4a9e49..7810077d263 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java @@ -79,9 +79,11 @@ public class PvaluefromZscorePostAggregator implements PostAggregator @Override public Object compute(Map combinedAggregators) { - - double zScoreValue = ((Number) zScore.compute(combinedAggregators)) - .doubleValue(); + Object result = zScore.compute(combinedAggregators); + if (!(result instanceof Number)) { + return null; + } + double zScoreValue = ((Number) result).doubleValue(); zScoreValue = Math.abs(zScoreValue); return 2 * (1 - cumulativeProbability(zScoreValue)); @@ -116,7 +118,7 @@ public class PvaluefromZscorePostAggregator implements PostAggregator } @JsonProperty - public PostAggregator getZscore() + public PostAggregator getzScore() { return zScore; } diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java index 53caa36d04c..bbe2f37c8a8 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java @@ -99,11 +99,18 @@ public class ZtestPostAggregator implements PostAggregator @Override public Object compute(Map combinedAggregators) { + Object sc1 = successCount1.compute(combinedAggregators); + Object ss1 = sample1Size.compute(combinedAggregators); + Object sc2 = successCount2.compute(combinedAggregators); + Object ss2 = sample2Size.compute(combinedAggregators); + if (!(sc1 instanceof Number) || !(sc2 instanceof Number) || !(ss1 instanceof Number) || !(ss2 instanceof Number)) { + return null; + } return zScoreTwoSamples( - ((Number) successCount1.compute(combinedAggregators)).doubleValue(), - ((Number) sample1Size.compute(combinedAggregators)).doubleValue(), - ((Number) successCount2.compute(combinedAggregators)).doubleValue(), - ((Number) sample2Size.compute(combinedAggregators)).doubleValue() + ((Number) sc1).doubleValue(), + ((Number) ss1).doubleValue(), + ((Number) sc2).doubleValue(), + ((Number) ss2).doubleValue() ); } diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregatorTest.java index 1e46f2bcf06..5b96914e0bd 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregatorTest.java @@ -20,32 +20,52 @@ package org.apache.druid.query.aggregation.teststats; import com.google.common.collect.ImmutableMap; +import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.HashMap; +import java.util.Map; public class PvaluefromZscorePostAggregatorTest { PvaluefromZscorePostAggregator pvaluefromZscorePostAggregator; - ConstantPostAggregator zscore; + FieldAccessPostAggregator zscore; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setup() + { + zscore = new FieldAccessPostAggregator("f1", "zscore"); + pvaluefromZscorePostAggregator = new PvaluefromZscorePostAggregator("pvalue", zscore); + } @Test public void testPvaluefromZscorePostAggregator() { - zscore = new ConstantPostAggregator("zscore", -1783.8762354220219); - - pvaluefromZscorePostAggregator = new PvaluefromZscorePostAggregator("pvalue", zscore); - - double pvalue = ((Number) pvaluefromZscorePostAggregator.compute(ImmutableMap.of( - "zscore", - -1783.8762354220219 - ))).doubleValue(); + Map row = ImmutableMap.of("zscore", -1783.8762354220219); + double pvalue = ((Number) pvaluefromZscorePostAggregator.compute(row)).doubleValue(); /* Assert P-value is positive and very small */ Assert.assertTrue(pvalue >= 0 && pvalue < 0.00001); } + @Test + public void testPvaluefromNullZscorePostAggregator() + { + Map row = new HashMap<>(); + row.put("zscore", null); + Object result = pvaluefromZscorePostAggregator.compute(row); + Assert.assertNull(result); + } + @Test public void testSerde() throws Exception { @@ -56,6 +76,25 @@ public class PvaluefromZscorePostAggregatorTest ); Assert.assertEquals(pvaluefromZscorePostAggregator, postAggregator1); + Assert.assertArrayEquals(pvaluefromZscorePostAggregator.getCacheKey(), postAggregator1.getCacheKey()); + Assert.assertEquals(pvaluefromZscorePostAggregator.getDependentFields(), postAggregator1.getDependentFields()); } + @Test + public void testToString() + { + Assert.assertEquals( + "PvaluefromZscorePostAggregator{name='pvalue', zScore=FieldAccessPostAggregator{name='f1', fieldName='zscore'}}", + pvaluefromZscorePostAggregator.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(PvaluefromZscorePostAggregator.class) + .withNonnullFields("name", "zScore") + .usingGetClass() + .verify(); + } } diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregatorTest.java index db98596fe64..c3073a422bb 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregatorTest.java @@ -19,43 +19,31 @@ package org.apache.druid.query.aggregation.teststats; -import com.google.common.collect.Lists; +import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.query.aggregation.PostAggregator; -import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.HashMap; -import java.util.List; import java.util.Map; public class ZtestPostAggregatorTest { + FieldAccessPostAggregator successCount1; + FieldAccessPostAggregator sample1Size; + FieldAccessPostAggregator successCount2; + FieldAccessPostAggregator sample2Size; ZtestPostAggregator ztestPostAggregator; - @Test - public void testZtestPostAggregator() + @Before + public void setup() { - ConstantPostAggregator successCount1, sample1Size, successCount2, sample2Size; - - successCount1 = new ConstantPostAggregator("successCountPopulation1", 39244); - sample1Size = new ConstantPostAggregator("sampleSizePopulation1", 394298); - successCount2 = new ConstantPostAggregator("successCountPopulation2", 8991275); - sample2Size = new ConstantPostAggregator("sampleSizePopulation2", 9385573); - - List postAggregatorList; - postAggregatorList = Lists.newArrayList( - successCount1, - sample1Size, - successCount2, - sample2Size - ); - - Map metricValues = new HashMap<>(); - for (PostAggregator pa : postAggregatorList) { - metricValues.put(pa.getName(), ((ConstantPostAggregator) pa).getConstantValue()); - } + successCount1 = new FieldAccessPostAggregator("sc1", "successCountPopulation1"); + sample1Size = new FieldAccessPostAggregator("ss1", "sampleSizePopulation1"); + successCount2 = new FieldAccessPostAggregator("sc2", "successCountPopulation2"); + sample2Size = new FieldAccessPostAggregator("ss2", "sampleSizePopulation2"); ztestPostAggregator = new ZtestPostAggregator( "zscore", @@ -64,14 +52,34 @@ public class ZtestPostAggregatorTest successCount2, sample2Size ); + } + + @Test + public void testZtestPostAggregator() + { + Map metricValues = new HashMap<>(); + + Object result = ztestPostAggregator.compute(metricValues); + Assert.assertNull(result); + + metricValues.put("successCountPopulation1", 39244); + result = ztestPostAggregator.compute(metricValues); + Assert.assertNull(result); + + metricValues.put("sampleSizePopulation1", 394298); + result = ztestPostAggregator.compute(metricValues); + Assert.assertNull(result); + + metricValues.put("successCountPopulation2", 8991275); + result = ztestPostAggregator.compute(metricValues); + metricValues.put("sampleSizePopulation2", 9385573); + Assert.assertNull(result); double zscore = ((Number) ztestPostAggregator.compute(metricValues)).doubleValue(); - - Assert.assertEquals(-1783.8762354220219, - zscore, 0.0001 - ); + Assert.assertEquals(-1783.8762354220219, zscore, 0.0001); } + @Test public void testSerde() throws Exception { @@ -83,5 +91,25 @@ public class ZtestPostAggregatorTest ); Assert.assertEquals(ztestPostAggregator, postAggregator1); + Assert.assertArrayEquals(ztestPostAggregator.getCacheKey(), postAggregator1.getCacheKey()); + Assert.assertEquals(ztestPostAggregator.getDependentFields(), postAggregator1.getDependentFields()); + } + + @Test + public void testToString() + { + Assert.assertEquals( + "ZtestPostAggregator{name='zscore', successCount1='FieldAccessPostAggregator{name='sc1', fieldName='successCountPopulation1'}', sample1Size='FieldAccessPostAggregator{name='ss1', fieldName='sampleSizePopulation1'}', successCount2='FieldAccessPostAggregator{name='sc2', fieldName='successCountPopulation2'}', sample2size='FieldAccessPostAggregator{name='ss2', fieldName='sampleSizePopulation2'}}", + ztestPostAggregator.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(ZtestPostAggregator.class) + .withNonnullFields("name", "successCount1", "sample1Size", "successCount2", "sample2Size") + .usingGetClass() + .verify(); } } diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregatorTest.java new file mode 100644 index 00000000000..c0768bb88ed --- /dev/null +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregatorTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.variance; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.query.aggregation.PostAggregator; +import org.junit.Assert; +import org.junit.Test; + +public class StandardDeviationPostAggregatorTest +{ + @Test + public void testSerde() throws Exception + { + StandardDeviationPostAggregator there = + new StandardDeviationPostAggregator("post", "test_field", "population"); + + DefaultObjectMapper mapper = new DefaultObjectMapper(); + StandardDeviationPostAggregator andBackAgain = mapper.readValue( + mapper.writeValueAsString(there), + StandardDeviationPostAggregator.class + ); + + Assert.assertEquals(there, andBackAgain); + Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey()); + Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields()); + } + + @Test + public void testToString() + { + PostAggregator postAgg = + new StandardDeviationPostAggregator("post", "test_field", "population"); + + Assert.assertEquals( + "StandardDeviationPostAggregator{name='post', fieldName='test_field', estimator='population', isVariancePop=true}", + postAgg.toString() + ); + } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(StandardDeviationPostAggregator.class) + .withNonnullFields("name", "fieldName") + .usingGetClass() + .verify(); + } +} diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java index 9a04aee18b8..7755f32f9d7 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceGroupByQueryTest.java @@ -26,6 +26,10 @@ import org.apache.druid.java.util.common.granularity.PeriodGranularity; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryRunnerTestHelper; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.teststats.PvaluefromZscorePostAggregator; +import org.apache.druid.query.aggregation.teststats.ZtestPostAggregator; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.groupby.GroupByQueryConfig; @@ -235,4 +239,82 @@ public class VarianceGroupByQueryTest extends InitializedNullHandlingTest results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, "limitSpec"); } + + @Test + public void testGroupByZtestPostAgg() + { + // test postaggs from 'teststats' package in here since we've already gone to the trouble of setting up the test + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setDimensions(new DefaultDimensionSpec("quality", "alias")) + .setAggregatorSpecs( + QueryRunnerTestHelper.ROWS_COUNT, + VarianceTestHelper.INDEX_VARIANCE_AGGR, + new LongSumAggregatorFactory("idx", "index") + ) + .setPostAggregatorSpecs( + ImmutableList.of( + VarianceTestHelper.STD_DEV_OF_INDEX_POST_AGGR, + // these inputs are totally nonsensical, i just want the code path to be executed + new ZtestPostAggregator( + "ztest", + new FieldAccessPostAggregator("f1", "idx"), + new ConstantPostAggregator("f2", 100000L), + new FieldAccessPostAggregator("f3", "index_stddev"), + new ConstantPostAggregator("f2", 100000L) + ) + ) + ) + .setLimitSpec(new DefaultLimitSpec(OrderByColumnSpec.descending("ztest"), 1)) + .setGranularity(QueryRunnerTestHelper.DAY_GRAN) + .build(); + + VarianceTestHelper.RowBuilder builder = + new VarianceTestHelper.RowBuilder(new String[]{"alias", "rows", "idx", "index_stddev", "index_var", "ztest"}); + + List expectedResults = builder + .add("2011-04-01", "premium", 3L, 2900.0, 726.632270328514, 527994.4562827706, 36.54266309285626) + .build(query); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy"); + } + + @Test + public void testGroupByTestPvalueZscorePostAgg() + { + // test postaggs from 'teststats' package in here since we've already gone to the trouble of setting up the test + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setDimensions(new DefaultDimensionSpec("quality", "alias")) + .setAggregatorSpecs( + QueryRunnerTestHelper.ROWS_COUNT, + VarianceTestHelper.INDEX_VARIANCE_AGGR, + new LongSumAggregatorFactory("idx", "index") + ) + .setPostAggregatorSpecs( + ImmutableList.of( + VarianceTestHelper.STD_DEV_OF_INDEX_POST_AGGR, + // nonsensical inputs + new PvaluefromZscorePostAggregator("pvalueZscore", new FieldAccessPostAggregator("f1", "index_stddev")) + ) + ) + .setLimitSpec(new DefaultLimitSpec(OrderByColumnSpec.descending("pvalueZscore"), 1)) + .setGranularity(QueryRunnerTestHelper.DAY_GRAN) + .build(); + + VarianceTestHelper.RowBuilder builder = + new VarianceTestHelper.RowBuilder(new String[]{"alias", "rows", "idx", "index_stddev", "index_var", "pvalueZscore"}); + + List expectedResults = builder + .add("2011-04-01", "automotive", 1L, 135.0, 0.0, 0.0, 1.0) + .build(query); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy"); + } }