fill out missing test coverage for druid-stats, druid-momentsketch, druid-tdigestsketch postaggs (#9740)

* postagg test coverage for druid-stats, druid-momentsketch, druid-tdigestsketch and fixes

* style fixes

* fix comparator for TDigestQuantilePostAggregator
This commit is contained in:
Clint Wylie 2020-05-07 13:48:33 -07:00 committed by GitHub
parent 267a6cc175
commit 339876b69d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 678 additions and 57 deletions

View File

@ -94,6 +94,11 @@
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId>

View File

@ -22,7 +22,7 @@ package org.apache.druid.query.aggregation.momentsketch.aggregator;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.IAE;
import com.google.common.primitives.Doubles;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper;
@ -71,9 +71,9 @@ public class MomentSketchMaxPostAggregator implements PostAggregator
}
@Override
public Comparator<double[]> getComparator()
public Comparator<Double> getComparator()
{
throw new IAE("Comparing arrays of quantiles is not supported");
return Doubles::compare;
}
@Override

View File

@ -22,7 +22,7 @@ package org.apache.druid.query.aggregation.momentsketch.aggregator;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.IAE;
import com.google.common.primitives.Doubles;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper;
@ -70,9 +70,9 @@ public class MomentSketchMinPostAggregator implements PostAggregator
}
@Override
public Comparator<double[]> getComparator()
public Comparator<Double> getComparator()
{
throw new IAE("Comparing arrays of quantiles is not supported");
return Doubles::compare;
}
@Override

View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.momentsketch.aggregator;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.junit.Assert;
import org.junit.Test;
public class MomentSketchMaxPostAggregatorTest
{
@Test
public void testSerde() throws Exception
{
MomentSketchMaxPostAggregator there =
new MomentSketchMaxPostAggregator("post", new ConstantPostAggregator("", 100));
DefaultObjectMapper mapper = new DefaultObjectMapper();
MomentSketchMaxPostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
MomentSketchMaxPostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg =
new MomentSketchMaxPostAggregator("post", new ConstantPostAggregator("", 100));
Assert.assertEquals(
"MomentSketchMaxPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}}",
postAgg.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(MomentSketchMaxPostAggregator.class)
.withNonnullFields("name", "field")
.usingGetClass()
.verify();
}
}

View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.momentsketch.aggregator;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.junit.Assert;
import org.junit.Test;
public class MomentSketchMinPostAggregatorTest
{
@Test
public void testSerde() throws Exception
{
MomentSketchMinPostAggregator there =
new MomentSketchMinPostAggregator("post", new ConstantPostAggregator("", 100));
DefaultObjectMapper mapper = new DefaultObjectMapper();
MomentSketchMinPostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
MomentSketchMinPostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg =
new MomentSketchMinPostAggregator("post", new ConstantPostAggregator("", 100));
Assert.assertEquals(
"MomentSketchMinPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}}",
postAgg.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(MomentSketchMinPostAggregator.class)
.withNonnullFields("name", "field")
.usingGetClass()
.verify();
}
}

View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.momentsketch.aggregator;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class MomentSketchQuantilePostAggregatorTest
{
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testSerde() throws Exception
{
MomentSketchQuantilePostAggregator there =
new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75});
DefaultObjectMapper mapper = new DefaultObjectMapper();
MomentSketchQuantilePostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
MomentSketchQuantilePostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg =
new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75});
Assert.assertEquals(
"MomentSketchQuantilePostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fractions=[0.25, 0.75]}",
postAgg.toString()
);
}
@Test
public void testComparator()
{
expectedException.expect(IAE.class);
expectedException.expectMessage("Comparing arrays of quantiles is not supported");
PostAggregator postAgg =
new MomentSketchQuantilePostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75});
postAgg.getComparator();
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(MomentSketchQuantilePostAggregator.class)
.withNonnullFields("name", "field", "fractions")
.usingGetClass()
.verify();
}
}

View File

@ -147,6 +147,11 @@
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId>

View File

@ -22,8 +22,8 @@ package org.apache.druid.query.aggregation.tdigestsketch;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
import com.tdunning.math.stats.MergingDigest;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.PostAggregatorIds;
@ -40,7 +40,6 @@ import java.util.Set;
*/
public class TDigestSketchToQuantilePostAggregator implements PostAggregator
{
private final String name;
private final PostAggregator field;
@ -87,9 +86,9 @@ public class TDigestSketchToQuantilePostAggregator implements PostAggregator
}
@Override
public Comparator<double[]> getComparator()
public Comparator<Double> getComparator()
{
throw new IAE("Comparing arrays of quantiles is not supported");
return Doubles::compare;
}
@Override

View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.tdigestsketch;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class TDigestSketchToQuantilePostAggregatorTest
{
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testSerde() throws Exception
{
TDigestSketchToQuantilePostAggregator there =
new TDigestSketchToQuantilePostAggregator("post", new ConstantPostAggregator("", 100), 0.5);
DefaultObjectMapper mapper = new DefaultObjectMapper();
TDigestSketchToQuantilePostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
TDigestSketchToQuantilePostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg =
new TDigestSketchToQuantilePostAggregator("post", new ConstantPostAggregator("", 100), 0.5);
Assert.assertEquals(
"TDigestSketchToQuantilePostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fraction=0.5}",
postAgg.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(TDigestSketchToQuantilePostAggregator.class)
.withNonnullFields("name", "field", "fraction")
.usingGetClass()
.verify();
}
}

View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.tdigestsketch;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class TDigestSketchToQuantilesPostAggregatorTest
{
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testSerde() throws Exception
{
TDigestSketchToQuantilesPostAggregator there =
new TDigestSketchToQuantilesPostAggregator("post", new ConstantPostAggregator("", 100), new double[]{0.25, 0.75});
DefaultObjectMapper mapper = new DefaultObjectMapper();
TDigestSketchToQuantilesPostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
TDigestSketchToQuantilesPostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg = new TDigestSketchToQuantilesPostAggregator(
"post",
new ConstantPostAggregator("", 100),
new double[]{0.25, 0.75}
);
Assert.assertEquals(
"TDigestSketchToQuantilesPostAggregator{name='post', field=ConstantPostAggregator{name='', constantValue=100}, fractions=[0.25, 0.75]}",
postAgg.toString()
);
}
@Test
public void testComparator()
{
expectedException.expect(IAE.class);
expectedException.expectMessage("Comparing arrays of quantiles is not supported");
PostAggregator postAgg = new TDigestSketchToQuantilesPostAggregator(
"post",
new ConstantPostAggregator("", 100),
new double[]{0.25, 0.75}
);
postAgg.getComparator();
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(TDigestSketchToQuantilesPostAggregator.class)
.withNonnullFields("name", "field", "fractions")
.usingGetClass()
.verify();
}
}

View File

@ -141,6 +141,11 @@
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -79,9 +79,11 @@ public class PvaluefromZscorePostAggregator implements PostAggregator
@Override
public Object compute(Map<String, Object> combinedAggregators)
{
double zScoreValue = ((Number) zScore.compute(combinedAggregators))
.doubleValue();
Object result = zScore.compute(combinedAggregators);
if (!(result instanceof Number)) {
return null;
}
double zScoreValue = ((Number) result).doubleValue();
zScoreValue = Math.abs(zScoreValue);
return 2 * (1 - cumulativeProbability(zScoreValue));
@ -116,7 +118,7 @@ public class PvaluefromZscorePostAggregator implements PostAggregator
}
@JsonProperty
public PostAggregator getZscore()
public PostAggregator getzScore()
{
return zScore;
}

View File

@ -99,11 +99,18 @@ public class ZtestPostAggregator implements PostAggregator
@Override
public Object compute(Map<String, Object> combinedAggregators)
{
Object sc1 = successCount1.compute(combinedAggregators);
Object ss1 = sample1Size.compute(combinedAggregators);
Object sc2 = successCount2.compute(combinedAggregators);
Object ss2 = sample2Size.compute(combinedAggregators);
if (!(sc1 instanceof Number) || !(sc2 instanceof Number) || !(ss1 instanceof Number) || !(ss2 instanceof Number)) {
return null;
}
return zScoreTwoSamples(
((Number) successCount1.compute(combinedAggregators)).doubleValue(),
((Number) sample1Size.compute(combinedAggregators)).doubleValue(),
((Number) successCount2.compute(combinedAggregators)).doubleValue(),
((Number) sample2Size.compute(combinedAggregators)).doubleValue()
((Number) sc1).doubleValue(),
((Number) ss1).doubleValue(),
((Number) sc2).doubleValue(),
((Number) ss2).doubleValue()
);
}

View File

@ -20,32 +20,52 @@
package org.apache.druid.query.aggregation.teststats;
import com.google.common.collect.ImmutableMap;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.util.HashMap;
import java.util.Map;
public class PvaluefromZscorePostAggregatorTest
{
PvaluefromZscorePostAggregator pvaluefromZscorePostAggregator;
ConstantPostAggregator zscore;
FieldAccessPostAggregator zscore;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setup()
{
zscore = new FieldAccessPostAggregator("f1", "zscore");
pvaluefromZscorePostAggregator = new PvaluefromZscorePostAggregator("pvalue", zscore);
}
@Test
public void testPvaluefromZscorePostAggregator()
{
zscore = new ConstantPostAggregator("zscore", -1783.8762354220219);
pvaluefromZscorePostAggregator = new PvaluefromZscorePostAggregator("pvalue", zscore);
double pvalue = ((Number) pvaluefromZscorePostAggregator.compute(ImmutableMap.of(
"zscore",
-1783.8762354220219
))).doubleValue();
Map<String, Object> row = ImmutableMap.of("zscore", -1783.8762354220219);
double pvalue = ((Number) pvaluefromZscorePostAggregator.compute(row)).doubleValue();
/* Assert P-value is positive and very small */
Assert.assertTrue(pvalue >= 0 && pvalue < 0.00001);
}
@Test
public void testPvaluefromNullZscorePostAggregator()
{
Map<String, Object> row = new HashMap<>();
row.put("zscore", null);
Object result = pvaluefromZscorePostAggregator.compute(row);
Assert.assertNull(result);
}
@Test
public void testSerde() throws Exception
{
@ -56,6 +76,25 @@ public class PvaluefromZscorePostAggregatorTest
);
Assert.assertEquals(pvaluefromZscorePostAggregator, postAggregator1);
Assert.assertArrayEquals(pvaluefromZscorePostAggregator.getCacheKey(), postAggregator1.getCacheKey());
Assert.assertEquals(pvaluefromZscorePostAggregator.getDependentFields(), postAggregator1.getDependentFields());
}
@Test
public void testToString()
{
Assert.assertEquals(
"PvaluefromZscorePostAggregator{name='pvalue', zScore=FieldAccessPostAggregator{name='f1', fieldName='zscore'}}",
pvaluefromZscorePostAggregator.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(PvaluefromZscorePostAggregator.class)
.withNonnullFields("name", "zScore")
.usingGetClass()
.verify();
}
}

View File

@ -19,43 +19,31 @@
package org.apache.druid.query.aggregation.teststats;
import com.google.common.collect.Lists;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ZtestPostAggregatorTest
{
FieldAccessPostAggregator successCount1;
FieldAccessPostAggregator sample1Size;
FieldAccessPostAggregator successCount2;
FieldAccessPostAggregator sample2Size;
ZtestPostAggregator ztestPostAggregator;
@Test
public void testZtestPostAggregator()
@Before
public void setup()
{
ConstantPostAggregator successCount1, sample1Size, successCount2, sample2Size;
successCount1 = new ConstantPostAggregator("successCountPopulation1", 39244);
sample1Size = new ConstantPostAggregator("sampleSizePopulation1", 394298);
successCount2 = new ConstantPostAggregator("successCountPopulation2", 8991275);
sample2Size = new ConstantPostAggregator("sampleSizePopulation2", 9385573);
List<PostAggregator> postAggregatorList;
postAggregatorList = Lists.newArrayList(
successCount1,
sample1Size,
successCount2,
sample2Size
);
Map<String, Object> metricValues = new HashMap<>();
for (PostAggregator pa : postAggregatorList) {
metricValues.put(pa.getName(), ((ConstantPostAggregator) pa).getConstantValue());
}
successCount1 = new FieldAccessPostAggregator("sc1", "successCountPopulation1");
sample1Size = new FieldAccessPostAggregator("ss1", "sampleSizePopulation1");
successCount2 = new FieldAccessPostAggregator("sc2", "successCountPopulation2");
sample2Size = new FieldAccessPostAggregator("ss2", "sampleSizePopulation2");
ztestPostAggregator = new ZtestPostAggregator(
"zscore",
@ -64,14 +52,34 @@ public class ZtestPostAggregatorTest
successCount2,
sample2Size
);
}
@Test
public void testZtestPostAggregator()
{
Map<String, Object> metricValues = new HashMap<>();
Object result = ztestPostAggregator.compute(metricValues);
Assert.assertNull(result);
metricValues.put("successCountPopulation1", 39244);
result = ztestPostAggregator.compute(metricValues);
Assert.assertNull(result);
metricValues.put("sampleSizePopulation1", 394298);
result = ztestPostAggregator.compute(metricValues);
Assert.assertNull(result);
metricValues.put("successCountPopulation2", 8991275);
result = ztestPostAggregator.compute(metricValues);
metricValues.put("sampleSizePopulation2", 9385573);
Assert.assertNull(result);
double zscore = ((Number) ztestPostAggregator.compute(metricValues)).doubleValue();
Assert.assertEquals(-1783.8762354220219,
zscore, 0.0001
);
Assert.assertEquals(-1783.8762354220219, zscore, 0.0001);
}
@Test
public void testSerde() throws Exception
{
@ -83,5 +91,25 @@ public class ZtestPostAggregatorTest
);
Assert.assertEquals(ztestPostAggregator, postAggregator1);
Assert.assertArrayEquals(ztestPostAggregator.getCacheKey(), postAggregator1.getCacheKey());
Assert.assertEquals(ztestPostAggregator.getDependentFields(), postAggregator1.getDependentFields());
}
@Test
public void testToString()
{
Assert.assertEquals(
"ZtestPostAggregator{name='zscore', successCount1='FieldAccessPostAggregator{name='sc1', fieldName='successCountPopulation1'}', sample1Size='FieldAccessPostAggregator{name='ss1', fieldName='sampleSizePopulation1'}', successCount2='FieldAccessPostAggregator{name='sc2', fieldName='successCountPopulation2'}', sample2size='FieldAccessPostAggregator{name='ss2', fieldName='sampleSizePopulation2'}}",
ztestPostAggregator.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(ZtestPostAggregator.class)
.withNonnullFields("name", "successCount1", "sample1Size", "successCount2", "sample2Size")
.usingGetClass()
.verify();
}
}

View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.aggregation.variance;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.aggregation.PostAggregator;
import org.junit.Assert;
import org.junit.Test;
public class StandardDeviationPostAggregatorTest
{
@Test
public void testSerde() throws Exception
{
StandardDeviationPostAggregator there =
new StandardDeviationPostAggregator("post", "test_field", "population");
DefaultObjectMapper mapper = new DefaultObjectMapper();
StandardDeviationPostAggregator andBackAgain = mapper.readValue(
mapper.writeValueAsString(there),
StandardDeviationPostAggregator.class
);
Assert.assertEquals(there, andBackAgain);
Assert.assertArrayEquals(there.getCacheKey(), andBackAgain.getCacheKey());
Assert.assertEquals(there.getDependentFields(), andBackAgain.getDependentFields());
}
@Test
public void testToString()
{
PostAggregator postAgg =
new StandardDeviationPostAggregator("post", "test_field", "population");
Assert.assertEquals(
"StandardDeviationPostAggregator{name='post', fieldName='test_field', estimator='population', isVariancePop=true}",
postAgg.toString()
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(StandardDeviationPostAggregator.class)
.withNonnullFields("name", "fieldName")
.usingGetClass()
.verify();
}
}

View File

@ -26,6 +26,10 @@ import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.query.QueryRunner;
import org.apache.druid.query.QueryRunnerTestHelper;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.aggregation.post.ConstantPostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.aggregation.teststats.PvaluefromZscorePostAggregator;
import org.apache.druid.query.aggregation.teststats.ZtestPostAggregator;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.groupby.GroupByQuery;
import org.apache.druid.query.groupby.GroupByQueryConfig;
@ -235,4 +239,82 @@ public class VarianceGroupByQueryTest extends InitializedNullHandlingTest
results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
TestHelper.assertExpectedObjects(expectedResults, results, "limitSpec");
}
@Test
public void testGroupByZtestPostAgg()
{
// test postaggs from 'teststats' package in here since we've already gone to the trouble of setting up the test
GroupByQuery query = GroupByQuery
.builder()
.setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
.setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
.setDimensions(new DefaultDimensionSpec("quality", "alias"))
.setAggregatorSpecs(
QueryRunnerTestHelper.ROWS_COUNT,
VarianceTestHelper.INDEX_VARIANCE_AGGR,
new LongSumAggregatorFactory("idx", "index")
)
.setPostAggregatorSpecs(
ImmutableList.of(
VarianceTestHelper.STD_DEV_OF_INDEX_POST_AGGR,
// these inputs are totally nonsensical, i just want the code path to be executed
new ZtestPostAggregator(
"ztest",
new FieldAccessPostAggregator("f1", "idx"),
new ConstantPostAggregator("f2", 100000L),
new FieldAccessPostAggregator("f3", "index_stddev"),
new ConstantPostAggregator("f2", 100000L)
)
)
)
.setLimitSpec(new DefaultLimitSpec(OrderByColumnSpec.descending("ztest"), 1))
.setGranularity(QueryRunnerTestHelper.DAY_GRAN)
.build();
VarianceTestHelper.RowBuilder builder =
new VarianceTestHelper.RowBuilder(new String[]{"alias", "rows", "idx", "index_stddev", "index_var", "ztest"});
List<ResultRow> expectedResults = builder
.add("2011-04-01", "premium", 3L, 2900.0, 726.632270328514, 527994.4562827706, 36.54266309285626)
.build(query);
Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
}
@Test
public void testGroupByTestPvalueZscorePostAgg()
{
// test postaggs from 'teststats' package in here since we've already gone to the trouble of setting up the test
GroupByQuery query = GroupByQuery
.builder()
.setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
.setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
.setDimensions(new DefaultDimensionSpec("quality", "alias"))
.setAggregatorSpecs(
QueryRunnerTestHelper.ROWS_COUNT,
VarianceTestHelper.INDEX_VARIANCE_AGGR,
new LongSumAggregatorFactory("idx", "index")
)
.setPostAggregatorSpecs(
ImmutableList.of(
VarianceTestHelper.STD_DEV_OF_INDEX_POST_AGGR,
// nonsensical inputs
new PvaluefromZscorePostAggregator("pvalueZscore", new FieldAccessPostAggregator("f1", "index_stddev"))
)
)
.setLimitSpec(new DefaultLimitSpec(OrderByColumnSpec.descending("pvalueZscore"), 1))
.setGranularity(QueryRunnerTestHelper.DAY_GRAN)
.build();
VarianceTestHelper.RowBuilder builder =
new VarianceTestHelper.RowBuilder(new String[]{"alias", "rows", "idx", "index_stddev", "index_var", "pvalueZscore"});
List<ResultRow> expectedResults = builder
.add("2011-04-01", "automotive", 1L, 135.0, 0.0, 0.0, 1.0)
.build(query);
Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
}
}