honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add unit tests for stats and extended stats aggregations (#23287) 

Add tests for InternalStats, InternalExtendedStats and StatsAggregator/ExtendedStatsAggregator

Relates #22278
This commit is contained in:
Jim Ferenczi 2017-02-21 15:14:54 +01:00 committed by GitHub
parent 0744a00001
commit cc865cbc96
7 changed files with 514 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class InternalStats extends InternalNumericMetricsAggregation.MultiValue implements Stats {
enum Metrics {
@ -198,4 +199,18 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue
protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params params) throws IOException {
return builder;
}
@Override
protected int doHashCode() {
return Objects.hash(count, min, max, sum);
}
@Override
protected boolean doEquals(Object obj) {
InternalStats other = (InternalStats) obj;
return count == other.count &&
min == other.min &&
max == other.max &&
Double.compare(count, other.count) == 0;
}
}

4
core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java
View File

@ -189,8 +189,8 @@ public class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue
@Override
public InternalAggregation buildEmptyAggregation() {
return new InternalExtendedStats(name, 0, 0d, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 0d, sigma, format, pipelineAggregators(),
metaData());
return new InternalExtendedStats(name, 0, 0d, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 0d,
sigma, format, pipelineAggregators(), metaData());
}
@Override

18
core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class InternalExtendedStats extends InternalStats implements ExtendedStats {
enum Metrics {
@ -90,6 +91,10 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat
return super.value(name);
}
public double getSigma() {
return this.sigma;
}
@Override
public double getSumOfSquares() {
return sumOfSqrs;
@ -186,4 +191,17 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat
}
return builder;
}
@Override
protected int doHashCode() {
return Objects.hash(super.doHashCode(), sumOfSqrs, sigma);
}
@Override
protected boolean doEquals(Object obj) {
InternalExtendedStats other = (InternalExtendedStats) obj;
return super.doEquals(obj) &&
Double.compare(sumOfSqrs, other.sumOfSqrs) == 0 &&
Double.compare(sigma, other.sigma) == 0;
}
}

176
core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
import java.io.IOException;
import java.util.function.Consumer;
public class ExtendedStatsAggregatorTests extends AggregatorTestCase {
private static final double TOLERANCE = 1e-5;
public void testEmpty() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
ft.setName("field");
testCase(ft, iw -> {},
stats -> {
assertEquals(0d, stats.getCount(), 0);
assertEquals(0d, stats.getSum(), 0);
assertEquals(Float.NaN, stats.getAvg(), 0);
assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
assertEquals(Double.NaN, stats.getVariance(), 0);
assertEquals(Double.NaN, stats.getStdDeviation(), 0);
assertEquals(0d, stats.getSumOfSquares(), 0);
}
);
}
public void testRandomDoubles() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
ft.setName("field");
final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator();
testCase(ft,
iw -> {
int numDocs = randomIntBetween(10, 50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
int numValues = randomIntBetween(1, 5);
for (int j = 0; j < numValues; j++) {
double value = randomDoubleBetween(-100d, 100d, true);
long valueAsLong = NumericUtils.doubleToSortableLong(value);
doc.add(new SortedNumericDocValuesField("field", valueAsLong));
expected.add(value);
}
iw.addDocument(doc);
}
},
stats -> {
assertEquals(expected.count, stats.getCount(), 0);
assertEquals(expected.sum, stats.getSum(), TOLERANCE);
assertEquals(expected.min, stats.getMin(), 0);
assertEquals(expected.max, stats.getMax(), 0);
assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
assertEquals(expected.sumOfSqrs, stats.getSumOfSquares(), TOLERANCE);
assertEquals(expected.stdDev(), stats.getStdDeviation(), TOLERANCE);
assertEquals(expected.variance(), stats.getVariance(), TOLERANCE);
assertEquals(expected.stdDevBound(ExtendedStats.Bounds.LOWER, stats.getSigma()),
stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), TOLERANCE);
assertEquals(expected.stdDevBound(ExtendedStats.Bounds.UPPER, stats.getSigma()),
stats.getStdDeviationBound(ExtendedStats.Bounds.UPPER), TOLERANCE);
}
);
}
public void testRandomLongs() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
ft.setName("field");
final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator();
testCase(ft,
iw -> {
int numDocs = randomIntBetween(10, 50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
int numValues = randomIntBetween(1, 5);
for (int j = 0; j < numValues; j++) {
long value = randomIntBetween(-100, 100);
doc.add(new SortedNumericDocValuesField("field", value));
expected.add(value);
}
iw.addDocument(doc);
}
},
stats -> {
assertEquals(expected.count, stats.getCount(), 0);
assertEquals(expected.sum, stats.getSum(), TOLERANCE);
assertEquals(expected.min, stats.getMin(), 0);
assertEquals(expected.max, stats.getMax(), 0);
assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
assertEquals(expected.sumOfSqrs, stats.getSumOfSquares(), TOLERANCE);
assertEquals(expected.stdDev(), stats.getStdDeviation(), TOLERANCE);
assertEquals(expected.variance(), stats.getVariance(), TOLERANCE);
assertEquals(expected.stdDevBound(ExtendedStats.Bounds.LOWER, stats.getSigma()),
stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), TOLERANCE);
assertEquals(expected.stdDevBound(ExtendedStats.Bounds.UPPER, stats.getSigma()),
stats.getStdDeviationBound(ExtendedStats.Bounds.UPPER), TOLERANCE);
}
);
}
public void testCase(MappedFieldType ft,
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
Consumer<InternalExtendedStats> verify) throws IOException {
try (Directory directory = newDirectory();
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
buildIndex.accept(indexWriter);
try (IndexReader reader = indexWriter.getReader()) {
IndexSearcher searcher = new IndexSearcher(reader);
ExtendedStatsAggregationBuilder aggBuilder = new ExtendedStatsAggregationBuilder("my_agg")
.field("field")
.sigma(randomDoubleBetween(0, 10, true));
InternalExtendedStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft);
verify.accept(stats);
}
}
}
static class ExtendedSimpleStatsAggregator extends StatsAggregatorTests.SimpleStatsAggregator {
double sumOfSqrs = 0;
void add(double value) {
super.add(value);
sumOfSqrs += (value * value);
}
double stdDev() {
return Math.sqrt(variance());
}
double stdDevBound(ExtendedStats.Bounds bounds, double sigma) {
if (bounds == ExtendedStats.Bounds.UPPER) {
return (sum / count) + (Math.sqrt(variance()) * sigma);
} else {
return (sum / count) - (Math.sqrt(variance()) * sigma);
}
}
double variance() {
return (sumOfSqrs - ((sum * sum) / count)) / count;
}
}
}

85
core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.junit.Before;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class InternalExtendedStatsTests extends InternalAggregationTestCase<InternalExtendedStats> {
private double sigma;
@Before
public void randomSigma() {
this.sigma = randomDoubleBetween(0, 10, true);
}
@Override
protected InternalExtendedStats createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) {
long count = randomIntBetween(1, 50);
double[] minMax = new double[2];
minMax[0] = randomDouble();
minMax[0] = randomDouble();
double sum = randomDoubleBetween(0, 100, true);
return new InternalExtendedStats(name, count, sum, minMax[0], minMax[1],
randomDouble(), sigma, DocValueFormat.RAW,
pipelineAggregators, Collections.emptyMap());
}
@Override
protected void assertReduced(InternalExtendedStats reduced, List<InternalExtendedStats> inputs) {
long expectedCount = 0;
double expectedSum = 0;
double expectedSumOfSquare = 0;
double expectedMin = Double.POSITIVE_INFINITY;
double expectedMax = Double.NEGATIVE_INFINITY;
for (InternalExtendedStats stats : inputs) {
assertEquals(sigma, stats.getSigma(), 0);
expectedCount += stats.getCount();
if (Double.compare(stats.getMin(), expectedMin) < 0) {
expectedMin = stats.getMin();
}
if (Double.compare(stats.getMax(), expectedMax) > 0) {
expectedMax = stats.getMax();
}
expectedSum += stats.getSum();
expectedSumOfSquare += stats.getSumOfSquares();
}
assertEquals(sigma, reduced.getSigma(), 0);
assertEquals(expectedCount, reduced.getCount());
assertEquals(expectedSum, reduced.getSum(), 1e-10);
assertEquals(expectedMin, reduced.getMin(), 0d);
assertEquals(expectedMax, reduced.getMax(), 0d);
assertEquals(expectedSumOfSquare, reduced.getSumOfSquares(), 1e-10);
}
@Override
protected Writeable.Reader<InternalExtendedStats> instanceReader() {
return InternalExtendedStats::new;
}
}

70
core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class InternalStatsTests extends InternalAggregationTestCase<InternalStats> {
@Override
protected InternalStats createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) {
long count = randomIntBetween(1, 50);
double[] minMax = new double[2];
minMax[0] = randomDouble();
minMax[0] = randomDouble();
double sum = randomDoubleBetween(0, 100, true);
return new InternalStats(name, count, sum, minMax[0], minMax[1], DocValueFormat.RAW,
pipelineAggregators, Collections.emptyMap());
}
@Override
protected void assertReduced(InternalStats reduced, List<InternalStats> inputs) {
long expectedCount = 0;
double expectedSum = 0;
double expectedMin = Double.POSITIVE_INFINITY;
double expectedMax = Double.NEGATIVE_INFINITY;
for (InternalStats stats : inputs) {
expectedCount += stats.getCount();
if (Double.compare(stats.getMin(), expectedMin) < 0) {
expectedMin = stats.getMin();
}
if (Double.compare(stats.getMax(), expectedMax) > 0) {
expectedMax = stats.getMax();
}
expectedSum += stats.getSum();
}
assertEquals(expectedCount, reduced.getCount());
assertEquals(expectedSum, reduced.getSum(), 1e-10);
assertEquals(expectedMin, reduced.getMin(), 0d);
assertEquals(expectedMax, reduced.getMax(), 0d);
}
@Override
protected Writeable.Reader<InternalStats> instanceReader() {
return InternalStats::new;
}
}

148
core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import java.io.IOException;
import java.util.function.Consumer;
public class StatsAggregatorTests extends AggregatorTestCase {
static final double TOLERANCE = 1e-10;
public void testEmpty() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
ft.setName("field");
testCase(ft, iw -> {},
stats -> {
assertEquals(0d, stats.getCount(), 0);
assertEquals(0d, stats.getSum(), 0);
assertEquals(Float.NaN, stats.getAvg(), 0);
assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
}
);
}
public void testRandomDoubles() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
ft.setName("field");
final SimpleStatsAggregator expected = new SimpleStatsAggregator();
testCase(ft,
iw -> {
int numDocs = randomIntBetween(10, 50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
int numValues = randomIntBetween(1, 5);
for (int j = 0; j < numValues; j++) {
double value = randomDoubleBetween(-100d, 100d, true);
long valueAsLong = NumericUtils.doubleToSortableLong(value);
doc.add(new SortedNumericDocValuesField("field", valueAsLong));
expected.add(value);
}
iw.addDocument(doc);
}
},
stats -> {
assertEquals(expected.count, stats.getCount(), 0);
assertEquals(expected.sum, stats.getSum(), TOLERANCE);
assertEquals(expected.min, stats.getMin(), 0);
assertEquals(expected.max, stats.getMax(), 0);
assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
}
);
}
public void testRandomLongs() throws IOException {
MappedFieldType ft =
new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
ft.setName("field");
final SimpleStatsAggregator expected = new SimpleStatsAggregator();
testCase(ft,
iw -> {
int numDocs = randomIntBetween(10, 50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
int numValues = randomIntBetween(1, 5);
for (int j = 0; j < numValues; j++) {
long value = randomIntBetween(-100, 100);
doc.add(new SortedNumericDocValuesField("field", value));
expected.add(value);
}
iw.addDocument(doc);
}
},
stats -> {
assertEquals(expected.count, stats.getCount(), 0);
assertEquals(expected.sum, stats.getSum(), TOLERANCE);
assertEquals(expected.min, stats.getMin(), 0);
assertEquals(expected.max, stats.getMax(), 0);
assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
}
);
}
public void testCase(MappedFieldType ft,
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
Consumer<InternalStats> verify) throws IOException {
try (Directory directory = newDirectory();
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
buildIndex.accept(indexWriter);
try (IndexReader reader = indexWriter.getReader()) {
IndexSearcher searcher = new IndexSearcher(reader);
StatsAggregationBuilder aggBuilder = new StatsAggregationBuilder("my_agg").field("field");
InternalStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft);
verify.accept(stats);
}
}
}
static class SimpleStatsAggregator {
long count = 0;
double min = Long.MAX_VALUE;
double max = Long.MIN_VALUE;
double sum = 0;
void add(double value) {
count ++;
if (Double.compare(value, min) < 0) {
min = value;
}
if (Double.compare(value, max) > 0) {
max = value;
}
sum += value;
}
}
}