From 51c74ce5476ed4d8b0f2ed3469fee955ff1e5fd1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 9 May 2017 15:04:28 +0200 Subject: [PATCH] Added unit tests for InternalMatrixStats. Also moved InternalAggregationTestCase to test-framework module in order to make use of it from other modules than core. Relates to #22278 --- ...ternalSingleBucketAggregationTestCase.java | 2 +- .../geogrid/InternalGeoHashGridTests.java | 2 +- .../histogram/InternalDateHistogramTests.java | 4 +- .../histogram/InternalHistogramTests.java | 2 +- .../range/InternalBinaryRangeTests.java | 2 +- .../InternalSignificantTermsTestCase.java | 2 +- .../bucket/terms/InternalTermsTestCase.java | 4 +- .../metrics/InternalExtendedStatsTests.java | 2 +- .../metrics/InternalMaxTests.java | 2 +- .../metrics/InternalStatsTests.java | 2 +- .../metrics/avg/InternalAvgTests.java | 3 +- .../cardinality/InternalCardinalityTests.java | 2 +- .../geobounds/InternalGeoBoundsTests.java | 2 +- .../geocentroid/InternalGeoCentroidTests.java | 2 +- .../metrics/min/InternalMinTests.java | 3 +- .../InternalPercentilesTestCase.java | 2 +- .../hdr/InternalHDRPercentilesRanksTests.java | 3 +- .../InternalTDigestPercentilesRanksTests.java | 2 +- .../scripted/InternalScriptedMetricTests.java | 2 +- .../metrics/sum/InternalSumTests.java | 2 +- .../metrics/tophits/InternalTopHitsTests.java | 6 +- .../valuecount/InternalValueCountTests.java | 2 +- .../pipeline/InternalSimpleValueTests.java | 2 +- .../InternalPercentilesBucketTests.java | 2 +- .../derivative/InternalDerivativeTests.java | 2 +- .../matrix/stats/InternalMatrixStats.java | 19 ++- .../matrix/stats/MatrixStatsResults.java | 15 ++ .../matrix/stats/RunningStats.java | 25 ++- .../matrix/stats/BaseMatrixStatsTestCase.java | 124 +------------- .../stats/InternalMatrixStatsTests.java | 103 ++++++++++++ .../matrix/stats/MultiPassStats.java | 155 ++++++++++++++++++ .../test}/InternalAggregationTestCase.java | 4 +- 32 files changed, 345 insertions(+), 161 deletions(-) create mode 100644 modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java create mode 100644 modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java rename {core/src/test/java/org/elasticsearch/search/aggregations => test/framework/src/main/java/org/elasticsearch/test}/InternalAggregationTestCase.java (98%) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java index 5d2e8affe78..8da1f0a90e9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java @@ -20,11 +20,11 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.metrics.min.InternalMin; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.ArrayList; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java index ace22b244a3..a27b47946f9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java @@ -21,9 +21,9 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.index.IndexWriter; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java index 40f268e6556..aafad557bc6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java @@ -19,13 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.histogram; -import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.joda.time.DateTime; import java.util.ArrayList; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java index 093496738fd..0dd4a163f38 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java @@ -22,9 +22,9 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.ArrayList; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java index 2f8d4f9e65f..6d83b74c828 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRangeTests.java @@ -22,9 +22,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; import java.util.ArrayList; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java index dde1562e1af..f1c54c4b503 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTermsTestCase.java @@ -19,8 +19,8 @@ package org.elasticsearch.search.aggregations.bucket.significant; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Arrays; import java.util.HashMap; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java index 03031633f74..118f190bcab 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java @@ -19,14 +19,14 @@ package org.elasticsearch.search.aggregations.bucket.terms; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Map.Entry; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java index 83e1815f398..2e4f2e05872 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java @@ -21,9 +21,9 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; import java.util.Collections; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java index e2149b24549..f3b43d40a2a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java index c99d208581c..c4da54ac6b6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.metrics.stats.InternalStats; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Collections; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java index 1cfb93d6dcc..ca902ae5684 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java @@ -20,9 +20,8 @@ package org.elasticsearch.search.aggregations.metrics.avg; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java index 7c5809f323b..336def450a0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.After; import org.junit.Before; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java index cd5d4d43d17..8ec1d924b56 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.search.aggregations.metrics.geobounds; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Collections; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java index 3bbe1a1b462..d7c4328579c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.elasticsearch.test.geo.RandomGeoGenerator; import java.util.Collections; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java index d75f03a5de3..a40276e64c6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java @@ -20,9 +20,8 @@ package org.elasticsearch.search.aggregations.metrics.min; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java index e94cf753205..52462fbdaf7 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java index 15207e1d314..e57e7ff1164 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java @@ -22,9 +22,8 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; -import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java index bc1df930f6a..c4bdf907280 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index 9d8ff94c47c..75975d5a39f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -30,8 +30,8 @@ import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.io.IOException; import java.util.Collections; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java index 44e5ea940ad..cdaef89e42f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.search.aggregations.metrics.sum; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java index 22479eb4349..db68e8537df 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java @@ -30,11 +30,11 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.text.Text; -import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java index 3de87898778..e67d6fbf1da 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.search.aggregations.metrics.valuecount; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.List; import java.util.Map; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java index 4dae0a8d653..b3791b26ae8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.pipeline; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Collections; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java index 0ea28f98384..4aaae089b3c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucketTests.java @@ -21,9 +21,9 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Collections; import java.util.Iterator; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java index d2e4f3cb720..34082f77a8d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivativeTests.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.aggregations.pipeline.derivative; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; import java.util.Collections; import java.util.List; diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java index 0914ea2910c..5b7d2cf288d 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Collections.emptyMap; @@ -41,7 +42,7 @@ public class InternalMatrixStats extends InternalAggregation implements MatrixSt private final MatrixStatsResults results; /** per shard ctor */ - protected InternalMatrixStats(String name, long count, RunningStats multiFieldStatsResults, MatrixStatsResults results, + InternalMatrixStats(String name, long count, RunningStats multiFieldStatsResults, MatrixStatsResults results, List pipelineAggregators, Map metaData) { super(name, pipelineAggregators, metaData); assert count >= 0; @@ -138,6 +139,10 @@ public class InternalMatrixStats extends InternalAggregation implements MatrixSt return results.getCorrelation(fieldX, fieldY); } + MatrixStatsResults getResults() { + return results; + } + static class Fields { public static final String FIELDS = "fields"; public static final String NAME = "name"; @@ -238,4 +243,16 @@ public class InternalMatrixStats extends InternalAggregation implements MatrixSt return new InternalMatrixStats(name, results.getDocCount(), runningStats, results, pipelineAggregators(), getMetaData()); } + + @Override + protected int doHashCode() { + return Objects.hash(stats, results); + } + + @Override + protected boolean doEquals(Object obj) { + InternalMatrixStats other = (InternalMatrixStats) obj; + return Objects.equals(this.stats, other.stats) && + Objects.equals(this.results, other.results); + } } diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java index f82c6df73be..4da8b7ca617 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; /** * Descriptive stats gathered per shard. Coordinating node computes final pearson product coefficient @@ -228,4 +229,18 @@ class MatrixStatsResults implements Writeable { correlation.put(rowName, corRow); } } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MatrixStatsResults that = (MatrixStatsResults) o; + return Objects.equals(results, that.results) && + Objects.equals(correlation, that.correlation); + } + + @Override + public int hashCode() { + return Objects.hash(results, correlation); + } } diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java index 81d0d0a4943..1be3279e8ea 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; /** * Descriptive stats gathered per shard. Coordinating node computes final correlation and covariance stats @@ -53,11 +54,11 @@ public class RunningStats implements Writeable, Cloneable { /** covariance values */ protected HashMap> covariances; - public RunningStats() { + RunningStats() { init(); } - public RunningStats(final String[] fieldNames, final double[] fieldVals) { + RunningStats(final String[] fieldNames, final double[] fieldVals) { if (fieldVals != null && fieldVals.length > 0) { init(); this.add(fieldNames, fieldVals); @@ -309,4 +310,24 @@ public class RunningStats implements Writeable, Cloneable { throw new ElasticsearchException("Error trying to create a copy of RunningStats"); } } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RunningStats that = (RunningStats) o; + return docCount == that.docCount && + Objects.equals(fieldSum, that.fieldSum) && + Objects.equals(counts, that.counts) && + Objects.equals(means, that.means) && + Objects.equals(variances, that.variances) && + Objects.equals(skewness, that.skewness) && + Objects.equals(kurtosis, that.kurtosis) && + Objects.equals(covariances, that.covariances); + } + + @Override + public int hashCode() { + return Objects.hash(docCount, fieldSum, counts, means, variances, skewness, kurtosis, covariances); + } } diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java index 81c9d514636..091235bf82e 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java @@ -22,15 +22,12 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; import java.util.ArrayList; -import java.util.HashMap; - -import static org.hamcrest.Matchers.equalTo; public abstract class BaseMatrixStatsTestCase extends ESTestCase { protected final int numObs = atLeast(10000); protected final ArrayList fieldA = new ArrayList<>(numObs); protected final ArrayList fieldB = new ArrayList<>(numObs); - protected final MultiPassStats actualStats = new MultiPassStats(); + protected final MultiPassStats actualStats = new MultiPassStats(fieldAKey, fieldBKey); protected static final String fieldAKey = "fieldA"; protected static final String fieldBKey = "fieldB"; @@ -47,123 +44,4 @@ public abstract class BaseMatrixStatsTestCase extends ESTestCase { actualStats.computeStats(fieldA, fieldB); } - static class MultiPassStats { - long count; - HashMap means = new HashMap<>(); - HashMap variances = new HashMap<>(); - HashMap skewness = new HashMap<>(); - HashMap kurtosis = new HashMap<>(); - HashMap> covariances = new HashMap<>(); - HashMap> correlations = new HashMap<>(); - - @SuppressWarnings("unchecked") - void computeStats(final ArrayList fieldA, final ArrayList fieldB) { - // set count - count = fieldA.size(); - double meanA = 0d; - double meanB = 0d; - - // compute mean - for (int n = 0; n < count; ++n) { - // fieldA - meanA += fieldA.get(n); - meanB += fieldB.get(n); - } - means.put(fieldAKey, meanA/count); - means.put(fieldBKey, meanB/count); - - // compute variance, skewness, and kurtosis - double dA; - double dB; - double skewA = 0d; - double skewB = 0d; - double kurtA = 0d; - double kurtB = 0d; - double varA = 0d; - double varB = 0d; - double cVar = 0d; - for (int n = 0; n < count; ++n) { - dA = fieldA.get(n) - means.get(fieldAKey); - varA += dA * dA; - skewA += dA * dA * dA; - kurtA += dA * dA * dA * dA; - dB = fieldB.get(n) - means.get(fieldBKey); - varB += dB * dB; - skewB += dB * dB * dB; - kurtB += dB * dB * dB * dB; - cVar += dA * dB; - } - variances.put(fieldAKey, varA / (count - 1)); - final double stdA = Math.sqrt(variances.get(fieldAKey)); - variances.put(fieldBKey, varB / (count - 1)); - final double stdB = Math.sqrt(variances.get(fieldBKey)); - skewness.put(fieldAKey, skewA / ((count - 1) * variances.get(fieldAKey) * stdA)); - skewness.put(fieldBKey, skewB / ((count - 1) * variances.get(fieldBKey) * stdB)); - kurtosis.put(fieldAKey, kurtA / ((count - 1) * variances.get(fieldAKey) * variances.get(fieldAKey))); - kurtosis.put(fieldBKey, kurtB / ((count - 1) * variances.get(fieldBKey) * variances.get(fieldBKey))); - - // compute covariance - final HashMap fieldACovar = new HashMap<>(2); - fieldACovar.put(fieldAKey, 1d); - cVar /= count - 1; - fieldACovar.put(fieldBKey, cVar); - covariances.put(fieldAKey, fieldACovar); - final HashMap fieldBCovar = new HashMap<>(2); - fieldBCovar.put(fieldAKey, cVar); - fieldBCovar.put(fieldBKey, 1d); - covariances.put(fieldBKey, fieldBCovar); - - // compute correlation - final HashMap fieldACorr = new HashMap<>(); - fieldACorr.put(fieldAKey, 1d); - double corr = covariances.get(fieldAKey).get(fieldBKey); - corr /= stdA * stdB; - fieldACorr.put(fieldBKey, corr); - correlations.put(fieldAKey, fieldACorr); - final HashMap fieldBCorr = new HashMap<>(); - fieldBCorr.put(fieldAKey, corr); - fieldBCorr.put(fieldBKey, 1d); - correlations.put(fieldBKey, fieldBCorr); - } - - public void assertNearlyEqual(MatrixStatsResults stats) { - assertThat(count, equalTo(stats.getDocCount())); - assertThat(count, equalTo(stats.getFieldCount(fieldAKey))); - assertThat(count, equalTo(stats.getFieldCount(fieldBKey))); - // means - assertTrue(nearlyEqual(means.get(fieldAKey), stats.getMean(fieldAKey), 1e-7)); - assertTrue(nearlyEqual(means.get(fieldBKey), stats.getMean(fieldBKey), 1e-7)); - // variances - assertTrue(nearlyEqual(variances.get(fieldAKey), stats.getVariance(fieldAKey), 1e-7)); - assertTrue(nearlyEqual(variances.get(fieldBKey), stats.getVariance(fieldBKey), 1e-7)); - // skewness (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance) - assertTrue(nearlyEqual(skewness.get(fieldAKey), stats.getSkewness(fieldAKey), 1e-4)); - assertTrue(nearlyEqual(skewness.get(fieldBKey), stats.getSkewness(fieldBKey), 1e-4)); - // kurtosis (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance) - assertTrue(nearlyEqual(kurtosis.get(fieldAKey), stats.getKurtosis(fieldAKey), 1e-4)); - assertTrue(nearlyEqual(kurtosis.get(fieldBKey), stats.getKurtosis(fieldBKey), 1e-4)); - // covariances - assertTrue(nearlyEqual(covariances.get(fieldAKey).get(fieldBKey), stats.getCovariance(fieldAKey, fieldBKey), 1e-7)); - assertTrue(nearlyEqual(covariances.get(fieldBKey).get(fieldAKey), stats.getCovariance(fieldBKey, fieldAKey), 1e-7)); - // correlation - assertTrue(nearlyEqual(correlations.get(fieldAKey).get(fieldBKey), stats.getCorrelation(fieldAKey, fieldBKey), 1e-7)); - assertTrue(nearlyEqual(correlations.get(fieldBKey).get(fieldAKey), stats.getCorrelation(fieldBKey, fieldAKey), 1e-7)); - } - } - - private static boolean nearlyEqual(double a, double b, double epsilon) { - final double absA = Math.abs(a); - final double absB = Math.abs(b); - final double diff = Math.abs(a - b); - - if (a == b) { // shortcut, handles infinities - return true; - } else if (a == 0 || b == 0 || diff < Double.MIN_NORMAL) { - // a or b is zero or both are extremely close to it - // relative error is less meaningful here - return diff < (epsilon * Double.MIN_NORMAL); - } else { // use relative error - return diff / Math.min((absA + absB), Double.MAX_VALUE) < epsilon; - } - } } diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java new file mode 100644 index 00000000000..277006da90d --- /dev/null +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.matrix.stats; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class InternalMatrixStatsTests extends InternalAggregationTestCase { + + @Override + protected InternalMatrixStats createTestInstance(String name, List pipelineAggregators, + Map metaData) { + int numFields = randomInt(128); + String[] fieldNames = new String[numFields]; + double[] fieldValues = new double[numFields]; + for (int i = 0; i < numFields; i++) { + fieldNames[i] = Integer.toString(i); + fieldValues[i] = randomDouble(); + } + RunningStats runningStats = new RunningStats(); + runningStats.add(fieldNames, fieldValues); + MatrixStatsResults matrixStatsResults = randomBoolean() ? new MatrixStatsResults(runningStats) : null; + return new InternalMatrixStats("_name", 1L, runningStats, matrixStatsResults, Collections.emptyList(), Collections.emptyMap()); + } + + @Override + protected Writeable.Reader instanceReader() { + return InternalMatrixStats::new; + } + + @Override + public void testReduceRandom() { + int numValues = 10000; + int numShards = randomIntBetween(1, 20); + int valuesPerShard = (int) Math.floor(numValues / numShards); + + List aValues = new ArrayList<>(); + List bValues = new ArrayList<>(); + + RunningStats runningStats = new RunningStats(); + List shardResults = new ArrayList<>(); + + int valuePerShardCounter = 0; + for (int i = 0; i < numValues; i++) { + double valueA = randomDouble(); + aValues.add(valueA); + double valueB = randomDouble(); + bValues.add(valueB); + + runningStats.add(new String[]{"a", "b"}, new double[]{valueA, valueB}); + if (++valuePerShardCounter == valuesPerShard) { + shardResults.add(new InternalMatrixStats("_name", 1L, runningStats, null, Collections.emptyList(), Collections.emptyMap())); + runningStats = new RunningStats(); + valuePerShardCounter = 0; + } + } + + if (valuePerShardCounter != 0) { + shardResults.add(new InternalMatrixStats("_name", 1L, runningStats, null, Collections.emptyList(), Collections.emptyMap())); + } + MultiPassStats multiPassStats = new MultiPassStats("a", "b"); + multiPassStats.computeStats(aValues, bValues); + + ScriptService mockScriptService = mockScriptService(); + MockBigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + InternalAggregation.ReduceContext context = + new InternalAggregation.ReduceContext(bigArrays, mockScriptService, true); + InternalMatrixStats reduced = (InternalMatrixStats) shardResults.get(0).reduce(shardResults, context); + multiPassStats.assertNearlyEqual(reduced.getResults()); + } + + @Override + protected void assertReduced(InternalMatrixStats reduced, List inputs) { + throw new UnsupportedOperationException(); + } +} diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java new file mode 100644 index 00000000000..70e2172ce92 --- /dev/null +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java @@ -0,0 +1,155 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.matrix.stats; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +class MultiPassStats { + + private final String fieldAKey; + private final String fieldBKey; + + private long count; + private Map means = new HashMap<>(); + private Map variances = new HashMap<>(); + private Map skewness = new HashMap<>(); + private Map kurtosis = new HashMap<>(); + private Map> covariances = new HashMap<>(); + private Map> correlations = new HashMap<>(); + + MultiPassStats(String fieldAName, String fieldBName) { + this.fieldAKey = fieldAName; + this.fieldBKey = fieldBName; + } + + @SuppressWarnings("unchecked") + void computeStats(final List fieldA, final List fieldB) { + // set count + count = fieldA.size(); + double meanA = 0d; + double meanB = 0d; + + // compute mean + for (int n = 0; n < count; ++n) { + // fieldA + meanA += fieldA.get(n); + meanB += fieldB.get(n); + } + means.put(fieldAKey, meanA / count); + means.put(fieldBKey, meanB / count); + + // compute variance, skewness, and kurtosis + double dA; + double dB; + double skewA = 0d; + double skewB = 0d; + double kurtA = 0d; + double kurtB = 0d; + double varA = 0d; + double varB = 0d; + double cVar = 0d; + for (int n = 0; n < count; ++n) { + dA = fieldA.get(n) - means.get(fieldAKey); + varA += dA * dA; + skewA += dA * dA * dA; + kurtA += dA * dA * dA * dA; + dB = fieldB.get(n) - means.get(fieldBKey); + varB += dB * dB; + skewB += dB * dB * dB; + kurtB += dB * dB * dB * dB; + cVar += dA * dB; + } + variances.put(fieldAKey, varA / (count - 1)); + final double stdA = Math.sqrt(variances.get(fieldAKey)); + variances.put(fieldBKey, varB / (count - 1)); + final double stdB = Math.sqrt(variances.get(fieldBKey)); + skewness.put(fieldAKey, skewA / ((count - 1) * variances.get(fieldAKey) * stdA)); + skewness.put(fieldBKey, skewB / ((count - 1) * variances.get(fieldBKey) * stdB)); + kurtosis.put(fieldAKey, kurtA / ((count - 1) * variances.get(fieldAKey) * variances.get(fieldAKey))); + kurtosis.put(fieldBKey, kurtB / ((count - 1) * variances.get(fieldBKey) * variances.get(fieldBKey))); + + // compute covariance + final HashMap fieldACovar = new HashMap<>(2); + fieldACovar.put(fieldAKey, 1d); + cVar /= count - 1; + fieldACovar.put(fieldBKey, cVar); + covariances.put(fieldAKey, fieldACovar); + final HashMap fieldBCovar = new HashMap<>(2); + fieldBCovar.put(fieldAKey, cVar); + fieldBCovar.put(fieldBKey, 1d); + covariances.put(fieldBKey, fieldBCovar); + + // compute correlation + final HashMap fieldACorr = new HashMap<>(); + fieldACorr.put(fieldAKey, 1d); + double corr = covariances.get(fieldAKey).get(fieldBKey); + corr /= stdA * stdB; + fieldACorr.put(fieldBKey, corr); + correlations.put(fieldAKey, fieldACorr); + final HashMap fieldBCorr = new HashMap<>(); + fieldBCorr.put(fieldAKey, corr); + fieldBCorr.put(fieldBKey, 1d); + correlations.put(fieldBKey, fieldBCorr); + } + + void assertNearlyEqual(MatrixStatsResults stats) { + assertEquals(count, stats.getDocCount()); + assertEquals(count, stats.getFieldCount(fieldAKey)); + assertEquals(count, stats.getFieldCount(fieldBKey)); + // means + assertTrue(nearlyEqual(means.get(fieldAKey), stats.getMean(fieldAKey), 1e-7)); + assertTrue(nearlyEqual(means.get(fieldBKey), stats.getMean(fieldBKey), 1e-7)); + // variances + assertTrue(nearlyEqual(variances.get(fieldAKey), stats.getVariance(fieldAKey), 1e-7)); + assertTrue(nearlyEqual(variances.get(fieldBKey), stats.getVariance(fieldBKey), 1e-7)); + // skewness (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance) + assertTrue(nearlyEqual(skewness.get(fieldAKey), stats.getSkewness(fieldAKey), 1e-4)); + assertTrue(nearlyEqual(skewness.get(fieldBKey), stats.getSkewness(fieldBKey), 1e-4)); + // kurtosis (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance) + assertTrue(nearlyEqual(kurtosis.get(fieldAKey), stats.getKurtosis(fieldAKey), 1e-4)); + assertTrue(nearlyEqual(kurtosis.get(fieldBKey), stats.getKurtosis(fieldBKey), 1e-4)); + // covariances + assertTrue(nearlyEqual(covariances.get(fieldAKey).get(fieldBKey),stats.getCovariance(fieldAKey, fieldBKey), 1e-7)); + assertTrue(nearlyEqual(covariances.get(fieldBKey).get(fieldAKey),stats.getCovariance(fieldBKey, fieldAKey), 1e-7)); + // correlation + assertTrue(nearlyEqual(correlations.get(fieldAKey).get(fieldBKey), stats.getCorrelation(fieldAKey, fieldBKey), 1e-7)); + assertTrue(nearlyEqual(correlations.get(fieldBKey).get(fieldAKey), stats.getCorrelation(fieldBKey, fieldAKey), 1e-7)); + } + + private static boolean nearlyEqual(double a, double b, double epsilon) { + final double absA = Math.abs(a); + final double absB = Math.abs(b); + final double diff = Math.abs(a - b); + + if (a == b) { // shortcut, handles infinities + return true; + } else if (a == 0 || b == 0 || diff < Double.MIN_NORMAL) { + // a or b is zero or both are extremely close to it + // relative error is less meaningful here + return diff < (epsilon * Double.MIN_NORMAL); + } else { // use relative error + return diff / Math.min((absA + absB), Double.MAX_VALUE) < epsilon; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java similarity index 98% rename from core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java rename to test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 02abb394d0f..15385f56621 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.aggregations; +package org.elasticsearch.test; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; @@ -26,8 +26,8 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; import java.util.Collections;