diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/AnalyticsTestsUtils.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/AnalyticsTestsUtils.java new file mode 100644 index 00000000000..82bf177cf94 --- /dev/null +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/AnalyticsTestsUtils.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.analytics; + +import java.io.IOException; + +import org.apache.lucene.document.BinaryDocValuesField; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.search.aggregations.metrics.TDigestState; + +import com.tdunning.math.stats.Centroid; +import com.tdunning.math.stats.TDigest; + +public final class AnalyticsTestsUtils { + + /** + * Generates an index fields for histogram fields. Used in tests of aggregations that work on histogram fields. + */ + public static BinaryDocValuesField histogramFieldDocValues(String fieldName, double[] values) throws IOException { + TDigest histogram = new TDigestState(100.0); //default + for (double value : values) { + histogram.add(value); + } + BytesStreamOutput streamOutput = new BytesStreamOutput(); + histogram.compress(); + for (Centroid centroid : histogram.centroids()) { + streamOutput.writeVInt(centroid.count()); + streamOutput.writeDouble(centroid.mean()); + } + return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); + } + +} diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java index bdc684d2509..437d483c704 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java @@ -6,15 +6,17 @@ package org.elasticsearch.xpack.analytics.aggregations.bucket.histogram; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.util.Collections; +import java.util.List; + import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -22,20 +24,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import static java.util.Collections.singleton; - public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -43,9 +36,9 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { public void testHistograms() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30}))); HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field(FIELD_NAME) @@ -80,9 +73,9 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { public void testMinDocCount() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30, 90}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30, 90}))); HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field(FIELD_NAME) @@ -109,8 +102,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { // Note, these values are carefully chosen to ensure that no matter what offset we pick, no two can end up in the same bucket - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3.2, 9.3}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3.2, 9.3}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 }))); final double offset = randomDouble(); final double interval = 5; @@ -143,8 +136,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-4.5, 4.3}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 }))); HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field(FIELD_NAME) @@ -178,8 +171,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3}))); - w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-4.5, 4.3}))); + w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 }))); HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field(FIELD_NAME) @@ -198,23 +191,6 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { } } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregatorTests.java index fb22b8adad6..29ee46b8d55 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregatorTests.java @@ -5,9 +5,6 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.RandomIndexWriter; @@ -16,7 +13,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -24,7 +20,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalAvg; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -34,14 +29,13 @@ import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; public class HistoBackedAvgAggregatorTests extends AggregatorTestCase { @@ -58,8 +52,8 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase { public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20}))); }, avg -> { assertEquals(Double.NaN, avg.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(avg)); @@ -68,9 +62,9 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase { public void testSimpleHistogram() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); }, avg -> { assertEquals(12.0463d, avg.getValue(), 0.01d); assertTrue(AggregationInspectionHelper.hasValue(avg)); @@ -81,23 +75,23 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase { testCase(new TermQuery(new Term("match", "yes")), iw -> { iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {5.3, 6, 20})) + histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) + histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) ); }, avg -> { assertEquals(12.651d, avg.getValue(), 0.01d); @@ -111,23 +105,6 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase { testCase(avg("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType()); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMaxAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMaxAggregatorTests.java index 9e9011f4ac6..83baca43abb 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMaxAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMaxAggregatorTests.java @@ -5,9 +5,16 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.RandomIndexWriter; @@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalMax; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.Collections.singleton; -import static org.elasticsearch.search.aggregations.AggregationBuilders.max; - public class HistoBackedMaxAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -58,8 +52,8 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase { public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20}))); }, max -> { assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(max)); @@ -68,9 +62,9 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase { public void testSimpleHistogram() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); }, max -> { assertEquals(90d, max.getValue(), 0.01d); assertTrue(AggregationInspectionHelper.hasValue(max)); @@ -81,23 +75,23 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase { testCase(new TermQuery(new Term("match", "yes")), iw -> { iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {5.3, 6, 20})) + histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-34, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {-34, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 100})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 100})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) + histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) ); }, min -> { assertEquals(90d, min.getValue(), 0.01d); @@ -111,23 +105,6 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase { testCase(max("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType()); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMinAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMinAggregatorTests.java index 75ce1af8890..d79cc9b32f5 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMinAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMinAggregatorTests.java @@ -5,9 +5,16 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.min; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.RandomIndexWriter; @@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalMin; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.Collections.singleton; -import static org.elasticsearch.search.aggregations.AggregationBuilders.min; - public class HistoBackedMinAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -58,8 +52,8 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase { public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20}))); }, min -> { assertEquals(Double.POSITIVE_INFINITY, min.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(min)); @@ -68,9 +62,9 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase { public void testSimpleHistogram() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); }, min -> { assertEquals(-10d, min.getValue(), 0.01d); assertTrue(AggregationInspectionHelper.hasValue(min)); @@ -81,23 +75,23 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase { testCase(new TermQuery(new Term("match", "yes")), iw -> { iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {5.3, 6, 20})) + histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-34, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {-34, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) + histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) ); }, min -> { assertEquals(-10d, min.getValue(), 0.01d); @@ -111,23 +105,6 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase { testCase(min("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType()); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregatorTests.java index b5b366b48d8..39556d5e9bb 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregatorTests.java @@ -5,9 +5,16 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.RandomIndexWriter; @@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalSum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.Collections.singleton; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; - public class HistoBackedSumAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -58,8 +52,8 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase { public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20}))); }, sum -> { assertEquals(0L, sum.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(sum)); @@ -68,9 +62,9 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase { public void testSimpleHistogram() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); }, sum -> { assertEquals(132.51d, sum.getValue(), 0.01d); assertTrue(AggregationInspectionHelper.hasValue(sum)); @@ -81,23 +75,23 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase { testCase(new TermQuery(new Term("match", "yes")), iw -> { iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {5.3, 6, 20})) + histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) + histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) ); }, sum -> { assertEquals(126.51d, sum.getValue(), 0.01d); @@ -111,23 +105,6 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase { testCase(sum("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType()); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedValueCountAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedValueCountAggregatorTests.java index e76a90590c9..5c0b5807231 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedValueCountAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedValueCountAggregatorTests.java @@ -5,9 +5,16 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.count; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.RandomIndexWriter; @@ -16,14 +23,12 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalValueCount; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.Collections.singleton; -import static org.elasticsearch.search.aggregations.AggregationBuilders.count; - public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -58,8 +52,8 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase { public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20}))); }, count -> { assertEquals(0L, count.getValue()); assertFalse(AggregationInspectionHelper.hasValue(count)); @@ -68,9 +62,9 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase { public void testSimpleHistogram() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); - iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))); }, count -> { assertEquals(11, count.getValue()); assertTrue(AggregationInspectionHelper.hasValue(count)); @@ -81,23 +75,23 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase { testCase(new TermQuery(new Term("match", "yes")), iw -> { iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {5.3, 6, 20})) + histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "no", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})) + histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})) ); iw.addDocument(Arrays.asList( new StringField("match", "yes", Field.Store.NO), - getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) + histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})) ); }, count -> { assertEquals(10, count.getValue()); @@ -112,23 +106,6 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase { testCase(count("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType()); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - @Override protected List getSearchPlugins() { return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java index ea039122208..8ffd9fa5081 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java @@ -5,16 +5,18 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import java.util.Arrays; +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -26,7 +28,6 @@ import org.elasticsearch.search.aggregations.metrics.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -35,13 +36,7 @@ import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSou import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; import org.hamcrest.Matchers; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; public class TDigestPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase { @@ -66,28 +61,11 @@ public class TDigestPreAggregatedPercentileRanksAggregatorTests extends Aggregat AnalyticsValuesSourceType.HISTOGRAM); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - public void testSimple() throws IOException { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { Document doc = new Document(); - doc.add(getDocValue("field", new double[] {3, 0.2, 10})); + doc.add(histogramFieldDocValues("field", new double[] {3, 0.2, 10})); w.addDocument(doc); PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[] {0.1, 0.5, 12}) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java index e23159be2c7..9f67c04a7b7 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java @@ -5,9 +5,15 @@ */ package org.elasticsearch.xpack.analytics.aggregations.metrics; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import org.apache.lucene.document.BinaryDocValuesField; +import static java.util.Collections.singleton; +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; + +import java.util.Arrays; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -17,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.SearchPlugin; @@ -28,7 +33,6 @@ import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; -import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -36,16 +40,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.Collections.singleton; - public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { @Override @@ -69,26 +63,9 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe AnalyticsValuesSourceType.HISTOGRAM); } - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { - TDigest histogram = new TDigestState(100.0); //default - for (double value : values) { - histogram.add(value); - } - BytesStreamOutput streamOutput = new BytesStreamOutput(); - histogram.compress(); - Collection centroids = histogram.centroids(); - Iterator iterator = centroids.iterator(); - while ( iterator.hasNext()) { - Centroid centroid = iterator.next(); - streamOutput.writeVInt(centroid.count()); - streamOutput.writeDouble(centroid.mean()); - } - return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } - public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("wrong_number", new double[]{7, 1}))); + iw.addDocument(singleton(histogramFieldDocValues("wrong_number", new double[]{7, 1}))); }, hdr -> { //assertEquals(0L, hdr.state.getTotalCount()); assertFalse(AggregationInspectionHelper.hasValue(hdr)); @@ -97,7 +74,7 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe public void testEmptyField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(getDocValue("number", new double[0]))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[0]))); }, hdr -> { assertFalse(AggregationInspectionHelper.hasValue(hdr)); }); @@ -105,7 +82,7 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe public void testSomeMatchesBinaryDocValues() throws IOException { testCase(new DocValuesFieldExistsQuery("number"), iw -> { - iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10}))); }, hdr -> { //assertEquals(4L, hdr.state.getTotalCount()); double approximation = 0.05d; @@ -119,10 +96,10 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe public void testSomeMatchesMultiBinaryDocValues() throws IOException { testCase(new DocValuesFieldExistsQuery("number"), iw -> { - iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10}))); - iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10}))); - iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10}))); - iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10}))); + iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10}))); }, hdr -> { //assertEquals(16L, hdr.state.getTotalCount()); double approximation = 0.05d;