honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract histogramFieldDocValues into an utility class (#63100) (#63148) 

This function will be needed in the upcoming rate aggs tests.
This commit is contained in:
Igor Motov 2020-10-01 15:44:37 -04:00 committed by GitHub
parent 3ad4b00c7e
commit fc13b72cea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 171 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/AnalyticsTestsUtils.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.analytics;
import java.io.IOException;
import org.apache.lucene.document.BinaryDocValuesField;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
public final class AnalyticsTestsUtils {
/**
* Generates an index fields for histogram fields. Used in tests of aggregations that work on histogram fields.
*/
public static BinaryDocValuesField histogramFieldDocValues(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
for (Centroid centroid : histogram.centroids()) {
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
}

60
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java
View File

@ -6,15 +6,17 @@
package org.elasticsearch.xpack.analytics.aggregations.bucket.histogram;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -22,20 +24,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import static java.util.Collections.singleton;
public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
private static final String FIELD_NAME = "field";
@ -43,9 +36,9 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
public void testHistograms() throws Exception {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30})));
HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
.field(FIELD_NAME)
@ -80,9 +73,9 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
public void testMinDocCount() throws Exception {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30, 90})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30, 90})));
HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
.field(FIELD_NAME)
@ -109,8 +102,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
// Note, these values are carefully chosen to ensure that no matter what offset we pick, no two can end up in the same bucket
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3.2, 9.3})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 })));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3.2, 9.3})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 })));
final double offset = randomDouble();
final double interval = 5;
@ -143,8 +136,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 })));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-4.5, 4.3})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 })));
HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
.field(FIELD_NAME)
@ -178,8 +171,8 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3})));
w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 })));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-4.5, 4.3})));
w.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-5, 3.2 })));
HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
.field(FIELD_NAME)
@ -198,23 +191,6 @@ public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase {
}
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

45
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregatorTests.java
View File

@ -5,9 +5,6 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.RandomIndexWriter;
@ -16,7 +13,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -24,7 +20,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.InternalAvg;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -34,14 +29,13 @@ import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
public class HistoBackedAvgAggregatorTests extends AggregatorTestCase {
@ -58,8 +52,8 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase {
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20})));
}, avg -> {
assertEquals(Double.NaN, avg.getValue(), 0d);
assertFalse(AggregationInspectionHelper.hasValue(avg));
@ -68,9 +62,9 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase {
public void testSimpleHistogram() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
}, avg -> {
assertEquals(12.0463d, avg.getValue(), 0.01d);
assertTrue(AggregationInspectionHelper.hasValue(avg));
@ -81,23 +75,23 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase {
testCase(new TermQuery(new Term("match", "yes")), iw -> {
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {5.3, 6, 20}))
histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
);
}, avg -> {
assertEquals(12.651d, avg.getValue(), 0.01d);
@ -111,23 +105,6 @@ public class HistoBackedAvgAggregatorTests extends AggregatorTestCase {
testCase(avg("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType());
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

63
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMaxAggregatorTests.java
View File

@ -5,9 +5,16 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.RandomIndexWriter;
@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.InternalMax;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
public class HistoBackedMaxAggregatorTests extends AggregatorTestCase {
private static final String FIELD_NAME = "field";
@ -58,8 +52,8 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase {
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20})));
}, max -> {
assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0d);
assertFalse(AggregationInspectionHelper.hasValue(max));
@ -68,9 +62,9 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase {
public void testSimpleHistogram() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
}, max -> {
assertEquals(90d, max.getValue(), 0.01d);
assertTrue(AggregationInspectionHelper.hasValue(max));
@ -81,23 +75,23 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase {
testCase(new TermQuery(new Term("match", "yes")), iw -> {
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {5.3, 6, 20}))
histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-34, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {-34, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 100}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 100}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
);
}, min -> {
assertEquals(90d, min.getValue(), 0.01d);
@ -111,23 +105,6 @@ public class HistoBackedMaxAggregatorTests extends AggregatorTestCase {
testCase(max("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType());
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

63
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedMinAggregatorTests.java
View File

@ -5,9 +5,16 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.min;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.RandomIndexWriter;
@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.InternalMin;
import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.min;
public class HistoBackedMinAggregatorTests extends AggregatorTestCase {
private static final String FIELD_NAME = "field";
@ -58,8 +52,8 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase {
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20})));
}, min -> {
assertEquals(Double.POSITIVE_INFINITY, min.getValue(), 0d);
assertFalse(AggregationInspectionHelper.hasValue(min));
@ -68,9 +62,9 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase {
public void testSimpleHistogram() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
}, min -> {
assertEquals(-10d, min.getValue(), 0.01d);
assertTrue(AggregationInspectionHelper.hasValue(min));
@ -81,23 +75,23 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase {
testCase(new TermQuery(new Term("match", "yes")), iw -> {
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {5.3, 6, 20}))
histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-34, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {-34, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
);
}, min -> {
assertEquals(-10d, min.getValue(), 0.01d);
@ -111,23 +105,6 @@ public class HistoBackedMinAggregatorTests extends AggregatorTestCase {
testCase(min("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType());
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

63
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregatorTests.java
View File

@ -5,9 +5,16 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.RandomIndexWriter;
@ -16,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -24,7 +30,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.InternalSum;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
public class HistoBackedSumAggregatorTests extends AggregatorTestCase {
private static final String FIELD_NAME = "field";
@ -58,8 +52,8 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase {
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20})));
}, sum -> {
assertEquals(0L, sum.getValue(), 0d);
assertFalse(AggregationInspectionHelper.hasValue(sum));
@ -68,9 +62,9 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase {
public void testSimpleHistogram() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
}, sum -> {
assertEquals(132.51d, sum.getValue(), 0.01d);
assertTrue(AggregationInspectionHelper.hasValue(sum));
@ -81,23 +75,23 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase {
testCase(new TermQuery(new Term("match", "yes")), iw -> {
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {5.3, 6, 20}))
histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
);
}, sum -> {
assertEquals(126.51d, sum.getValue(), 0.01d);
@ -111,23 +105,6 @@ public class HistoBackedSumAggregatorTests extends AggregatorTestCase {
testCase(sum("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType());
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

63
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedValueCountAggregatorTests.java
View File

@ -5,9 +5,16 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.count;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.RandomIndexWriter;
@ -16,14 +23,12 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.metrics.InternalValueCount;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
@ -32,17 +37,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
import static org.elasticsearch.search.aggregations.AggregationBuilders.count;
public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase {
private static final String FIELD_NAME = "field";
@ -58,8 +52,8 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase {
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue("wrong_field", new double[] {5.3, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_field", new double[] {5.3, 6, 20})));
}, count -> {
assertEquals(0L, count.getValue());
assertFalse(AggregationInspectionHelper.hasValue(count));
@ -68,9 +62,9 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase {
public void testSimpleHistogram() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 6, 20})));
iw.addDocument(singleton(histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90})));
}, count -> {
assertEquals(11, count.getValue());
assertTrue(AggregationInspectionHelper.hasValue(count));
@ -81,23 +75,23 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase {
testCase(new TermQuery(new Term("match", "yes")), iw -> {
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {5.3, 6, 20}))
histogramFieldDocValues(FIELD_NAME, new double[] {5.3, 6, 20}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "no", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {3, 1.2, 10}))
histogramFieldDocValues(FIELD_NAME, new double[] {3, 1.2, 10}))
);
iw.addDocument(Arrays.asList(
new StringField("match", "yes", Field.Store.NO),
getDocValue(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
histogramFieldDocValues(FIELD_NAME, new double[] {-10, 0.01, 1, 90}))
);
}, count -> {
assertEquals(10, count.getValue());
@ -112,23 +106,6 @@ public class HistoBackedValueCountAggregatorTests extends AggregatorTestCase {
testCase(count("_name").field(FIELD_NAME), query, indexer, verify, defaultFieldType());
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
@Override
protected List<SearchPlugin> getSearchPlugins() {
return org.elasticsearch.common.collect.List.of(new AnalyticsPlugin(Settings.EMPTY));

38
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java
View File

@ -5,16 +5,18 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import java.util.Arrays;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -26,7 +28,6 @@ import org.elasticsearch.search.aggregations.metrics.PercentileRanks;
import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.PercentilesConfig;
import org.elasticsearch.search.aggregations.metrics.PercentilesMethod;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -35,13 +36,7 @@ import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSou
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
public class TDigestPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase {
@ -66,28 +61,11 @@ public class TDigestPreAggregatedPercentileRanksAggregatorTests extends Aggregat
AnalyticsValuesSourceType.HISTOGRAM);
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
public void testSimple() throws IOException {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
Document doc = new Document();
doc.add(getDocValue("field", new double[] {3, 0.2, 10}));
doc.add(histogramFieldDocValues("field", new double[] {3, 0.2, 10}));
w.addDocument(doc);
PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[] {0.1, 0.5, 12})

55
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java
View File

@ -5,9 +5,15 @@
*/
package org.elasticsearch.xpack.analytics.aggregations.metrics;
import com.tdunning.math.stats.Centroid;
import com.tdunning.math.stats.TDigest;
import org.apache.lucene.document.BinaryDocValuesField;
import static java.util.Collections.singleton;
import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
import java.util.Arrays;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
@ -17,7 +23,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.plugins.SearchPlugin;
@ -28,7 +33,6 @@ import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles;
import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.PercentilesConfig;
import org.elasticsearch.search.aggregations.metrics.PercentilesMethod;
import org.elasticsearch.search.aggregations.metrics.TDigestState;
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -36,16 +40,6 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import static java.util.Collections.singleton;
public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase {
@Override
@ -69,26 +63,9 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe
AnalyticsValuesSourceType.HISTOGRAM);
}
private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException {
TDigest histogram = new TDigestState(100.0); //default
for (double value : values) {
histogram.add(value);
}
BytesStreamOutput streamOutput = new BytesStreamOutput();
histogram.compress();
Collection<Centroid> centroids = histogram.centroids();
Iterator<Centroid> iterator = centroids.iterator();
while ( iterator.hasNext()) {
Centroid centroid = iterator.next();
streamOutput.writeVInt(centroid.count());
streamOutput.writeDouble(centroid.mean());
}
return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef());
}
public void testNoMatchingField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("wrong_number", new double[]{7, 1})));
iw.addDocument(singleton(histogramFieldDocValues("wrong_number", new double[]{7, 1})));
}, hdr -> {
//assertEquals(0L, hdr.state.getTotalCount());
assertFalse(AggregationInspectionHelper.hasValue(hdr));
@ -97,7 +74,7 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe
public void testEmptyField() throws IOException {
testCase(new MatchAllDocsQuery(), iw -> {
iw.addDocument(singleton(getDocValue("number", new double[0])));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[0])));
}, hdr -> {
assertFalse(AggregationInspectionHelper.hasValue(hdr));
});
@ -105,7 +82,7 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe
public void testSomeMatchesBinaryDocValues() throws IOException {
testCase(new DocValuesFieldExistsQuery("number"), iw -> {
iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10})));
}, hdr -> {
//assertEquals(4L, hdr.state.getTotalCount());
double approximation = 0.05d;
@ -119,10 +96,10 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTe
public void testSomeMatchesMultiBinaryDocValues() throws IOException {
testCase(new DocValuesFieldExistsQuery("number"), iw -> {
iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(getDocValue("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10})));
iw.addDocument(singleton(histogramFieldDocValues("number", new double[]{60, 40, 20, 10})));
}, hdr -> {
//assertEquals(16L, hdr.state.getTotalCount());
double approximation = 0.05d;