Merge branch 'master' into feature/query-refactoring
This commit is contained in:
commit
3d53413650
|
@ -0,0 +1 @@
|
|||
d1831874fb3c769fd126c4826e69e6b40c703ee0
|
|
@ -0,0 +1,41 @@
|
|||
The code in this repository code was Written by Gil Tene, Michael Barker,
|
||||
and Matt Warren, and released to the public domain, as explained at
|
||||
http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
For users of this code who wish to consume it under the "BSD" license
|
||||
rather than under the public domain or CC0 contribution text mentioned
|
||||
above, the code found under this directory is *also* provided under the
|
||||
following license (commonly referred to as the BSD 2-Clause License). This
|
||||
license does not detract from the above stated release of the code into
|
||||
the public domain, and simply represents an additional license granted by
|
||||
the Author.
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
** Beginning of "BSD 2-Clause License" text. **
|
||||
|
||||
Copyright (c) 2012, 2013, 2014 Gil Tene
|
||||
Copyright (c) 2014 Michael Barker
|
||||
Copyright (c) 2014 Matt Warren
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||
THE POSSIBILITY OF SUCH DAMAGE.
|
|
@ -178,6 +178,10 @@
|
|||
<groupId>com.tdunning</groupId>
|
||||
<artifactId>t-digest</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hdrhistogram</groupId>
|
||||
<artifactId>HdrHistogram</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
|
|
|
@ -26,6 +26,7 @@
|
|||
<include>org.apache.commons:commons-lang3</include>
|
||||
<include>commons-cli:commons-cli</include>
|
||||
<include>com.twitter:jsr166e</include>
|
||||
<include>org.hdrhistogram:HdrHistogram</include>
|
||||
</includes>
|
||||
</dependencySet>
|
||||
<dependencySet>
|
||||
|
|
|
@ -51,8 +51,10 @@ import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardina
|
|||
import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds;
|
||||
import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
|
||||
import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.scripted.InternalScriptedMetric;
|
||||
import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
|
||||
import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
|
||||
|
@ -65,8 +67,8 @@ import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucke
|
|||
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivative;
|
||||
import org.elasticsearch.search.aggregations.pipeline.having.BucketSelectorPipelineAggregator;
|
||||
|
@ -90,8 +92,10 @@ public class TransportAggregationModule extends AbstractModule implements SpawnM
|
|||
InternalStats.registerStreams();
|
||||
InternalExtendedStats.registerStreams();
|
||||
InternalValueCount.registerStreams();
|
||||
InternalPercentiles.registerStreams();
|
||||
InternalPercentileRanks.registerStreams();
|
||||
InternalTDigestPercentiles.registerStreams();
|
||||
InternalTDigestPercentileRanks.registerStreams();
|
||||
InternalHDRPercentiles.registerStreams();
|
||||
InternalHDRPercentileRanks.registerStreams();
|
||||
InternalCardinality.registerStreams();
|
||||
InternalScriptedMetric.registerStreams();
|
||||
|
||||
|
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.search.aggregations.metrics.ValuesSourceMetricsAggregationBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
abstract class AbstractPercentilesBuilder<PB extends AbstractPercentilesBuilder<PB>> extends
|
||||
ValuesSourceMetricsAggregationBuilder<PB> {
|
||||
|
||||
private Double compression;
|
||||
private PercentilesMethod method;
|
||||
private Integer numberOfSignificantValueDigits;
|
||||
|
||||
public AbstractPercentilesBuilder(String name, String type) {
|
||||
super(name, type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: Set the method to use to compute the percentiles.
|
||||
*/
|
||||
public PB method(PercentilesMethod method) {
|
||||
this.method = method;
|
||||
return (PB) this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: set the compression. Higher values improve accuracy but also
|
||||
* memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}.
|
||||
*/
|
||||
public PB compression(double compression) {
|
||||
this.compression = compression;
|
||||
return (PB) this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: set the number of significant digits in the values. Only relevant
|
||||
* when using {@link PercentilesMethod#HDR}.
|
||||
*/
|
||||
public PB numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
return (PB) this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void internalXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
super.internalXContent(builder, params);
|
||||
|
||||
doInternalXContent(builder, params);
|
||||
|
||||
if (method != null) {
|
||||
builder.startObject(method.getName());
|
||||
|
||||
if (compression != null) {
|
||||
builder.field(AbstractPercentilesParser.COMPRESSION_FIELD.getPreferredName(), compression);
|
||||
}
|
||||
|
||||
if (numberOfSignificantValueDigits != null) {
|
||||
builder.field(AbstractPercentilesParser.NUMBER_SIGNIFICANT_DIGITS_FIELD.getPreferredName(), numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
builder.endObject();
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract void doInternalXContent(XContentBuilder builder, Params params) throws IOException;
|
||||
|
||||
}
|
|
@ -21,10 +21,12 @@ package org.elasticsearch.search.aggregations.metrics.percentiles;
|
|||
|
||||
import com.carrotsearch.hppc.DoubleArrayList;
|
||||
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.search.SearchParseException;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactory;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
|
@ -36,6 +38,11 @@ import java.util.Arrays;
|
|||
|
||||
public abstract class AbstractPercentilesParser implements Aggregator.Parser {
|
||||
|
||||
public static final ParseField KEYED_FIELD = new ParseField("keyed");
|
||||
public static final ParseField METHOD_FIELD = new ParseField("method");
|
||||
public static final ParseField COMPRESSION_FIELD = new ParseField("compression");
|
||||
public static final ParseField NUMBER_SIGNIFICANT_DIGITS_FIELD = new ParseField("number_of_significant_value_digits");
|
||||
|
||||
private boolean formattable;
|
||||
|
||||
public AbstractPercentilesParser(boolean formattable) {
|
||||
|
@ -44,14 +51,16 @@ public abstract class AbstractPercentilesParser implements Aggregator.Parser {
|
|||
|
||||
@Override
|
||||
public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException {
|
||||
|
||||
ValuesSourceParser<ValuesSource.Numeric> vsParser = ValuesSourceParser.numeric(aggregationName, InternalPercentiles.TYPE, context)
|
||||
|
||||
ValuesSourceParser<ValuesSource.Numeric> vsParser = ValuesSourceParser.numeric(aggregationName, InternalTDigestPercentiles.TYPE, context)
|
||||
.formattable(formattable).build();
|
||||
|
||||
|
||||
double[] keys = null;
|
||||
boolean keyed = true;
|
||||
double compression = 100;
|
||||
|
||||
Double compression = null;
|
||||
Integer numberOfSignificantValueDigits = null;
|
||||
PercentilesMethod method = null;
|
||||
|
||||
XContentParser.Token token;
|
||||
String currentFieldName = null;
|
||||
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
|
||||
|
@ -60,7 +69,7 @@ public abstract class AbstractPercentilesParser implements Aggregator.Parser {
|
|||
} else if (vsParser.token(currentFieldName, token, parser)) {
|
||||
continue;
|
||||
} else if (token == XContentParser.Token.START_ARRAY) {
|
||||
if (keysFieldName().equals(currentFieldName)) {
|
||||
if (context.parseFieldMatcher().match(currentFieldName, keysField())) {
|
||||
DoubleArrayList values = new DoubleArrayList(10);
|
||||
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
|
||||
double value = parser.doubleValue();
|
||||
|
@ -73,30 +82,104 @@ public abstract class AbstractPercentilesParser implements Aggregator.Parser {
|
|||
+ currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
|
||||
if ("keyed".equals(currentFieldName)) {
|
||||
if (context.parseFieldMatcher().match(currentFieldName, KEYED_FIELD)) {
|
||||
keyed = parser.booleanValue();
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: ["
|
||||
+ currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
} else if (token == XContentParser.Token.VALUE_NUMBER) {
|
||||
if ("compression".equals(currentFieldName)) {
|
||||
compression = parser.doubleValue();
|
||||
} else if (token == XContentParser.Token.START_OBJECT) {
|
||||
if (method != null) {
|
||||
throw new SearchParseException(context, "Found multiple methods in [" + aggregationName + "]: [" + currentFieldName
|
||||
+ "]. only one of [" + PercentilesMethod.TDIGEST.getName() + "] and [" + PercentilesMethod.HDR.getName()
|
||||
+ "] may be used.", parser.getTokenLocation());
|
||||
}
|
||||
method = PercentilesMethod.resolveFromName(currentFieldName);
|
||||
if (method == null) {
|
||||
throw new SearchParseException(context, "Unexpected token " + token + " in [" + aggregationName + "].",
|
||||
parser.getTokenLocation());
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: ["
|
||||
+ currentFieldName + "].", parser.getTokenLocation());
|
||||
switch (method) {
|
||||
case TDIGEST:
|
||||
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
|
||||
if (token == XContentParser.Token.FIELD_NAME) {
|
||||
currentFieldName = parser.currentName();
|
||||
} else if (token == XContentParser.Token.VALUE_NUMBER) {
|
||||
if (context.parseFieldMatcher().match(currentFieldName, COMPRESSION_FIELD)) {
|
||||
compression = parser.doubleValue();
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName
|
||||
+ "]: [" + currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: ["
|
||||
+ currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case HDR:
|
||||
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
|
||||
if (token == XContentParser.Token.FIELD_NAME) {
|
||||
currentFieldName = parser.currentName();
|
||||
} else if (token == XContentParser.Token.VALUE_NUMBER) {
|
||||
if (context.parseFieldMatcher().match(currentFieldName, NUMBER_SIGNIFICANT_DIGITS_FIELD)) {
|
||||
numberOfSignificantValueDigits = parser.intValue();
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName
|
||||
+ "]: [" + currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: ["
|
||||
+ currentFieldName + "].", parser.getTokenLocation());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new SearchParseException(context, "Unexpected token " + token + " in [" + aggregationName + "].",
|
||||
parser.getTokenLocation());
|
||||
}
|
||||
}
|
||||
return buildFactory(context, aggregationName, vsParser.config(), keys, compression, keyed);
|
||||
|
||||
if (method == null) {
|
||||
method = PercentilesMethod.TDIGEST;
|
||||
}
|
||||
|
||||
switch (method) {
|
||||
case TDIGEST:
|
||||
if (numberOfSignificantValueDigits != null) {
|
||||
throw new SearchParseException(context, "[number_of_significant_value_digits] cannot be used with method [tdigest] in ["
|
||||
+ aggregationName + "].", parser.getTokenLocation());
|
||||
}
|
||||
if (compression == null) {
|
||||
compression = 100.0;
|
||||
}
|
||||
break;
|
||||
case HDR:
|
||||
if (compression != null) {
|
||||
throw new SearchParseException(context, "[compression] cannot be used with method [hdr] in [" + aggregationName + "].",
|
||||
parser.getTokenLocation());
|
||||
}
|
||||
if (numberOfSignificantValueDigits == null) {
|
||||
numberOfSignificantValueDigits = 3;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
// Shouldn't get here but if we do, throw a parse exception for
|
||||
// invalid method
|
||||
throw new SearchParseException(context, "Unknown value for [" + currentFieldName + "] in [" + aggregationName + "]: [" + method
|
||||
+ "].", parser.getTokenLocation());
|
||||
}
|
||||
|
||||
return buildFactory(context, aggregationName, vsParser.config(), keys, method, compression,
|
||||
numberOfSignificantValueDigits, keyed);
|
||||
}
|
||||
|
||||
protected abstract AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> config, double[] cdfValues,
|
||||
double compression, boolean keyed);
|
||||
protected abstract AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> config,
|
||||
double[] cdfValues, PercentilesMethod method, Double compression, Integer numberOfSignificantValueDigits, boolean keyed);
|
||||
|
||||
protected abstract String keysFieldName();
|
||||
protected abstract ParseField keysField();
|
||||
|
||||
}
|
||||
|
|
|
@ -19,12 +19,12 @@
|
|||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
class InternalPercentile implements Percentile {
|
||||
public class InternalPercentile implements Percentile {
|
||||
|
||||
private final double percent;
|
||||
private final double value;
|
||||
|
||||
InternalPercentile(double percent, double value) {
|
||||
public InternalPercentile(double percent, double value) {
|
||||
this.percent = percent;
|
||||
this.value = value;
|
||||
}
|
||||
|
|
|
@ -26,6 +26,8 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
|
|||
*/
|
||||
public interface PercentileRanks extends NumericMetricsAggregation.MultiValue, Iterable<Percentile> {
|
||||
|
||||
public static final String TYPE_NAME = "percentile_ranks";
|
||||
|
||||
/**
|
||||
* Return the percentile for the given value.
|
||||
*/
|
||||
|
|
|
@ -19,23 +19,21 @@
|
|||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.search.aggregations.metrics.ValuesSourceMetricsAggregationBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Builder for the {@link PercentileRanks} aggregation.
|
||||
*/
|
||||
public class PercentileRanksBuilder extends ValuesSourceMetricsAggregationBuilder<PercentileRanksBuilder> {
|
||||
public class PercentileRanksBuilder extends AbstractPercentilesBuilder<PercentileRanksBuilder> {
|
||||
|
||||
private double[] values;
|
||||
private Double compression;
|
||||
|
||||
/**
|
||||
* Sole constructor.
|
||||
*/
|
||||
public PercentileRanksBuilder(String name) {
|
||||
super(name, InternalPercentileRanks.TYPE.name());
|
||||
super(name, PercentileRanks.TYPE_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -46,24 +44,11 @@ public class PercentileRanksBuilder extends ValuesSourceMetricsAggregationBuilde
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: set the compression. Higher values improve accuracy but also memory usage.
|
||||
*/
|
||||
public PercentileRanksBuilder compression(double compression) {
|
||||
this.compression = compression;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void internalXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
super.internalXContent(builder, params);
|
||||
protected void doInternalXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
|
||||
if (values != null) {
|
||||
builder.field("values", values);
|
||||
}
|
||||
|
||||
if (compression != null) {
|
||||
builder.field("compression", compression);
|
||||
builder.field(PercentileRanksParser.VALUES_FIELD.getPreferredName(), values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,8 +18,12 @@
|
|||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.search.SearchParseException;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactory;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentileRanksAggregator;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentileRanksAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
@ -29,26 +33,36 @@ import org.elasticsearch.search.internal.SearchContext;
|
|||
*/
|
||||
public class PercentileRanksParser extends AbstractPercentilesParser {
|
||||
|
||||
public static final ParseField VALUES_FIELD = new ParseField("values");
|
||||
|
||||
public PercentileRanksParser() {
|
||||
super(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return InternalPercentileRanks.TYPE.name();
|
||||
return InternalTDigestPercentileRanks.TYPE.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String keysFieldName() {
|
||||
return "values";
|
||||
protected ParseField keysField() {
|
||||
return VALUES_FIELD;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> valuesSourceConfig, double[] keys, double compression, boolean keyed) {
|
||||
protected AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> valuesSourceConfig,
|
||||
double[] keys, PercentilesMethod method, Double compression, Integer numberOfSignificantValueDigits, boolean keyed) {
|
||||
if (keys == null) {
|
||||
throw new SearchParseException(context, "Missing token values in [" + aggregationName + "].", null);
|
||||
}
|
||||
return new PercentileRanksAggregator.Factory(aggregationName, valuesSourceConfig, keys, compression, keyed);
|
||||
if (method == PercentilesMethod.TDIGEST) {
|
||||
return new TDigestPercentileRanksAggregator.Factory(aggregationName, valuesSourceConfig, keys, compression, keyed);
|
||||
} else if (method == PercentilesMethod.HDR) {
|
||||
return new HDRPercentileRanksAggregator.Factory(aggregationName, valuesSourceConfig, keys, numberOfSignificantValueDigits,
|
||||
keyed);
|
||||
} else {
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -25,6 +25,8 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
|
|||
*/
|
||||
public interface Percentiles extends NumericMetricsAggregation.MultiValue, Iterable<Percentile> {
|
||||
|
||||
public static final String TYPE_NAME = "percentiles";
|
||||
|
||||
/**
|
||||
* Return the value associated with the provided percentile.
|
||||
*/
|
||||
|
|
|
@ -19,23 +19,21 @@
|
|||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.search.aggregations.metrics.ValuesSourceMetricsAggregationBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
||||
/**
|
||||
* Builder for the {@link Percentiles} aggregation.
|
||||
*/
|
||||
public class PercentilesBuilder extends ValuesSourceMetricsAggregationBuilder<PercentilesBuilder> {
|
||||
|
||||
private double[] percentiles;
|
||||
private Double compression;
|
||||
public class PercentilesBuilder extends AbstractPercentilesBuilder<PercentilesBuilder> {
|
||||
|
||||
double[] percentiles;
|
||||
/**
|
||||
* Sole constructor.
|
||||
*/
|
||||
public PercentilesBuilder(String name) {
|
||||
super(name, InternalPercentiles.TYPE.name());
|
||||
super(name, Percentiles.TYPE_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -52,24 +50,11 @@ public class PercentilesBuilder extends ValuesSourceMetricsAggregationBuilder<Pe
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: set the compression. Higher values improve accuracy but also memory usage.
|
||||
*/
|
||||
public PercentilesBuilder compression(double compression) {
|
||||
this.compression = compression;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void internalXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
super.internalXContent(builder, params);
|
||||
|
||||
protected void doInternalXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
if (percentiles != null) {
|
||||
builder.field("percents", percentiles);
|
||||
}
|
||||
|
||||
if (compression != null) {
|
||||
builder.field("compression", compression);
|
||||
builder.field(PercentilesParser.PERCENTS_FIELD.getPreferredName(), percentiles);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
|
||||
/**
|
||||
* An enum representing the methods for calculating percentiles
|
||||
*/
|
||||
public enum PercentilesMethod {
|
||||
/**
|
||||
* The TDigest method for calculating percentiles
|
||||
*/
|
||||
TDIGEST("tdigest"),
|
||||
/**
|
||||
* The HDRHistogram method of calculating percentiles
|
||||
*/
|
||||
HDR("hdr");
|
||||
|
||||
private String name;
|
||||
|
||||
private PercentilesMethod(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the name of the method
|
||||
*/
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link PercentilesMethod} for this method name. returns
|
||||
* <code>null</code> if no {@link PercentilesMethod} exists for the name.
|
||||
*/
|
||||
public static PercentilesMethod resolveFromName(String name) {
|
||||
for (PercentilesMethod method : values()) {
|
||||
if (method.name.equalsIgnoreCase(name)) {
|
||||
return method;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
|
@ -18,7 +18,11 @@
|
|||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactory;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentilesAggregator;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentilesAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
@ -28,6 +32,8 @@ import org.elasticsearch.search.internal.SearchContext;
|
|||
*/
|
||||
public class PercentilesParser extends AbstractPercentilesParser {
|
||||
|
||||
public static final ParseField PERCENTS_FIELD = new ParseField("percents");
|
||||
|
||||
public PercentilesParser() {
|
||||
super(true);
|
||||
}
|
||||
|
@ -36,20 +42,27 @@ public class PercentilesParser extends AbstractPercentilesParser {
|
|||
|
||||
@Override
|
||||
public String type() {
|
||||
return InternalPercentiles.TYPE.name();
|
||||
return InternalTDigestPercentiles.TYPE.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String keysFieldName() {
|
||||
return "percents";
|
||||
protected ParseField keysField() {
|
||||
return PERCENTS_FIELD;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> valuesSourceConfig, double[] keys, double compression, boolean keyed) {
|
||||
protected AggregatorFactory buildFactory(SearchContext context, String aggregationName, ValuesSourceConfig<Numeric> valuesSourceConfig,
|
||||
double[] keys, PercentilesMethod method, Double compression, Integer numberOfSignificantValueDigits, boolean keyed) {
|
||||
if (keys == null) {
|
||||
keys = DEFAULT_PERCENTS;
|
||||
}
|
||||
return new PercentilesAggregator.Factory(aggregationName, valuesSourceConfig, keys, compression, keyed);
|
||||
if (method == PercentilesMethod.TDIGEST) {
|
||||
return new TDigestPercentilesAggregator.Factory(aggregationName, valuesSourceConfig, keys, compression, keyed);
|
||||
} else if (method == PercentilesMethod.HDR) {
|
||||
return new HDRPercentilesAggregator.Factory(aggregationName, valuesSourceConfig, keys, numberOfSignificantValueDigits, keyed);
|
||||
} else {
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,125 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.elasticsearch.common.lease.Releasables;
|
||||
import org.elasticsearch.common.util.ArrayUtils;
|
||||
import org.elasticsearch.common.util.BigArrays;
|
||||
import org.elasticsearch.common.util.ObjectArray;
|
||||
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.LeafBucketCollector;
|
||||
import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
|
||||
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAggregator.MultiValue {
|
||||
|
||||
private static int indexOfKey(double[] keys, double key) {
|
||||
return ArrayUtils.binarySearch(keys, key, 0.001);
|
||||
}
|
||||
|
||||
protected final double[] keys;
|
||||
protected final ValuesSource.Numeric valuesSource;
|
||||
protected final ValueFormatter formatter;
|
||||
protected ObjectArray<DoubleHistogram> states;
|
||||
protected final int numberOfSignificantValueDigits;
|
||||
protected final boolean keyed;
|
||||
|
||||
public AbstractHDRPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, AggregationContext context, Aggregator parent,
|
||||
double[] keys, int numberOfSignificantValueDigits, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
super(name, context, parent, pipelineAggregators, metaData);
|
||||
this.valuesSource = valuesSource;
|
||||
this.keyed = keyed;
|
||||
this.formatter = formatter;
|
||||
this.states = context.bigArrays().newObjectArray(1);
|
||||
this.keys = keys;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needsScores() {
|
||||
return valuesSource != null && valuesSource.needsScores();
|
||||
}
|
||||
|
||||
@Override
|
||||
public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
|
||||
final LeafBucketCollector sub) throws IOException {
|
||||
if (valuesSource == null) {
|
||||
return LeafBucketCollector.NO_OP_COLLECTOR;
|
||||
}
|
||||
final BigArrays bigArrays = context.bigArrays();
|
||||
final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx);
|
||||
return new LeafBucketCollectorBase(sub, values) {
|
||||
@Override
|
||||
public void collect(int doc, long bucket) throws IOException {
|
||||
states = bigArrays.grow(states, bucket + 1);
|
||||
|
||||
DoubleHistogram state = states.get(bucket);
|
||||
if (state == null) {
|
||||
state = new DoubleHistogram(numberOfSignificantValueDigits);
|
||||
// Set the histogram to autosize so it can resize itself as
|
||||
// the data range increases. Resize operations should be
|
||||
// rare as the histogram buckets are exponential (on the top
|
||||
// level). In the future we could expose the range as an
|
||||
// option on the request so the histogram can be fixed at
|
||||
// initialisation and doesn't need resizing.
|
||||
state.setAutoResize(true);
|
||||
states.set(bucket, state);
|
||||
}
|
||||
|
||||
values.setDocument(doc);
|
||||
final int valueCount = values.count();
|
||||
for (int i = 0; i < valueCount; i++) {
|
||||
state.recordValue(values.valueAt(i));
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasMetric(String name) {
|
||||
return indexOfKey(keys, Double.parseDouble(name)) >= 0;
|
||||
}
|
||||
|
||||
protected DoubleHistogram getState(long bucketOrd) {
|
||||
if (bucketOrd >= states.size()) {
|
||||
return null;
|
||||
}
|
||||
final DoubleHistogram state = states.get(bucketOrd);
|
||||
return state;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doClose() {
|
||||
Releasables.close(states);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,144 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatterStreams;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.DataFormatException;
|
||||
|
||||
abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggregation.MultiValue {
|
||||
|
||||
protected double[] keys;
|
||||
protected DoubleHistogram state;
|
||||
private boolean keyed;
|
||||
|
||||
AbstractInternalHDRPercentiles() {} // for serialization
|
||||
|
||||
public AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators,
|
||||
Map<String, Object> metaData) {
|
||||
super(name, pipelineAggregators, metaData);
|
||||
this.keys = keys;
|
||||
this.state = state;
|
||||
this.keyed = keyed;
|
||||
this.valueFormatter = formatter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double value(String name) {
|
||||
return value(Double.parseDouble(name));
|
||||
}
|
||||
|
||||
public abstract double value(double key);
|
||||
|
||||
public long getEstimatedMemoryFootprint() {
|
||||
return state.getEstimatedFootprintInBytes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractInternalHDRPercentiles doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
|
||||
DoubleHistogram merged = null;
|
||||
for (InternalAggregation aggregation : aggregations) {
|
||||
final AbstractInternalHDRPercentiles percentiles = (AbstractInternalHDRPercentiles) aggregation;
|
||||
if (merged == null) {
|
||||
merged = new DoubleHistogram(percentiles.state);
|
||||
merged.setAutoResize(true);
|
||||
}
|
||||
merged.add(percentiles.state);
|
||||
}
|
||||
return createReduced(getName(), keys, merged, keyed, pipelineAggregators(), getMetaData());
|
||||
}
|
||||
|
||||
protected abstract AbstractInternalHDRPercentiles createReduced(String name, double[] keys, DoubleHistogram merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
|
||||
|
||||
@Override
|
||||
protected void doReadFrom(StreamInput in) throws IOException {
|
||||
valueFormatter = ValueFormatterStreams.readOptional(in);
|
||||
keys = new double[in.readInt()];
|
||||
for (int i = 0; i < keys.length; ++i) {
|
||||
keys[i] = in.readDouble();
|
||||
}
|
||||
long minBarForHighestToLowestValueRatio = in.readLong();
|
||||
ByteBuffer stateBuffer = ByteBuffer.wrap(in.readByteArray());
|
||||
try {
|
||||
state = DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
|
||||
} catch (DataFormatException e) {
|
||||
throw new IOException("Failed to decode DoubleHistogram for aggregation [" + name + "]", e);
|
||||
}
|
||||
keyed = in.readBoolean();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doWriteTo(StreamOutput out) throws IOException {
|
||||
ValueFormatterStreams.writeOptional(valueFormatter, out);
|
||||
out.writeInt(keys.length);
|
||||
for (int i = 0 ; i < keys.length; ++i) {
|
||||
out.writeDouble(keys[i]);
|
||||
}
|
||||
out.writeLong(state.getHighestToLowestValueRatio());
|
||||
ByteBuffer stateBuffer = ByteBuffer.allocate(state.getNeededByteBufferCapacity());
|
||||
state.encodeIntoCompressedByteBuffer(stateBuffer);
|
||||
out.writeByteArray(stateBuffer.array());
|
||||
out.writeBoolean(keyed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
|
||||
if (keyed) {
|
||||
builder.startObject(CommonFields.VALUES);
|
||||
for(int i = 0; i < keys.length; ++i) {
|
||||
String key = String.valueOf(keys[i]);
|
||||
double value = value(keys[i]);
|
||||
builder.field(key, value);
|
||||
if (!(valueFormatter instanceof ValueFormatter.Raw)) {
|
||||
builder.field(key + "_as_string", valueFormatter.format(value));
|
||||
}
|
||||
}
|
||||
builder.endObject();
|
||||
} else {
|
||||
builder.startArray(CommonFields.VALUES);
|
||||
for (int i = 0; i < keys.length; i++) {
|
||||
double value = value(keys[i]);
|
||||
builder.startObject();
|
||||
builder.field(CommonFields.KEY, keys[i]);
|
||||
builder.field(CommonFields.VALUE, value);
|
||||
if (!(valueFormatter instanceof ValueFormatter.Raw)) {
|
||||
builder.field(CommonFields.VALUE_AS_STRING, valueFormatter.format(value));
|
||||
}
|
||||
builder.endObject();
|
||||
}
|
||||
builder.endArray();
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class HDRPercentileRanksAggregator extends AbstractHDRPercentilesAggregator {
|
||||
|
||||
public HDRPercentileRanksAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent,
|
||||
double[] percents, int numberOfSignificantValueDigits, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter, pipelineAggregators,
|
||||
metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InternalAggregation buildAggregation(long owningBucketOrdinal) {
|
||||
DoubleHistogram state = getState(owningBucketOrdinal);
|
||||
if (state == null) {
|
||||
return buildEmptyAggregation();
|
||||
} else {
|
||||
return new InternalHDRPercentileRanks(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InternalAggregation buildEmptyAggregation() {
|
||||
DoubleHistogram state;
|
||||
state = new DoubleHistogram(numberOfSignificantValueDigits);
|
||||
state.setAutoResize(true);
|
||||
return new InternalHDRPercentileRanks(name, keys, state,
|
||||
keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
|
||||
@Override
|
||||
public double metric(String name, long bucketOrd) {
|
||||
DoubleHistogram state = getState(bucketOrd);
|
||||
if (state == null) {
|
||||
return Double.NaN;
|
||||
} else {
|
||||
return InternalHDRPercentileRanks.percentileRank(state, Double.valueOf(name));
|
||||
}
|
||||
}
|
||||
|
||||
public static class Factory extends ValuesSourceAggregatorFactory.LeafOnly<ValuesSource.Numeric> {
|
||||
|
||||
private final double[] values;
|
||||
private final int numberOfSignificantValueDigits;
|
||||
private final boolean keyed;
|
||||
|
||||
public Factory(String name, ValuesSourceConfig<ValuesSource.Numeric> valuesSourceConfig, double[] values,
|
||||
int numberOfSignificantValueDigits, boolean keyed) {
|
||||
super(name, InternalHDRPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
this.values = values;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.keyed = keyed;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
return new HDRPercentileRanksAggregator(name, null, aggregationContext, parent, values, numberOfSignificantValueDigits, keyed,
|
||||
config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, AggregationContext aggregationContext, Aggregator parent,
|
||||
boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
|
||||
throws IOException {
|
||||
return new HDRPercentileRanksAggregator(name, valuesSource, aggregationContext, parent, values, numberOfSignificantValueDigits,
|
||||
keyed, config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class HDRPercentilesAggregator extends AbstractHDRPercentilesAggregator {
|
||||
|
||||
public HDRPercentilesAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent, double[] percents,
|
||||
int numberOfSignificantValueDigits, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter,
|
||||
pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InternalAggregation buildAggregation(long owningBucketOrdinal) {
|
||||
DoubleHistogram state = getState(owningBucketOrdinal);
|
||||
if (state == null) {
|
||||
return buildEmptyAggregation();
|
||||
} else {
|
||||
return new InternalHDRPercentiles(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public double metric(String name, long bucketOrd) {
|
||||
DoubleHistogram state = getState(bucketOrd);
|
||||
if (state == null) {
|
||||
return Double.NaN;
|
||||
} else {
|
||||
return state.getValueAtPercentile(Double.parseDouble(name));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InternalAggregation buildEmptyAggregation() {
|
||||
DoubleHistogram state;
|
||||
state = new DoubleHistogram(numberOfSignificantValueDigits);
|
||||
state.setAutoResize(true);
|
||||
return new InternalHDRPercentiles(name, keys, state,
|
||||
keyed,
|
||||
formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
|
||||
public static class Factory extends ValuesSourceAggregatorFactory.LeafOnly<ValuesSource.Numeric> {
|
||||
|
||||
private final double[] percents;
|
||||
private final int numberOfSignificantValueDigits;
|
||||
private final boolean keyed;
|
||||
|
||||
public Factory(String name, ValuesSourceConfig<ValuesSource.Numeric> valuesSourceConfig, double[] percents,
|
||||
int numberOfSignificantValueDigits, boolean keyed) {
|
||||
super(name, InternalTDigestPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
this.percents = percents;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.keyed = keyed;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
return new HDRPercentilesAggregator(name, null, aggregationContext, parent, percents, numberOfSignificantValueDigits, keyed,
|
||||
config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, AggregationContext aggregationContext, Aggregator parent,
|
||||
boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
|
||||
throws IOException {
|
||||
return new HDRPercentilesAggregator(name, valuesSource, aggregationContext, parent, percents, numberOfSignificantValueDigits,
|
||||
keyed, config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import com.google.common.collect.UnmodifiableIterator;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.search.aggregations.AggregationStreams;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles implements PercentileRanks {
|
||||
|
||||
public final static Type TYPE = new Type(PercentileRanks.TYPE_NAME, "hdr_percentile_ranks");
|
||||
|
||||
public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
|
||||
@Override
|
||||
public InternalHDRPercentileRanks readResult(StreamInput in) throws IOException {
|
||||
InternalHDRPercentileRanks result = new InternalHDRPercentileRanks();
|
||||
result.readFrom(in);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
public static void registerStreams() {
|
||||
AggregationStreams.registerStream(STREAM, TYPE.stream());
|
||||
}
|
||||
|
||||
InternalHDRPercentileRanks() {
|
||||
} // for serialization
|
||||
|
||||
public InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Percentile> iterator() {
|
||||
return new Iter(keys, state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public double percent(double value) {
|
||||
return percentileRank(state, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String percentAsString(double value) {
|
||||
return valueAsString(String.valueOf(value));
|
||||
}
|
||||
|
||||
@Override
|
||||
public double value(double key) {
|
||||
return percent(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AbstractInternalHDRPercentiles createReduced(String name, double[] keys, DoubleHistogram merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
return new InternalHDRPercentileRanks(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Type type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
static double percentileRank(DoubleHistogram state, double value) {
|
||||
if (state.getTotalCount() == 0) {
|
||||
return Double.NaN;
|
||||
}
|
||||
double percentileRank = state.getPercentileAtOrBelowValue(value);
|
||||
if (percentileRank < 0) {
|
||||
percentileRank = 0;
|
||||
} else if (percentileRank > 100) {
|
||||
percentileRank = 100;
|
||||
}
|
||||
return percentileRank;
|
||||
}
|
||||
|
||||
public static class Iter extends UnmodifiableIterator<Percentile> {
|
||||
|
||||
private final double[] values;
|
||||
private final DoubleHistogram state;
|
||||
private int i;
|
||||
|
||||
public Iter(double[] values, DoubleHistogram state) {
|
||||
this.values = values;
|
||||
this.state = state;
|
||||
i = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return i < values.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Percentile next() {
|
||||
final Percentile next = new InternalPercentile(percentileRank(state, values[i]), values[i]);
|
||||
++i;
|
||||
return next;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,123 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
|
||||
|
||||
import com.google.common.collect.UnmodifiableIterator;
|
||||
|
||||
import org.HdrHistogram.DoubleHistogram;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.search.aggregations.AggregationStreams;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles implements Percentiles {
|
||||
|
||||
public final static Type TYPE = new Type(Percentiles.TYPE_NAME, "hdr_percentiles");
|
||||
|
||||
public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
|
||||
@Override
|
||||
public InternalHDRPercentiles readResult(StreamInput in) throws IOException {
|
||||
InternalHDRPercentiles result = new InternalHDRPercentiles();
|
||||
result.readFrom(in);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
public static void registerStreams() {
|
||||
AggregationStreams.registerStream(STREAM, TYPE.stream());
|
||||
}
|
||||
|
||||
InternalHDRPercentiles() {
|
||||
} // for serialization
|
||||
|
||||
public InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
super(name, percents, state, keyed, formatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Percentile> iterator() {
|
||||
return new Iter(keys, state);
|
||||
}
|
||||
|
||||
@Override
|
||||
public double percentile(double percent) {
|
||||
if (state.getTotalCount() == 0) {
|
||||
return Double.NaN;
|
||||
}
|
||||
return state.getValueAtPercentile(percent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String percentileAsString(double percent) {
|
||||
return valueAsString(String.valueOf(percent));
|
||||
}
|
||||
|
||||
@Override
|
||||
public double value(double key) {
|
||||
return percentile(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AbstractInternalHDRPercentiles createReduced(String name, double[] keys, DoubleHistogram merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
return new InternalHDRPercentiles(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Type type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
public static class Iter extends UnmodifiableIterator<Percentile> {
|
||||
|
||||
private final double[] percents;
|
||||
private final DoubleHistogram state;
|
||||
private int i;
|
||||
|
||||
public Iter(double[] percents, DoubleHistogram state) {
|
||||
this.percents = percents;
|
||||
this.state = state;
|
||||
i = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return i < percents.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Percentile next() {
|
||||
final Percentile next = new InternalPercentile(percents[i], state.getValueAtPercentile(percents[i]));
|
||||
++i;
|
||||
return next;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -17,14 +17,13 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatterStreams;
|
||||
|
@ -33,15 +32,15 @@ import java.io.IOException;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
abstract class AbstractInternalPercentiles extends InternalNumericMetricsAggregation.MultiValue {
|
||||
abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetricsAggregation.MultiValue {
|
||||
|
||||
protected double[] keys;
|
||||
protected TDigestState state;
|
||||
private boolean keyed;
|
||||
|
||||
AbstractInternalPercentiles() {} // for serialization
|
||||
AbstractInternalTDigestPercentiles() {} // for serialization
|
||||
|
||||
public AbstractInternalPercentiles(String name, double[] keys, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
public AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators,
|
||||
Map<String, Object> metaData) {
|
||||
super(name, pipelineAggregators, metaData);
|
||||
|
@ -58,11 +57,15 @@ abstract class AbstractInternalPercentiles extends InternalNumericMetricsAggrega
|
|||
|
||||
public abstract double value(double key);
|
||||
|
||||
public long getEstimatedMemoryFootprint() {
|
||||
return state.byteSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractInternalPercentiles doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
|
||||
public AbstractInternalTDigestPercentiles doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
|
||||
TDigestState merged = null;
|
||||
for (InternalAggregation aggregation : aggregations) {
|
||||
final AbstractInternalPercentiles percentiles = (AbstractInternalPercentiles) aggregation;
|
||||
final AbstractInternalTDigestPercentiles percentiles = (AbstractInternalTDigestPercentiles) aggregation;
|
||||
if (merged == null) {
|
||||
merged = new TDigestState(percentiles.state.compression());
|
||||
}
|
||||
|
@ -71,7 +74,7 @@ abstract class AbstractInternalPercentiles extends InternalNumericMetricsAggrega
|
|||
return createReduced(getName(), keys, merged, keyed, pipelineAggregators(), getMetaData());
|
||||
}
|
||||
|
||||
protected abstract AbstractInternalPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
protected abstract AbstractInternalTDigestPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
|
||||
|
||||
@Override
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.elasticsearch.common.lease.Releasables;
|
||||
|
@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
|
|||
import org.elasticsearch.search.aggregations.LeafBucketCollector;
|
||||
import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
|
||||
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
|
@ -39,7 +38,7 @@ import java.io.IOException;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public abstract class AbstractPercentilesAggregator extends NumericMetricsAggregator.MultiValue {
|
||||
public abstract class AbstractTDigestPercentilesAggregator extends NumericMetricsAggregator.MultiValue {
|
||||
|
||||
private static int indexOfKey(double[] keys, double key) {
|
||||
return ArrayUtils.binarySearch(keys, key, 0.001);
|
||||
|
@ -52,7 +51,7 @@ public abstract class AbstractPercentilesAggregator extends NumericMetricsAggreg
|
|||
protected final double compression;
|
||||
protected final boolean keyed;
|
||||
|
||||
public AbstractPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, AggregationContext context, Aggregator parent,
|
||||
public AbstractTDigestPercentilesAggregator(String name, ValuesSource.Numeric valuesSource, AggregationContext context, Aggregator parent,
|
||||
double[] keys, double compression, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
super(name, context, parent, pipelineAggregators, metaData);
|
|
@ -16,13 +16,15 @@
|
|||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import com.google.common.collect.UnmodifiableIterator;
|
||||
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.search.aggregations.AggregationStreams;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
|
@ -34,14 +36,14 @@ import java.util.Map;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class InternalPercentileRanks extends AbstractInternalPercentiles implements PercentileRanks {
|
||||
public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPercentiles implements PercentileRanks {
|
||||
|
||||
public final static Type TYPE = new Type("percentile_ranks");
|
||||
public final static Type TYPE = new Type(PercentileRanks.TYPE_NAME, "t_digest_percentile_ranks");
|
||||
|
||||
public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
|
||||
@Override
|
||||
public InternalPercentileRanks readResult(StreamInput in) throws IOException {
|
||||
InternalPercentileRanks result = new InternalPercentileRanks();
|
||||
public InternalTDigestPercentileRanks readResult(StreamInput in) throws IOException {
|
||||
InternalTDigestPercentileRanks result = new InternalTDigestPercentileRanks();
|
||||
result.readFrom(in);
|
||||
return result;
|
||||
}
|
||||
|
@ -51,9 +53,9 @@ public class InternalPercentileRanks extends AbstractInternalPercentiles impleme
|
|||
AggregationStreams.registerStream(STREAM, TYPE.stream());
|
||||
}
|
||||
|
||||
InternalPercentileRanks() {} // for serialization
|
||||
InternalTDigestPercentileRanks() {} // for serialization
|
||||
|
||||
public InternalPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
public InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
@ -79,9 +81,9 @@ public class InternalPercentileRanks extends AbstractInternalPercentiles impleme
|
|||
}
|
||||
|
||||
@Override
|
||||
protected AbstractInternalPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
protected AbstractInternalTDigestPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
return new InternalPercentileRanks(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
return new InternalTDigestPercentileRanks(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
|
@ -16,13 +16,15 @@
|
|||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import com.google.common.collect.UnmodifiableIterator;
|
||||
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.search.aggregations.AggregationStreams;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.format.ValueFormatter;
|
||||
|
||||
|
@ -34,14 +36,14 @@ import java.util.Map;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class InternalPercentiles extends AbstractInternalPercentiles implements Percentiles {
|
||||
public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentiles implements Percentiles {
|
||||
|
||||
public final static Type TYPE = new Type("percentiles");
|
||||
public final static Type TYPE = new Type(Percentiles.TYPE_NAME, "t_digest_percentiles");
|
||||
|
||||
public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
|
||||
@Override
|
||||
public InternalPercentiles readResult(StreamInput in) throws IOException {
|
||||
InternalPercentiles result = new InternalPercentiles();
|
||||
public InternalTDigestPercentiles readResult(StreamInput in) throws IOException {
|
||||
InternalTDigestPercentiles result = new InternalTDigestPercentiles();
|
||||
result.readFrom(in);
|
||||
return result;
|
||||
}
|
||||
|
@ -51,10 +53,10 @@ public class InternalPercentiles extends AbstractInternalPercentiles implements
|
|||
AggregationStreams.registerStream(STREAM, TYPE.stream());
|
||||
}
|
||||
|
||||
InternalPercentiles() {
|
||||
InternalTDigestPercentiles() {
|
||||
} // for serialization
|
||||
|
||||
public InternalPercentiles(String name, double[] percents, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
public InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, ValueFormatter formatter,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
super(name, percents, state, keyed, formatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
@ -80,9 +82,9 @@ public class InternalPercentiles extends AbstractInternalPercentiles implements
|
|||
}
|
||||
|
||||
@Override
|
||||
protected AbstractInternalPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
protected AbstractInternalTDigestPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
|
||||
return new InternalPercentiles(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
return new InternalTDigestPercentiles(name, keys, merged, keyed, valueFormatter, pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
@Override
|
|
@ -16,11 +16,10 @@
|
|||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
|
@ -36,9 +35,9 @@ import java.util.Map;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
||||
public class TDigestPercentileRanksAggregator extends AbstractTDigestPercentilesAggregator {
|
||||
|
||||
public PercentileRanksAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent, double[] percents,
|
||||
public TDigestPercentileRanksAggregator(String name, Numeric valuesSource, AggregationContext context, Aggregator parent, double[] percents,
|
||||
double compression, boolean keyed, ValueFormatter formatter, List<PipelineAggregator> pipelineAggregators,
|
||||
Map<String, Object> metaData)
|
||||
throws IOException {
|
||||
|
@ -51,13 +50,13 @@ public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
|||
if (state == null) {
|
||||
return buildEmptyAggregation();
|
||||
} else {
|
||||
return new InternalPercentileRanks(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
return new InternalTDigestPercentileRanks(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InternalAggregation buildEmptyAggregation() {
|
||||
return new InternalPercentileRanks(name, keys, new TDigestState(compression), keyed, formatter, pipelineAggregators(), metaData());
|
||||
return new InternalTDigestPercentileRanks(name, keys, new TDigestState(compression), keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -66,7 +65,7 @@ public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
|||
if (state == null) {
|
||||
return Double.NaN;
|
||||
} else {
|
||||
return InternalPercentileRanks.percentileRank(state, Double.valueOf(name));
|
||||
return InternalTDigestPercentileRanks.percentileRank(state, Double.valueOf(name));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -78,7 +77,7 @@ public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
|||
|
||||
public Factory(String name, ValuesSourceConfig<ValuesSource.Numeric> valuesSourceConfig,
|
||||
double[] values, double compression, boolean keyed) {
|
||||
super(name, InternalPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
super(name, InternalTDigestPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
this.values = values;
|
||||
this.compression = compression;
|
||||
this.keyed = keyed;
|
||||
|
@ -87,7 +86,7 @@ public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
|||
@Override
|
||||
protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
return new PercentileRanksAggregator(name, null, aggregationContext, parent, values, compression, keyed, config.formatter(),
|
||||
return new TDigestPercentileRanksAggregator(name, null, aggregationContext, parent, values, compression, keyed, config.formatter(),
|
||||
pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
|
@ -95,7 +94,7 @@ public class PercentileRanksAggregator extends AbstractPercentilesAggregator {
|
|||
protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, AggregationContext aggregationContext, Aggregator parent,
|
||||
boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
|
||||
throws IOException {
|
||||
return new PercentileRanksAggregator(name, valuesSource, aggregationContext, parent, values, compression, keyed,
|
||||
return new TDigestPercentileRanksAggregator(name, valuesSource, aggregationContext, parent, values, compression, keyed,
|
||||
config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
}
|
|
@ -16,11 +16,10 @@
|
|||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles;
|
||||
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
|
||||
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.AggregationContext;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
|
@ -36,9 +35,9 @@ import java.util.Map;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
||||
public class TDigestPercentilesAggregator extends AbstractTDigestPercentilesAggregator {
|
||||
|
||||
public PercentilesAggregator(String name, Numeric valuesSource, AggregationContext context,
|
||||
public TDigestPercentilesAggregator(String name, Numeric valuesSource, AggregationContext context,
|
||||
Aggregator parent, double[] percents,
|
||||
double compression, boolean keyed, ValueFormatter formatter, List<PipelineAggregator> pipelineAggregators,
|
||||
Map<String, Object> metaData) throws IOException {
|
||||
|
@ -51,7 +50,7 @@ public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
|||
if (state == null) {
|
||||
return buildEmptyAggregation();
|
||||
} else {
|
||||
return new InternalPercentiles(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
return new InternalTDigestPercentiles(name, keys, state, keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -67,7 +66,7 @@ public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
|||
|
||||
@Override
|
||||
public InternalAggregation buildEmptyAggregation() {
|
||||
return new InternalPercentiles(name, keys, new TDigestState(compression), keyed, formatter, pipelineAggregators(), metaData());
|
||||
return new InternalTDigestPercentiles(name, keys, new TDigestState(compression), keyed, formatter, pipelineAggregators(), metaData());
|
||||
}
|
||||
|
||||
public static class Factory extends ValuesSourceAggregatorFactory.LeafOnly<ValuesSource.Numeric> {
|
||||
|
@ -78,7 +77,7 @@ public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
|||
|
||||
public Factory(String name, ValuesSourceConfig<ValuesSource.Numeric> valuesSourceConfig,
|
||||
double[] percents, double compression, boolean keyed) {
|
||||
super(name, InternalPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
super(name, InternalTDigestPercentiles.TYPE.name(), valuesSourceConfig);
|
||||
this.percents = percents;
|
||||
this.compression = compression;
|
||||
this.keyed = keyed;
|
||||
|
@ -87,7 +86,7 @@ public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
|||
@Override
|
||||
protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent,
|
||||
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
|
||||
return new PercentilesAggregator(name, null, aggregationContext, parent, percents, compression, keyed, config.formatter(),
|
||||
return new TDigestPercentilesAggregator(name, null, aggregationContext, parent, percents, compression, keyed, config.formatter(),
|
||||
pipelineAggregators, metaData);
|
||||
}
|
||||
|
||||
|
@ -95,7 +94,7 @@ public class PercentilesAggregator extends AbstractPercentilesAggregator {
|
|||
protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, AggregationContext aggregationContext, Aggregator parent,
|
||||
boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
|
||||
throws IOException {
|
||||
return new PercentilesAggregator(name, valuesSource, aggregationContext, parent, percents, compression, keyed,
|
||||
return new TDigestPercentilesAggregator(name, valuesSource, aggregationContext, parent, percents, compression, keyed,
|
||||
config.formatter(), pipelineAggregators, metaData);
|
||||
}
|
||||
}
|
|
@ -49,6 +49,7 @@ import org.junit.Test;
|
|||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
@ -335,11 +336,11 @@ public class TermVectorsUnitTests extends ElasticsearchTestCase {
|
|||
}
|
||||
|
||||
void checkParsedFilterParameters(MultiTermVectorsRequest multiRequest) {
|
||||
int id = 1;
|
||||
Set<String> ids = new HashSet<>(Arrays.asList("1", "2"));
|
||||
for (TermVectorsRequest request : multiRequest.requests) {
|
||||
assertThat(request.index(), equalTo("testidx"));
|
||||
assertThat(request.type(), equalTo("test"));
|
||||
assertThat(request.id(), equalTo(id+""));
|
||||
assertTrue(ids.remove(request.id()));
|
||||
assertNotNull(request.filterSettings());
|
||||
assertThat(request.filterSettings().maxNumTerms, equalTo(20));
|
||||
assertThat(request.filterSettings().minTermFreq, equalTo(1));
|
||||
|
@ -348,7 +349,7 @@ public class TermVectorsUnitTests extends ElasticsearchTestCase {
|
|||
assertThat(request.filterSettings().maxDocFreq, equalTo(20));
|
||||
assertThat(request.filterSettings().minWordLength, equalTo(1));
|
||||
assertThat(request.filterSettings().maxWordLength, equalTo(20));
|
||||
id++;
|
||||
}
|
||||
assertTrue(ids.isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.benchmark.search.aggregations;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomInts;
|
||||
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.StopWatch;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.SizeUnit;
|
||||
import org.elasticsearch.common.unit.SizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
|
||||
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
|
||||
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
|
||||
|
||||
public class HDRPercentilesAggregationBenchmark {
|
||||
|
||||
private static final String TYPE_NAME = "type";
|
||||
private static final String INDEX_NAME = "index";
|
||||
private static final String HIGH_CARD_FIELD_NAME = "high_card";
|
||||
private static final String LOW_CARD_FIELD_NAME = "low_card";
|
||||
private static final String GAUSSIAN_FIELD_NAME = "gauss";
|
||||
private static final Random R = new Random();
|
||||
private static final String CLUSTER_NAME = HDRPercentilesAggregationBenchmark.class.getSimpleName();
|
||||
private static final int NUM_DOCS = 10000000;
|
||||
private static final int LOW_CARD = 1000;
|
||||
private static final int HIGH_CARD = 1000000;
|
||||
private static final int BATCH = 100;
|
||||
private static final int WARM = 5;
|
||||
private static final int RUNS = 10;
|
||||
private static final int ITERS = 5;
|
||||
|
||||
public static void main(String[] args) {
|
||||
long overallStartTime = System.currentTimeMillis();
|
||||
Settings settings = settingsBuilder()
|
||||
.put("index.refresh_interval", "-1")
|
||||
.put(SETTING_NUMBER_OF_SHARDS, 5)
|
||||
.put(SETTING_NUMBER_OF_REPLICAS, 0)
|
||||
.build();
|
||||
|
||||
Node[] nodes = new Node[1];
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
nodes[i] = nodeBuilder().clusterName(CLUSTER_NAME)
|
||||
.settings(settingsBuilder().put(settings).put("name", "node" + i))
|
||||
.node();
|
||||
}
|
||||
|
||||
Node clientNode = nodeBuilder()
|
||||
.clusterName(CLUSTER_NAME)
|
||||
.settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();
|
||||
|
||||
Client client = clientNode.client();
|
||||
|
||||
try {
|
||||
client.admin().indices().prepareCreate(INDEX_NAME);
|
||||
|
||||
System.out.println("Indexing " + NUM_DOCS + " documents");
|
||||
|
||||
StopWatch stopWatch = new StopWatch().start();
|
||||
for (int i = 0; i < NUM_DOCS; ) {
|
||||
BulkRequestBuilder request = client.prepareBulk();
|
||||
for (int j = 0; j < BATCH && i < NUM_DOCS; ++j) {
|
||||
final int lowCard = RandomInts.randomInt(R, LOW_CARD);
|
||||
final int highCard = RandomInts.randomInt(R, HIGH_CARD);
|
||||
int gauss = -1;
|
||||
while (gauss < 0) {
|
||||
gauss = (int) (R.nextGaussian() * 1000) + 5000; // mean: 5 sec, std deviation: 1 sec
|
||||
}
|
||||
request.add(client.prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i)).setSource(LOW_CARD_FIELD_NAME, lowCard,
|
||||
HIGH_CARD_FIELD_NAME, highCard, GAUSSIAN_FIELD_NAME, gauss));
|
||||
++i;
|
||||
}
|
||||
BulkResponse response = request.execute().actionGet();
|
||||
if (response.hasFailures()) {
|
||||
System.err.println("--> failures...");
|
||||
System.err.println(response.buildFailureMessage());
|
||||
}
|
||||
if ((i % 100000) == 0) {
|
||||
System.out.println("--> Indexed " + i + " took " + stopWatch.stop().lastTaskTime());
|
||||
stopWatch.start();
|
||||
}
|
||||
}
|
||||
|
||||
client.admin().indices().prepareRefresh(INDEX_NAME).execute().actionGet();
|
||||
} catch (Exception e) {
|
||||
System.out.println("Index already exists, skipping index creation");
|
||||
}
|
||||
|
||||
ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
|
||||
if (clusterHealthResponse.isTimedOut()) {
|
||||
System.err.println("--> Timed out waiting for cluster health");
|
||||
}
|
||||
|
||||
System.out.println("Run\tField\tMethod\tAggregationTime\tEstimatedMemory");
|
||||
for (int i = 0; i < WARM + RUNS; ++i) {
|
||||
for (String field : new String[] { LOW_CARD_FIELD_NAME, HIGH_CARD_FIELD_NAME, GAUSSIAN_FIELD_NAME }) {
|
||||
for (PercentilesMethod method : new PercentilesMethod[] {PercentilesMethod.TDIGEST, PercentilesMethod.HDR}) {
|
||||
long start = System.nanoTime();
|
||||
SearchResponse resp = null;
|
||||
for (int j = 0; j < ITERS; ++j) {
|
||||
resp = client.prepareSearch(INDEX_NAME).setSize(0).addAggregation(percentiles("percentiles").field(field).method(method)).execute().actionGet();
|
||||
}
|
||||
long end = System.nanoTime();
|
||||
long memoryEstimate = 0;
|
||||
switch (method) {
|
||||
case TDIGEST:
|
||||
memoryEstimate = ((InternalTDigestPercentiles) resp.getAggregations().get("percentiles"))
|
||||
.getEstimatedMemoryFootprint();
|
||||
break;
|
||||
case HDR:
|
||||
memoryEstimate = ((InternalHDRPercentiles) resp.getAggregations().get("percentiles")).getEstimatedMemoryFootprint();
|
||||
break;
|
||||
}
|
||||
if (i >= WARM) {
|
||||
System.out.println((i - WARM) + "\t" + field + "\t" + method + "\t"
|
||||
+ new TimeValue((end - start) / ITERS, TimeUnit.NANOSECONDS).millis() + "\t"
|
||||
+ new SizeValue(memoryEstimate, SizeUnit.SINGLE).singles());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
long overallEndTime = System.currentTimeMillis();
|
||||
System.out.println("Benchmark completed in " + ((overallEndTime - overallStartTime) / 1000) + " seconds");
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,502 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.script.Script;
|
||||
import org.elasticsearch.script.ScriptService.ScriptType;
|
||||
import org.elasticsearch.search.aggregations.bucket.global.Global;
|
||||
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
|
||||
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.lessThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.Matchers.sameInstance;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class HDRPercentileRanksTests extends AbstractNumericTests {
|
||||
|
||||
private static double[] randomPercents(long minValue, long maxValue) {
|
||||
|
||||
final int length = randomIntBetween(1, 20);
|
||||
final double[] percents = new double[length];
|
||||
for (int i = 0; i < percents.length; ++i) {
|
||||
switch (randomInt(20)) {
|
||||
case 0:
|
||||
percents[i] = minValue;
|
||||
break;
|
||||
case 1:
|
||||
percents[i] = maxValue;
|
||||
break;
|
||||
default:
|
||||
percents[i] = (randomDouble() * (maxValue - minValue)) + minValue;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Arrays.sort(percents);
|
||||
Loggers.getLogger(HDRPercentileRanksTests.class).info("Using percentiles={}", Arrays.toString(percents));
|
||||
return percents;
|
||||
}
|
||||
|
||||
private static int randomSignificantDigits() {
|
||||
return randomIntBetween(0, 5);
|
||||
}
|
||||
|
||||
private void assertConsistent(double[] pcts, PercentileRanks percentiles, long minValue, long maxValue, int numberSigDigits) {
|
||||
final List<Percentile> percentileList = Lists.newArrayList(percentiles);
|
||||
assertEquals(pcts.length, percentileList.size());
|
||||
for (int i = 0; i < pcts.length; ++i) {
|
||||
final Percentile percentile = percentileList.get(i);
|
||||
assertThat(percentile.getValue(), equalTo(pcts[i]));
|
||||
assertThat(percentile.getPercent(), greaterThanOrEqualTo(0.0));
|
||||
assertThat(percentile.getPercent(), lessThanOrEqualTo(100.0));
|
||||
|
||||
if (percentile.getPercent() == 0) {
|
||||
double allowedError = minValue / Math.pow(10, numberSigDigits);
|
||||
assertThat(percentile.getValue(), lessThanOrEqualTo(minValue + allowedError));
|
||||
}
|
||||
if (percentile.getPercent() == 100) {
|
||||
double allowedError = maxValue / Math.pow(10, numberSigDigits);
|
||||
assertThat(percentile.getValue(), greaterThanOrEqualTo(maxValue - allowedError));
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 1; i < percentileList.size(); ++i) {
|
||||
assertThat(percentileList.get(i).getValue(), greaterThanOrEqualTo(percentileList.get(i - 1).getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testEmptyAggregation() throws Exception {
|
||||
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("empty_bucket_idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
histogram("histo")
|
||||
.field("value")
|
||||
.interval(1l)
|
||||
.minDocCount(0)
|
||||
.subAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR)
|
||||
.numberOfSignificantValueDigits(sigDigits).percentiles(10, 15))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
|
||||
Histogram histo = searchResponse.getAggregations().get("histo");
|
||||
assertThat(histo, notNullValue());
|
||||
Histogram.Bucket bucket = histo.getBuckets().get(1);
|
||||
assertThat(bucket, notNullValue());
|
||||
|
||||
PercentileRanks reversePercentiles = bucket.getAggregations().get("percentile_ranks");
|
||||
assertThat(reversePercentiles, notNullValue());
|
||||
assertThat(reversePercentiles.getName(), equalTo("percentile_ranks"));
|
||||
assertThat(reversePercentiles.percent(10), equalTo(Double.NaN));
|
||||
assertThat(reversePercentiles.percent(15), equalTo(Double.NaN));
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testUnmapped() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx_unmapped")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").percentiles(0, 10, 15, 100)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l));
|
||||
|
||||
PercentileRanks reversePercentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertThat(reversePercentiles, notNullValue());
|
||||
assertThat(reversePercentiles.getName(), equalTo("percentile_ranks"));
|
||||
assertThat(reversePercentiles.percent(0), equalTo(Double.NaN));
|
||||
assertThat(reversePercentiles.percent(10), equalTo(Double.NaN));
|
||||
assertThat(reversePercentiles.percent(15), equalTo(Double.NaN));
|
||||
assertThat(reversePercentiles.percent(100), equalTo(Double.NaN));
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValue, maxValue);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_getProperty() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValue, maxValue);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
global("global").subAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").percentiles(pcts))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
Global global = searchResponse.getAggregations().get("global");
|
||||
assertThat(global, notNullValue());
|
||||
assertThat(global.getName(), equalTo("global"));
|
||||
assertThat(global.getDocCount(), equalTo(10l));
|
||||
assertThat(global.getAggregations(), notNullValue());
|
||||
assertThat(global.getAggregations().asMap().size(), equalTo(1));
|
||||
|
||||
PercentileRanks percentiles = global.getAggregations().get("percentile_ranks");
|
||||
assertThat(percentiles, notNullValue());
|
||||
assertThat(percentiles.getName(), equalTo("percentile_ranks"));
|
||||
assertThat((PercentileRanks) global.getProperty("percentile_ranks"), sameInstance(percentiles));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleValuedFieldOutsideRange() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = new double[] { minValue - 1, maxValue + 1 };
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_PartiallyUnmapped() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValue, maxValue);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx", "idx_unmapped")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_WithValueScript() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_WithValueScript_WithParams() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("value").script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValues, maxValues);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("values").percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("values").script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript_Reverse() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(20 - maxValues, 20 - minValues);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("values").script(new Script("20 - _value")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, 20 - maxValues, 20 - minValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript_WithParams() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.field("values").script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_SingleValued() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValue, maxValue);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script("doc['value'].value")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_SingleValued_WithParams() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_ExplicitSingleValued_WithParams() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_MultiValued() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValues, maxValues);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_ExplicitMultiValued() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
final double[] pcts = randomPercents(minValues, maxValues);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_MultiValued_WithParams() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentileRanks("percentile_ranks")
|
||||
.method(PercentilesMethod.HDR)
|
||||
.numberOfSignificantValueDigits(sigDigits)
|
||||
.script(new Script(
|
||||
"List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
|
||||
ScriptType.INLINE, null, params)).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOrderBySubAggregation() {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
boolean asc = randomBoolean();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
histogram("histo").field("value").interval(2l)
|
||||
.subAggregation(
|
||||
percentileRanks("percentile_ranks").method(PercentilesMethod.HDR)
|
||||
.numberOfSignificantValueDigits(sigDigits).percentiles(99))
|
||||
.order(Order.aggregation("percentile_ranks", "99", asc))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
Histogram histo = searchResponse.getAggregations().get("histo");
|
||||
double previous = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
|
||||
for (Histogram.Bucket bucket : histo.getBuckets()) {
|
||||
PercentileRanks percentiles = bucket.getAggregations().get("percentile_ranks");
|
||||
double p99 = percentiles.percent(99);
|
||||
if (asc) {
|
||||
assertThat(p99, greaterThanOrEqualTo(previous));
|
||||
} else {
|
||||
assertThat(p99, lessThanOrEqualTo(previous));
|
||||
}
|
||||
previous = p99;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,489 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.search.aggregations.metrics;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.script.Script;
|
||||
import org.elasticsearch.script.ScriptService.ScriptType;
|
||||
import org.elasticsearch.search.aggregations.bucket.global.Global;
|
||||
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
|
||||
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
|
||||
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
|
||||
import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
|
||||
import static org.hamcrest.Matchers.closeTo;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.lessThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.Matchers.sameInstance;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class HDRPercentilesTests extends AbstractNumericTests {
|
||||
|
||||
private static double[] randomPercentiles() {
|
||||
final int length = randomIntBetween(1, 20);
|
||||
final double[] percentiles = new double[length];
|
||||
for (int i = 0; i < percentiles.length; ++i) {
|
||||
switch (randomInt(20)) {
|
||||
case 0:
|
||||
percentiles[i] = 0;
|
||||
break;
|
||||
case 1:
|
||||
percentiles[i] = 100;
|
||||
break;
|
||||
default:
|
||||
percentiles[i] = randomDouble() * 100;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Arrays.sort(percentiles);
|
||||
Loggers.getLogger(HDRPercentilesTests.class).info("Using percentiles={}", Arrays.toString(percentiles));
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
private static int randomSignificantDigits() {
|
||||
return randomIntBetween(0, 5);
|
||||
}
|
||||
|
||||
private void assertConsistent(double[] pcts, Percentiles percentiles, long minValue, long maxValue, int numberSigDigits) {
|
||||
final List<Percentile> percentileList = Lists.newArrayList(percentiles);
|
||||
assertEquals(pcts.length, percentileList.size());
|
||||
for (int i = 0; i < pcts.length; ++i) {
|
||||
final Percentile percentile = percentileList.get(i);
|
||||
assertThat(percentile.getPercent(), equalTo(pcts[i]));
|
||||
double value = percentile.getValue();
|
||||
double allowedError = value / Math.pow(10, numberSigDigits);
|
||||
assertThat(value, greaterThanOrEqualTo(minValue - allowedError));
|
||||
assertThat(value, lessThanOrEqualTo(maxValue + allowedError));
|
||||
|
||||
if (percentile.getPercent() == 0) {
|
||||
assertThat(value, closeTo(minValue, allowedError));
|
||||
}
|
||||
if (percentile.getPercent() == 100) {
|
||||
assertThat(value, closeTo(maxValue, allowedError));
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 1; i < percentileList.size(); ++i) {
|
||||
assertThat(percentileList.get(i).getValue(), greaterThanOrEqualTo(percentileList.get(i - 1).getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testEmptyAggregation() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("empty_bucket_idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
histogram("histo")
|
||||
.field("value")
|
||||
.interval(1l)
|
||||
.minDocCount(0)
|
||||
.subAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.percentiles(10,
|
||||
15))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
|
||||
Histogram histo = searchResponse.getAggregations().get("histo");
|
||||
assertThat(histo, notNullValue());
|
||||
Histogram.Bucket bucket = histo.getBuckets().get(1);
|
||||
assertThat(bucket, notNullValue());
|
||||
|
||||
Percentiles percentiles = bucket.getAggregations().get("percentiles");
|
||||
assertThat(percentiles, notNullValue());
|
||||
assertThat(percentiles.getName(), equalTo("percentiles"));
|
||||
assertThat(percentiles.percentile(10), equalTo(Double.NaN));
|
||||
assertThat(percentiles.percentile(15), equalTo(Double.NaN));
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testUnmapped() throws Exception {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx_unmapped")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
|
||||
.percentiles(0, 10, 15, 100)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l));
|
||||
|
||||
Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertThat(percentiles, notNullValue());
|
||||
assertThat(percentiles.getName(), equalTo("percentiles"));
|
||||
assertThat(percentiles.percentile(0), equalTo(Double.NaN));
|
||||
assertThat(percentiles.percentile(10), equalTo(Double.NaN));
|
||||
assertThat(percentiles.percentile(15), equalTo(Double.NaN));
|
||||
assertThat(percentiles.percentile(100), equalTo(Double.NaN));
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomIntBetween(1, 5);
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
|
||||
.percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_getProperty() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
global("global").subAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.field("value")
|
||||
.percentiles(pcts))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
Global global = searchResponse.getAggregations().get("global");
|
||||
assertThat(global, notNullValue());
|
||||
assertThat(global.getName(), equalTo("global"));
|
||||
assertThat(global.getDocCount(), equalTo(10l));
|
||||
assertThat(global.getAggregations(), notNullValue());
|
||||
assertThat(global.getAggregations().asMap().size(), equalTo(1));
|
||||
|
||||
Percentiles percentiles = global.getAggregations().get("percentiles");
|
||||
assertThat(percentiles, notNullValue());
|
||||
assertThat(percentiles.getName(), equalTo("percentiles"));
|
||||
assertThat((Percentiles) global.getProperty("percentiles"), sameInstance(percentiles));
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_PartiallyUnmapped() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx", "idx_unmapped")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
|
||||
.percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_WithValueScript() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
|
||||
.script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testSingleValuedField_WithValueScript_WithParams() throws Exception {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
|
||||
.script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)).execute()
|
||||
.actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
|
||||
.percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
|
||||
.script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript_Reverse() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
|
||||
.script(new Script("20 - _value")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, 20 - maxValues, 20 - minValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testMultiValuedField_WithValueScript_WithParams() throws Exception {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
|
||||
.script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)).execute()
|
||||
.actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_SingleValued() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.script(new Script("doc['value'].value")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue, maxValue, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_SingleValued_WithParams() throws Exception {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_ExplicitSingleValued_WithParams() throws Exception {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_MultiValued() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_ExplicitMultiValued() throws Exception {
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
|
||||
.script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
public void testScript_MultiValued_WithParams() throws Exception {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("dec", 1);
|
||||
final double[] pcts = randomPercentiles();
|
||||
int sigDigits = randomSignificantDigits();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
percentiles("percentiles")
|
||||
.numberOfSignificantValueDigits(sigDigits)
|
||||
.method(PercentilesMethod.HDR)
|
||||
.script(new Script(
|
||||
"List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
|
||||
ScriptType.INLINE, null, params)).percentiles(pcts)).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
final Percentiles percentiles = searchResponse.getAggregations().get("percentiles");
|
||||
assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOrderBySubAggregation() {
|
||||
int sigDigits = randomSignificantDigits();
|
||||
boolean asc = randomBoolean();
|
||||
SearchResponse searchResponse = client()
|
||||
.prepareSearch("idx")
|
||||
.setQuery(matchAllQuery())
|
||||
.addAggregation(
|
||||
histogram("histo").field("value").interval(2l)
|
||||
.subAggregation(
|
||||
percentiles("percentiles").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
|
||||
.percentiles(99))
|
||||
.order(Order.aggregation("percentiles", "99", asc))).execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
|
||||
|
||||
Histogram histo = searchResponse.getAggregations().get("histo");
|
||||
double previous = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
|
||||
for (Histogram.Bucket bucket : histo.getBuckets()) {
|
||||
Percentiles percentiles = bucket.getAggregations().get("percentiles");
|
||||
double p99 = percentiles.percentile(99);
|
||||
if (asc) {
|
||||
assertThat(p99, greaterThanOrEqualTo(previous));
|
||||
} else {
|
||||
assertThat(p99, lessThanOrEqualTo(previous));
|
||||
}
|
||||
previous = p99;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -50,7 +50,7 @@ import static org.hamcrest.Matchers.sameInstance;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class PercentileRanksTests extends AbstractNumericTests {
|
||||
public class TDigestPercentileRanksTests extends AbstractNumericTests {
|
||||
|
||||
private static double[] randomPercents(long minValue, long maxValue) {
|
||||
|
||||
|
@ -70,7 +70,7 @@ public class PercentileRanksTests extends AbstractNumericTests {
|
|||
}
|
||||
}
|
||||
Arrays.sort(percents);
|
||||
Loggers.getLogger(PercentileRanksTests.class).info("Using percentiles={}", Arrays.toString(percents));
|
||||
Loggers.getLogger(TDigestPercentileRanksTests.class).info("Using percentiles={}", Arrays.toString(percents));
|
||||
return percents;
|
||||
}
|
||||
|
|
@ -50,7 +50,7 @@ import static org.hamcrest.Matchers.sameInstance;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
public class PercentilesTests extends AbstractNumericTests {
|
||||
public class TDigestPercentilesTests extends AbstractNumericTests {
|
||||
|
||||
private static double[] randomPercentiles() {
|
||||
final int length = randomIntBetween(1, 20);
|
||||
|
@ -69,7 +69,7 @@ public class PercentilesTests extends AbstractNumericTests {
|
|||
}
|
||||
}
|
||||
Arrays.sort(percentiles);
|
||||
Loggers.getLogger(PercentilesTests.class).info("Using percentiles={}", Arrays.toString(percentiles));
|
||||
Loggers.getLogger(TDigestPercentilesTests.class).info("Using percentiles={}", Arrays.toString(percentiles));
|
||||
return percentiles;
|
||||
}
|
||||
|
|
@ -213,6 +213,40 @@ of data which arrives sorted and in-order) the default settings will produce a
|
|||
TDigest roughly 64KB in size. In practice data tends to be more random and
|
||||
the TDigest will use less memory.
|
||||
|
||||
==== HDR Histogram
|
||||
|
||||
experimental[]
|
||||
|
||||
https://github.com/HdrHistogram/HdrHistogram[HDR Histogram] (High Dynamic Range Histogram) is an alternative implementation
|
||||
that can be useful when calculating percentiles for latency measurements as it can be faster than the t-digest implementation
|
||||
with the trade-off of a larger memory footprint. This implementation maintains a fixed worse-case percentage error (specified
|
||||
as a number of significant digits). This means that if data is recorded with values from 1 microsecond up to 1 hour
|
||||
(3,600,000,000 microseconds) in a histogram set to 3 significant digits, it will maintain a value resolution of 1 microsecond
|
||||
for values up to 1 millisecond and 3.6 seconds (or better) for the maximum tracked value (1 hour).
|
||||
|
||||
The HDR Histogram can be used by specifying the `method` parameter in the request:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"aggs" : {
|
||||
"load_time_outlier" : {
|
||||
"percentiles" : {
|
||||
"field" : "load_time",
|
||||
"percents" : [95, 99, 99.9],
|
||||
"method" : "hdr", <1>
|
||||
"number_of_significant_value_digits" : 3 <2>
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
<1> The `method` parameter is set to `hdr` to indicate that HDR Histogram should be used to calculate the percentiles
|
||||
<2> `number_of_significant_value_digits` specifies the resolution of values for the histogram in number of significant digits
|
||||
|
||||
The HDRHistogram only supports positive values and will error if it is passed a negative value. It is also not a good idea to use
|
||||
the HDRHistogram if the range of values is unknown as this could lead to high memory usage.
|
||||
|
||||
==== Missing value
|
||||
|
||||
The `missing` parameter defines how documents that are missing a value should be treated.
|
||||
|
|
|
@ -110,6 +110,40 @@ This will interpret the `script` parameter as an `inline` script with the defaul
|
|||
|
||||
TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
|
||||
|
||||
==== HDR Histogram
|
||||
|
||||
experimental[]
|
||||
|
||||
https://github.com/HdrHistogram/HdrHistogram[HDR Histogram] (High Dynamic Range Histogram) is an alternative implementation
|
||||
that can be useful when calculating percentile ranks for latency measurements as it can be faster than the t-digest implementation
|
||||
with the trade-off of a larger memory footprint. This implementation maintains a fixed worse-case percentage error (specified as a
|
||||
number of significant digits). This means that if data is recorded with values from 1 microsecond up to 1 hour (3,600,000,000
|
||||
microseconds) in a histogram set to 3 significant digits, it will maintain a value resolution of 1 microsecond for values up to
|
||||
1 millisecond and 3.6 seconds (or better) for the maximum tracked value (1 hour).
|
||||
|
||||
The HDR Histogram can be used by specifying the `method` parameter in the request:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"aggs" : {
|
||||
"load_time_outlier" : {
|
||||
"percentile_ranks" : {
|
||||
"field" : "load_time",
|
||||
"values" : [15, 30],
|
||||
"method" : "hdr", <1>
|
||||
"number_of_significant_value_digits" : 3 <2>
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
<1> The `method` parameter is set to `hdr` to indicate that HDR Histogram should be used to calculate the percentile_ranks
|
||||
<2> `number_of_significant_value_digits` specifies the resolution of values for the histogram in number of significant digits
|
||||
|
||||
The HDRHistogram only supports positive values and will error if it is passed a negative value. It is also not a good idea to use
|
||||
the HDRHistogram if the range of values is unknown as this could lead to high memory usage.
|
||||
|
||||
==== Missing value
|
||||
|
||||
The `missing` parameter defines how documents that are missing a value should be treated.
|
||||
|
|
Loading…
Reference in New Issue