honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade t-digest to 3.2 (#28295) (#28305)

This commit is contained in:
Ke Li 2018-02-15 16:23:20 +08:00 committed by Colin Goodheart-Smithe
parent cd54c96d56
commit fc406c9a5a
7 changed files with 98 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
View File

@ -53,13 +53,13 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this:
"aggregations": {
"load_time_outlier": {
"values" : {
"1.0": 9.9,
"5.0": 29.500000000000004,
"25.0": 167.5,
"1.0": 5.0,
"5.0": 25.0,
"25.0": 165.0,
"50.0": 445.0,
"75.0": 722.5,
"95.0": 940.5,
"99.0": 980.1000000000001
"75.0": 725.0,
"95.0": 945.0,
"99.0": 985.0
}
}
}
@ -129,15 +129,15 @@ Response:
"values": [
{
"key": 1.0,
"value": 9.9
"value": 5.0
},
{
"key": 5.0,
"value": 29.500000000000004
"value": 25.0
},
{
"key": 25.0,
"value": 167.5
"value": 165.0
},
{
"key": 50.0,
@ -145,15 +145,15 @@ Response:
},
{
"key": 75.0,
"value": 722.5
"value": 725.0
},
{
"key": 95.0,
"value": 940.5
"value": 945.0
},
{
"key": 99.0,
"value": 980.1000000000001
"value": 985.0
}
]
}

122
rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml
View File

@ -65,21 +65,21 @@ setup:
- match: { hits.total: 4 }
- length: { hits.hits: 4 }
- match: { aggregations.percentiles_int.values.1\.0: 2.5 }
- match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.25\.0: 38.5 }
- match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- match: { aggregations.percentiles_int.values.25\.0: 26.0 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_int.values.75\.0: 113.5 }
- match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_int.values.99\.0: 149.5 }
- match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- match: { aggregations.percentiles_int.values.99\.0: 151.0 }
- match: { aggregations.percentiles_double.values.1\.0: 2.5 }
- match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_double.values.25\.0: 38.5 }
- match: { aggregations.percentiles_double.values.1\.0: 1.0 }
- match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- match: { aggregations.percentiles_double.values.25\.0: 26.0 }
- match: { aggregations.percentiles_double.values.50\.0: 76.0 }
- match: { aggregations.percentiles_double.values.75\.0: 113.5 }
- match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_double.values.99\.0: 149.5 }
- match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- match: { aggregations.percentiles_double.values.99\.0: 151.0 }
- do:
search:
@ -100,21 +100,21 @@ setup:
- match: { hits.total: 4 }
- length: { hits.hits: 4 }
- match: { aggregations.percentiles_int.values.1\.0: 2.5 }
- match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.25\.0: 38.5 }
- match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- match: { aggregations.percentiles_int.values.25\.0: 26.0 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_int.values.75\.0: 113.5 }
- match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_int.values.99\.0: 149.5 }
- match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- match: { aggregations.percentiles_int.values.99\.0: 151.0 }
- match: { aggregations.percentiles_double.values.1\.0: 2.5 }
- match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_double.values.25\.0: 38.5 }
- match: { aggregations.percentiles_double.values.1\.0: 1.0 }
- match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- match: { aggregations.percentiles_double.values.25\.0: 26.0 }
- match: { aggregations.percentiles_double.values.50\.0: 76.0 }
- match: { aggregations.percentiles_double.values.75\.0: 113.5 }
- match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_double.values.99\.0: 149.5 }
- match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- match: { aggregations.percentiles_double.values.99\.0: 151.0 }
---
@ -135,21 +135,21 @@ setup:
- match: { hits.total: 4 }
- length: { hits.hits: 0 }
- match: { aggregations.percentiles_int.values.1\.0: 2.5 }
- match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.25\.0: 38.5 }
- match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- match: { aggregations.percentiles_int.values.25\.0: 26.0 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_int.values.75\.0: 113.5 }
- match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_int.values.99\.0: 149.5 }
- match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- match: { aggregations.percentiles_int.values.99\.0: 151.0 }
- match: { aggregations.percentiles_double.values.1\.0: 2.5 }
- match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_double.values.25\.0: 38.5 }
- match: { aggregations.percentiles_double.values.1\.0: 1.0 }
- match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- match: { aggregations.percentiles_double.values.25\.0: 26.0 }
- match: { aggregations.percentiles_double.values.50\.0: 76.0 }
- match: { aggregations.percentiles_double.values.75\.0: 113.5 }
- match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_double.values.99\.0: 149.5 }
- match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- match: { aggregations.percentiles_double.values.99\.0: 151.0 }
@ -176,21 +176,21 @@ setup:
- match: { hits.total: 3 }
- length: { hits.hits: 3 }
- match: { aggregations.percentiles_int.values.1\.0: 52.0 }
- match: { aggregations.percentiles_int.values.5\.0: 56.0 }
- match: { aggregations.percentiles_int.values.25\.0: 76.0 }
- match: { aggregations.percentiles_int.values.1\.0: 51.0 }
- match: { aggregations.percentiles_int.values.5\.0: 51.0 }
- match: { aggregations.percentiles_int.values.25\.0: 63.5 }
- match: { aggregations.percentiles_int.values.50\.0: 101.0 }
- match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- match: { aggregations.percentiles_int.values.95\.0: 146.0 }
- match: { aggregations.percentiles_int.values.99\.0: 150.0 }
- match: { aggregations.percentiles_int.values.75\.0: 138.5 }
- match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- match: { aggregations.percentiles_int.values.99\.0: 151.0 }
- match: { aggregations.percentiles_double.values.1\.0: 52.0 }
- match: { aggregations.percentiles_double.values.5\.0: 56.0 }
- match: { aggregations.percentiles_double.values.25\.0: 76.0 }
- match: { aggregations.percentiles_double.values.1\.0: 51.0 }
- match: { aggregations.percentiles_double.values.5\.0: 51.0 }
- match: { aggregations.percentiles_double.values.25\.0: 63.5 }
- match: { aggregations.percentiles_double.values.50\.0: 101.0 }
- match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- match: { aggregations.percentiles_double.values.95\.0: 146.0 }
- match: { aggregations.percentiles_double.values.99\.0: 150.0 }
- match: { aggregations.percentiles_double.values.75\.0: 138.5 }
- match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- match: { aggregations.percentiles_double.values.99\.0: 151.0 }
---
"Missing field with missing param":
@ -248,13 +248,13 @@ setup:
- match: { aggregations.percentiles_int.meta.foo: "bar" }
- match: { aggregations.percentiles_int.values.1\.0: 2.5 }
- match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.25\.0: 38.5 }
- match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- match: { aggregations.percentiles_int.values.25\.0: 26.0 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_int.values.75\.0: 113.5 }
- match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
- match: { aggregations.percentiles_int.values.99\.0: 149.5 }
- match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- match: { aggregations.percentiles_int.values.99\.0: 151.0 }
---
"Invalid params test":
@ -329,12 +329,12 @@ setup:
- match: { hits.total: 4 }
- length: { hits.hits: 4 }
- match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.25\.0: 38.5 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- match: { aggregations.percentiles_int.values.25\.0: 26.0 }
- match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
- match: { aggregations.percentiles_double.values.25\.0: 38.5 }
- match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- match: { aggregations.percentiles_double.values.25\.0: 26.0 }
- match: { aggregations.percentiles_double.values.50\.0: 76.0 }
---
@ -355,9 +355,9 @@ setup:
- length: { hits.hits: 4 }
- match: { aggregations.percentiles_int.values.0.key: 5.0 }
- match: { aggregations.percentiles_int.values.0.value: 8.500000000000002 }
- match: { aggregations.percentiles_int.values.0.value: 1.0 }
- match: { aggregations.percentiles_int.values.1.key: 25.0 }
- match: { aggregations.percentiles_int.values.1.value: 38.5 }
- match: { aggregations.percentiles_int.values.1.value: 26.0 }
- match: { aggregations.percentiles_int.values.2.key: 50.0 }
- match: { aggregations.percentiles_int.values.2.value: 76.0 }

2
server/build.gradle
View File

@ -99,7 +99,7 @@ dependencies {
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
// percentiles aggregation
compile 'com.tdunning:t-digest:3.0'
compile 'com.tdunning:t-digest:3.2'
// precentil ranks aggregation
compile 'org.hdrhistogram:HdrHistogram:2.1.9'

1
server/licenses/t-digest-3.0.jar.sha1
View File

@ -1 +0,0 @@
84ccf145ac2215e6bfa63baa3101c0af41017cfc

1
server/licenses/t-digest-3.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2ab94758b0276a8a26102adf8d528cf6d0567b9a

32
server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
View File

@ -69,7 +69,6 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
}
private static double[] randomPercents(long minValue, long maxValue) {
final int length = randomIntBetween(1, 20);
final double[] percents = new double[length];
for (int i = 0; i < percents.length; ++i) {
@ -97,7 +96,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
return builder;
}
private void assertConsistent(double[] pcts, PercentileRanks values, long minValue, long maxValue) {
private void assertConsistent(double[] pcts, PercentileRanks values, long minValue) {
final List<Percentile> percentileList = CollectionUtils.iterableAsArrayList(values);
assertEquals(pcts.length, percentileList.size());
for (int i = 0; i < pcts.length; ++i) {
@ -109,9 +108,6 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
if (percentile.getPercent() == 0) {
assertThat(percentile.getValue(), lessThanOrEqualTo((double) minValue));
}
if (percentile.getPercent() == 100) {
assertThat(percentile.getValue(), greaterThanOrEqualTo((double) maxValue));
}
}
for (int i = 1; i < percentileList.size(); ++i) {
@ -193,7 +189,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue, maxValue);
assertConsistent(pcts, values, minValue);
}
@Override
@ -233,7 +229,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue, maxValue);
assertConsistent(pcts, values, minValue);
}
@Override
@ -248,7 +244,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue, maxValue);
assertConsistent(pcts, values, minValue);
}
@Override
@ -266,7 +262,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue - 1, maxValue - 1);
assertConsistent(pcts, values, minValue - 1);
}
@Override
@ -286,7 +282,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue - 1, maxValue - 1);
assertConsistent(pcts, values, minValue - 1);
}
@Override
@ -301,7 +297,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValues, maxValues);
assertConsistent(pcts, values, minValues);
}
@Override
@ -319,7 +315,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValues - 1, maxValues - 1);
assertConsistent(pcts, values, minValues - 1);
}
public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
@ -336,7 +332,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, -maxValues, -minValues);
assertConsistent(pcts, values, -maxValues);
}
@Override
@ -356,7 +352,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValues - 1, maxValues - 1);
assertConsistent(pcts, values, minValues - 1);
}
@Override
@ -373,7 +369,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue, maxValue);
assertConsistent(pcts, values, minValue);
}
@Override
@ -394,7 +390,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValue - 1, maxValue - 1);
assertConsistent(pcts, values, minValue - 1);
}
@Override
@ -412,7 +408,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValues, maxValues);
assertConsistent(pcts, values, minValues);
}
@Override
@ -431,7 +427,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
assertHitCount(searchResponse, 10);
final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
assertConsistent(pcts, values, minValues - 1, maxValues - 1);
assertConsistent(pcts, values, minValues - 1);
}
public void testOrderBySubAggregation() {

18
server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java
View File

@ -74,12 +74,12 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
}, tdigest -> {
assertEquals(7L, tdigest.state.size());
assertEquals(7L, tdigest.state.centroidCount());
assertEquals(4.0d, tdigest.percentile(75), 0.0d);
assertEquals("4.0", tdigest.percentileAsString(75));
assertEquals(4.5d, tdigest.percentile(75), 0.0d);
assertEquals("4.5", tdigest.percentileAsString(75));
assertEquals(2.0d, tdigest.percentile(50), 0.0d);
assertEquals("2.0", tdigest.percentileAsString(50));
assertEquals(1.0d, tdigest.percentile(20), 0.0d);
assertEquals("1.0", tdigest.percentileAsString(20));
assertEquals(1.0d, tdigest.percentile(22), 0.0d);
assertEquals("1.0", tdigest.percentileAsString(22));
});
}
@ -97,14 +97,14 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
assertEquals(tdigest.state.centroidCount(), 7L);
assertEquals(8.0d, tdigest.percentile(100), 0.0d);
assertEquals("8.0", tdigest.percentileAsString(100));
assertEquals(5.48d, tdigest.percentile(86), 0.0d);
assertEquals("5.48", tdigest.percentileAsString(86));
assertEquals(6.98d, tdigest.percentile(88), 0.0d);
assertEquals("6.98", tdigest.percentileAsString(88));
assertEquals(1.0d, tdigest.percentile(33), 0.0d);
assertEquals("1.0", tdigest.percentileAsString(33));
assertEquals(1.0d, tdigest.percentile(25), 0.0d);
assertEquals("1.0", tdigest.percentileAsString(25));
assertEquals(0.06d, tdigest.percentile(1), 0.0d);
assertEquals("0.06", tdigest.percentileAsString(1));
assertEquals(0.0d, tdigest.percentile(1), 0.0d);
assertEquals("0.0", tdigest.percentileAsString(1));
});
}
@ -124,7 +124,7 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
assertEquals(4L, tdigest.state.centroidCount());
assertEquals(2.0d, tdigest.percentile(100), 0.0d);
assertEquals(1.0d, tdigest.percentile(50), 0.0d);
assertEquals(0.75d, tdigest.percentile(25), 0.0d);
assertEquals(0.5d, tdigest.percentile(25), 0.0d);
});
testCase(LongPoint.newRangeQuery("row", 100, 110), docs, tdigest -> {