From 420fa8c400b39ff1af90eb598d44785697732e42 Mon Sep 17 00:00:00 2001 From: javanna Date: Fri, 7 Apr 2017 11:11:06 +0200 Subject: [PATCH 01/62] Add ParsedAggregation as base Aggregation impl for high level client ParsedAggregation is the base Aggregation implementation for the high level client, which parses aggs responses into java objects. --- .../aggregations/ParsedAggregation.java | 80 ++++++++++ .../aggregations/ParsedAggregationTests.java | 145 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java new file mode 100644 index 00000000000..f1f97052149 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * An implementation of {@link Aggregation} that is parsed from a REST response. + * Serves as a base class for all aggregation implementations that are parsed from REST. + */ +public abstract class ParsedAggregation implements Aggregation, ToXContent { + + //TODO move CommonFields out of InternalAggregation + protected static void declareCommonFields(ObjectParser objectParser) { + objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata.putAll(metadata), + (parser, context) -> parser.map(), InternalAggregation.CommonFields.META); + } + + String name; + final Map metadata = new HashMap<>(); + + @Override + public final String getName() { + return name; + } + + @Override + public final Map getMetaData() { + return Collections.unmodifiableMap(metadata); + } + + /** + * Returns a string representing the type of the aggregation. This type is added to + * the aggregation name in the response, so that it can later be used by REST clients + * to determine the internal type of the aggregation. + */ + //TODO it may make sense to move getType to the Aggregation interface given that we are duplicating it in both implementations + protected abstract String getType(); + + //TODO the only way to avoid duplicating this method is making Aggregation extend ToXContent + //and declare toXContent as a default method in it. Doesn't sound like the right thing to do. + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + //TODO move TYPED_KEYS_DELIMITER constant out of InternalAggregation + // Concatenates the type and the name of the aggregation (ex: top_hits#foo) + builder.startObject(String.join(InternalAggregation.TYPED_KEYS_DELIMITER, getType(), name)); + if (metadata.isEmpty() == false) { + builder.field(InternalAggregation.CommonFields.META.getPreferredName()); + builder.map(metadata); + } + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException; +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java new file mode 100644 index 00000000000..4b75bf1990c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -0,0 +1,145 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.CoreMatchers.instanceOf; + +public class ParsedAggregationTests extends ESTestCase { + + //TODO maybe this test will no longer be needed once we have real tests for ParsedAggregation subclasses + public void testParse() throws IOException { + String name = randomAlphaOfLengthBetween(5, 10); + int numMetas = randomIntBetween(0, 5); + Map meta = new HashMap<>(numMetas); + for (int i = 0; i < numMetas; i++) { + meta.put(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)); + } + TestInternalAggregation testAgg = new TestInternalAggregation(name, meta); + XContentType xContentType = randomFrom(XContentType.values()); + FakeRestRequest params = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withParams(Collections.singletonMap("typed_keys", "true")).build(); + BytesReference bytesAgg = XContentHelper.toXContent(testAgg, xContentType, params, randomBoolean()); + try (XContentParser parser = createParser(xContentType.xContent(), bytesAgg)) { + parser.nextToken(); + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + parser.nextToken(); + assert parser.currentToken() == XContentParser.Token.FIELD_NAME; + String currentName = parser.currentName(); + int i = currentName.indexOf(InternalAggregation.TYPED_KEYS_DELIMITER); + String aggType = currentName.substring(0, i); + String aggName = currentName.substring(i + 1); + Aggregation parsedAgg = parser.namedObject(Aggregation.class, aggType, aggName); + assertThat(parsedAgg, instanceOf(TestParsedAggregation.class)); + assertEquals(testAgg.getName(), parsedAgg.getName()); + assertEquals(testAgg.getMetaData(), parsedAgg.getMetaData()); + BytesReference finalAgg = XContentHelper.toXContent((ToXContent) parsedAgg, xContentType, randomBoolean()); + assertToXContentEquivalent(bytesAgg, finalAgg, xContentType); + } + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + //TODO we may want to un-deprecate this Entry constructor if we are going to use it extensively, which I think we are + NamedXContentRegistry.Entry entry = new NamedXContentRegistry.Entry(Aggregation.class, new ParseField("type"), + (parser, name) -> TestParsedAggregation.fromXContent(parser, (String)name)); + return new NamedXContentRegistry(Collections.singletonList(entry)); + } + + private static class TestParsedAggregation extends ParsedAggregation { + private static ObjectParser PARSER = new ObjectParser<>("testAggParser", TestParsedAggregation::new); + + static { + ParsedAggregation.declareCommonFields(PARSER); + } + + @Override + protected String getType() { + return "type"; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + return builder; + } + + public static TestParsedAggregation fromXContent(XContentParser parser, String name) throws IOException { + TestParsedAggregation parsedAgg = PARSER.parse(parser, null); + parsedAgg.name = name; + return parsedAgg; + } + } + + private static class TestInternalAggregation extends InternalAggregation { + + private TestInternalAggregation(String name, Map metaData) { + super(name, Collections.emptyList(), metaData); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException(); + } + + @Override + protected String getType() { + return "type"; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public InternalAggregation doReduce(List aggregations, ReduceContext reduceContext) { + throw new UnsupportedOperationException(); + } + + @Override + public Object getProperty(List path) { + throw new UnsupportedOperationException(); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + return builder; + } + } +} From 67e087088dab9f3bb9ee9f4702d919a19b0ea3eb Mon Sep 17 00:00:00 2001 From: javanna Date: Fri, 7 Apr 2017 12:00:51 +0200 Subject: [PATCH 02/62] add class to suppressions list for line length check --- buildSrc/src/main/resources/checkstyle_suppressions.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 11882b2faa9..ec67f8bb142 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -1749,6 +1749,7 @@ + @@ -3062,6 +3063,7 @@ + From 306ef086c5218a8ef46debd4bc76c72dd60e85e2 Mon Sep 17 00:00:00 2001 From: javanna Date: Fri, 7 Apr 2017 21:54:46 +0200 Subject: [PATCH 03/62] Align ParsedAggregation meta to InternalAggregation behaviour Empty meta gets printed out, which means that if the request contains an empty meta object, that is returned with the response as well. On the other hand null, meaning when the object is not in the request, is not printed out. ParsedAggregation used to not print out empty metadata, and didn't allow the null value. Aligned behaviour to the existing behaviour from InternalAggregation. --- .../search/aggregations/ParsedAggregation.java | 11 +++++------ .../aggregations/ParsedAggregationTests.java | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index f1f97052149..16b58dbf1f7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.Map; /** @@ -36,12 +35,12 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { //TODO move CommonFields out of InternalAggregation protected static void declareCommonFields(ObjectParser objectParser) { - objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata.putAll(metadata), + objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata), (parser, context) -> parser.map(), InternalAggregation.CommonFields.META); } String name; - final Map metadata = new HashMap<>(); + Map metadata; @Override public final String getName() { @@ -50,7 +49,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { @Override public final Map getMetaData() { - return Collections.unmodifiableMap(metadata); + return metadata; } /** @@ -67,9 +66,9 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { //TODO move TYPED_KEYS_DELIMITER constant out of InternalAggregation // Concatenates the type and the name of the aggregation (ex: top_hits#foo) builder.startObject(String.join(InternalAggregation.TYPED_KEYS_DELIMITER, getType(), name)); - if (metadata.isEmpty() == false) { + if (this.metadata != null) { builder.field(InternalAggregation.CommonFields.META.getPreferredName()); - builder.map(metadata); + builder.map(this.metadata); } doXContentBody(builder, params); builder.endObject(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java index 4b75bf1990c..8962ba21956 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -46,15 +47,18 @@ public class ParsedAggregationTests extends ESTestCase { //TODO maybe this test will no longer be needed once we have real tests for ParsedAggregation subclasses public void testParse() throws IOException { String name = randomAlphaOfLengthBetween(5, 10); - int numMetas = randomIntBetween(0, 5); - Map meta = new HashMap<>(numMetas); - for (int i = 0; i < numMetas; i++) { - meta.put(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)); + Map meta = null; + if (randomBoolean()) { + int numMetas = randomIntBetween(0, 5); + meta = new HashMap<>(numMetas); + for (int i = 0; i < numMetas; i++) { + meta.put(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)); + } } TestInternalAggregation testAgg = new TestInternalAggregation(name, meta); XContentType xContentType = randomFrom(XContentType.values()); FakeRestRequest params = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) - .withParams(Collections.singletonMap("typed_keys", "true")).build(); + .withParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")).build(); BytesReference bytesAgg = XContentHelper.toXContent(testAgg, xContentType, params, randomBoolean()); try (XContentParser parser = createParser(xContentType.xContent(), bytesAgg)) { parser.nextToken(); @@ -69,6 +73,9 @@ public class ParsedAggregationTests extends ESTestCase { assertThat(parsedAgg, instanceOf(TestParsedAggregation.class)); assertEquals(testAgg.getName(), parsedAgg.getName()); assertEquals(testAgg.getMetaData(), parsedAgg.getMetaData()); + if (meta != null) { + expectThrows(UnsupportedOperationException.class, () -> parsedAgg.getMetaData().put("test", "test")); + } BytesReference finalAgg = XContentHelper.toXContent((ToXContent) parsedAgg, xContentType, randomBoolean()); assertToXContentEquivalent(bytesAgg, finalAgg, xContentType); } From 8464b4755fc0f2da53909de6775e9ba4f84265d9 Mon Sep 17 00:00:00 2001 From: javanna Date: Sat, 8 Apr 2017 00:18:56 +0200 Subject: [PATCH 04/62] [TEST] replace FareRestRequest usage with ToXContent.MapParams --- .../search/aggregations/ParsedAggregationTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java index 8962ba21956..0a83bf3eeb8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; import java.util.Collections; @@ -57,8 +56,7 @@ public class ParsedAggregationTests extends ESTestCase { } TestInternalAggregation testAgg = new TestInternalAggregation(name, meta); XContentType xContentType = randomFrom(XContentType.values()); - FakeRestRequest params = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) - .withParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")).build(); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); BytesReference bytesAgg = XContentHelper.toXContent(testAgg, xContentType, params, randomBoolean()); try (XContentParser parser = createParser(xContentType.xContent(), bytesAgg)) { parser.nextToken(); From 9e7b0205781ce8518b84e7e53178091b065f7892 Mon Sep 17 00:00:00 2001 From: javanna Date: Sat, 8 Apr 2017 00:28:49 +0200 Subject: [PATCH 05/62] Remove TODO on un-deprecating NamedContentRegistry.Entry ctor that takes a context --- .../search/aggregations/ParsedAggregationTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java index 0a83bf3eeb8..f4015ca8d6b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -81,7 +81,6 @@ public class ParsedAggregationTests extends ESTestCase { @Override protected NamedXContentRegistry xContentRegistry() { - //TODO we may want to un-deprecate this Entry constructor if we are going to use it extensively, which I think we are NamedXContentRegistry.Entry entry = new NamedXContentRegistry.Entry(Aggregation.class, new ParseField("type"), (parser, name) -> TestParsedAggregation.fromXContent(parser, (String)name)); return new NamedXContentRegistry(Collections.singletonList(entry)); From 12e8a45de708ea9ae0acc65fb6e68c42577da792 Mon Sep 17 00:00:00 2001 From: javanna Date: Sat, 8 Apr 2017 00:41:32 +0200 Subject: [PATCH 06/62] remove some more TODOs from ParsedAggregation --- .../elasticsearch/search/aggregations/ParsedAggregation.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index 16b58dbf1f7..92cbe302296 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -33,7 +33,6 @@ import java.util.Map; */ public abstract class ParsedAggregation implements Aggregation, ToXContent { - //TODO move CommonFields out of InternalAggregation protected static void declareCommonFields(ObjectParser objectParser) { objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata), (parser, context) -> parser.map(), InternalAggregation.CommonFields.META); @@ -60,10 +59,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { //TODO it may make sense to move getType to the Aggregation interface given that we are duplicating it in both implementations protected abstract String getType(); - //TODO the only way to avoid duplicating this method is making Aggregation extend ToXContent - //and declare toXContent as a default method in it. Doesn't sound like the right thing to do. public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - //TODO move TYPED_KEYS_DELIMITER constant out of InternalAggregation // Concatenates the type and the name of the aggregation (ex: top_hits#foo) builder.startObject(String.join(InternalAggregation.TYPED_KEYS_DELIMITER, getType(), name)); if (this.metadata != null) { From 356532816ab781f1aa8bfd1d4bcbea10fda65c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 12 Apr 2017 11:58:57 +0200 Subject: [PATCH 07/62] Adding ParsedCardinality (#23973) Adding parsing of InternalCardinality xContent output. Parsing method will return a new implementation of the Cardinality interface, ParsedCardinality. --- .../aggregations/ParsedAggregation.java | 9 ++- .../cardinality/InternalCardinality.java | 1 + .../cardinality/ParsedCardinality.java | 80 +++++++++++++++++++ .../InternalAggregationTestCase.java | 11 ++- .../aggregations/ParsedAggregationTests.java | 2 +- .../cardinality/InternalCardinalityTests.java | 48 +++++++++++ 6 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index 92cbe302296..ae620ba601e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -38,7 +38,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { (parser, context) -> parser.map(), InternalAggregation.CommonFields.META); } - String name; + private String name; Map metadata; @Override @@ -46,6 +46,10 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { return name; } + protected void setName(String name) { + this.name = name; + } + @Override public final Map getMetaData() { return metadata; @@ -59,6 +63,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { //TODO it may make sense to move getType to the Aggregation interface given that we are duplicating it in both implementations protected abstract String getType(); + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { // Concatenates the type and the name of the aggregation (ex: top_hits#foo) builder.startObject(String.join(InternalAggregation.TYPED_KEYS_DELIMITER, getType(), name)); @@ -72,4 +77,4 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { } protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException; -} +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java index 028e97a69ff..ce1e9fc8939 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java @@ -128,3 +128,4 @@ public final class InternalCardinality extends InternalNumericMetricsAggregation return counts; } } + diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java new file mode 100644 index 00000000000..58f24953aac --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.cardinality; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; + +public class ParsedCardinality extends ParsedAggregation implements Cardinality { + + private long cardinalityValue; + + @Override + public String getValueAsString() { + // InternalCardinality doesn't print "value_as_string", but you can get a formated value using + // getValueAsString(). That method uses the raw formatter so we also use it here. + return DocValueFormat.RAW.format((double) cardinalityValue); + } + + @Override + public double value() { + return getValue(); + } + + @Override + public long getValue() { + return cardinalityValue; + } + + private void setValue(long cardinalityValue) { + this.cardinalityValue = cardinalityValue; + } + + @Override + protected String getType() { + return CardinalityAggregationBuilder.NAME; + } + + private static final ObjectParser PARSER = new ObjectParser<>( + CardinalityAggregationBuilder.NAME, true, ParsedCardinality::new); + + static { + declareCommonFields(PARSER); + PARSER.declareLong(ParsedCardinality::setValue, CommonFields.VALUE); + } + + public static ParsedCardinality fromXContent(XContentParser parser, final String name) { + ParsedCardinality cardinality = PARSER.apply(parser, null); + cardinality.setName(name); + return cardinality; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) + throws IOException { + builder.field(CommonFields.VALUE.getPreferredName(), cardinalityValue); + return builder; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index f899fde087c..1f0e9d2d74f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -98,10 +98,13 @@ public abstract class InternalAggregationTestCase private T createTestInstance(String name) { List pipelineAggregators = new ArrayList<>(); // TODO populate pipelineAggregators - Map metaData = new HashMap<>(); - int metaDataCount = randomBoolean() ? 0 : between(1, 10); - while (metaData.size() < metaDataCount) { - metaData.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + Map metaData = null; + if (randomBoolean()) { + metaData = new HashMap<>(); + int metaDataCount = between(0, 10); + while (metaData.size() < metaDataCount) { + metaData.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } } return createTestInstance(name, pipelineAggregators, metaData); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java index f4015ca8d6b..a79ccf5c91f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -105,7 +105,7 @@ public class ParsedAggregationTests extends ESTestCase { public static TestParsedAggregation fromXContent(XContentParser parser, String name) throws IOException { TestParsedAggregation parsedAgg = PARSER.parse(parser, null); - parsedAgg.name = name; + parsedAgg.setName(name); return parsedAgg; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java index 7c5809f323b..83b9dfbe46e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java @@ -19,20 +19,33 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.After; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; + public class InternalCardinalityTests extends InternalAggregationTestCase { private static List algos; private static int p; @@ -73,6 +86,41 @@ public class InternalCardinalityTests extends InternalAggregationTestCase ParsedCardinality.fromXContent(parser, (String) name)); + return new NamedXContentRegistry(Collections.singletonList(entry)); + } + @After public void cleanup() { Releasables.close(algos); From 5ccb4a0bbdb82674773b3b177f69fc96705d7913 Mon Sep 17 00:00:00 2001 From: javanna Date: Fri, 14 Apr 2017 11:22:44 +0200 Subject: [PATCH 08/62] fix typo in ParsedCardinality comment and add //norelease comment on DocValueFormat dep --- .../aggregations/metrics/cardinality/ParsedCardinality.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java index 58f24953aac..cac367cc7a2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java @@ -33,8 +33,9 @@ public class ParsedCardinality extends ParsedAggregation implements Cardinality @Override public String getValueAsString() { - // InternalCardinality doesn't print "value_as_string", but you can get a formated value using + // InternalCardinality doesn't print "value_as_string", but you can get a formatted value using // getValueAsString(). That method uses the raw formatter so we also use it here. + //norelease is it worth doing Double.toString(value) and removing the dependency to DocValueFormat.RAW ? return DocValueFormat.RAW.format((double) cardinalityValue); } From c0036d8516970a52eb33f963deb5d69c0b506fbc Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 18 Apr 2017 10:19:30 +0200 Subject: [PATCH 09/62] Add parsing for percentiles ranks (#23974) This commit adds the logic for parsing the percentiles ranks aggregations. --- .../elasticsearch/search/DocValueFormat.java | 17 ++ .../AbstractParsedPercentiles.java | 177 ++++++++++++++++++ .../percentiles/ParsedPercentileRanks.java | 33 ++++ .../hdr/ParsedHDRPercentileRanks.java | 47 +++++ .../tdigest/ParsedTDigestPercentileRanks.java | 47 +++++ .../InternalAggregationTestCase.java | 84 +++++++++ .../cardinality/InternalCardinalityTests.java | 52 +---- .../InternalPercentilesRanksTestCase.java | 68 +++++++ .../hdr/InternalHDRPercentilesRanksTests.java | 26 ++- .../InternalTDigestPercentilesRanksTests.java | 19 +- 10 files changed, 502 insertions(+), 68 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java index 4c32667aa2a..3e9bad6fe64 100644 --- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -393,5 +393,22 @@ public interface DocValueFormat extends NamedWriteable { public BytesRef parseBytesRef(String value) { throw new UnsupportedOperationException(); } + + @Override + public int hashCode() { + return Objects.hash(pattern); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Decimal that = (Decimal) o; + return Objects.equals(pattern, that.pattern); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java new file mode 100644 index 00000000000..f48e7257b1c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java @@ -0,0 +1,177 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +public abstract class AbstractParsedPercentiles extends ParsedAggregation implements Iterable { + + private final Map percentiles = new LinkedHashMap<>(); + private final Map percentilesAsString = new HashMap<>(); + + private boolean keyed; + + void addPercentile(Double key, Double value) { + percentiles.put(key, value); + } + + void addPercentileAsString(Double key, String valueAsString) { + percentilesAsString.put(key, valueAsString); + } + + Double getPercentile(double percent) { + if (percentiles.isEmpty()) { + return Double.NaN; + } + return percentiles.get(percent); + } + + String getPercentileAsString(double percent) { + String valueAsString = percentilesAsString.get(percent); + if (valueAsString != null) { + return valueAsString; + } + Double value = getPercentile(percent); + if (value != null) { + return DocValueFormat.RAW.format(value); + } + return null; + } + + void setKeyed(boolean keyed) { + this.keyed = keyed; + } + + @Override + public Iterator iterator() { + return new Iterator() { + final Iterator> iterator = percentiles.entrySet().iterator(); + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Percentile next() { + Map.Entry next = iterator.next(); + return new InternalPercentile(next.getKey(), next.getValue()); + } + }; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + final boolean valuesAsString = (percentilesAsString.isEmpty() == false); + if (keyed) { + builder.startObject(CommonFields.VALUES.getPreferredName()); + for (Map.Entry percentile : percentiles.entrySet()) { + Double key = percentile.getKey(); + builder.field(String.valueOf(key), percentile.getValue()); + + if (valuesAsString) { + builder.field(key + "_as_string", getPercentileAsString(key)); + } + } + builder.endObject(); + } else { + builder.startArray(CommonFields.VALUES.getPreferredName()); + for (Map.Entry percentile : percentiles.entrySet()) { + Double key = percentile.getKey(); + builder.startObject(); + { + builder.field(CommonFields.KEY.getPreferredName(), key); + builder.field(CommonFields.VALUE.getPreferredName(), percentile.getValue()); + if (valuesAsString) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), getPercentileAsString(key)); + } + } + builder.endObject(); + } + builder.endArray(); + } + return builder; + } + + protected static void declarePercentilesFields(ObjectParser objectParser) { + ParsedAggregation.declareCommonFields(objectParser); + + objectParser.declareField((parser, aggregation, context) -> { + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + aggregation.setKeyed(true); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token.isValue()) { + if (token == XContentParser.Token.VALUE_NUMBER) { + aggregation.addPercentile(Double.valueOf(parser.currentName()), parser.doubleValue()); + + } else if (token == XContentParser.Token.VALUE_STRING) { + int i = parser.currentName().indexOf("_as_string"); + if (i > 0) { + double key = Double.valueOf(parser.currentName().substring(0, i)); + aggregation.addPercentileAsString(key, parser.text()); + } else { + aggregation.addPercentile(Double.valueOf(parser.currentName()), Double.valueOf(parser.text())); + } + } + } + } + } else if (token == XContentParser.Token.START_ARRAY) { + aggregation.setKeyed(false); + + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + Double key = null; + Double value = null; + String valueAsString = null; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + key = parser.doubleValue(); + } else if (CommonFields.VALUE.getPreferredName().equals(currentFieldName)) { + value = parser.doubleValue(); + } else if (CommonFields.VALUE_AS_STRING.getPreferredName().equals(currentFieldName)) { + valueAsString = parser.text(); + } + } + } + if (key != null) { + aggregation.addPercentile(key, value); + if (valueAsString != null) { + aggregation.addPercentileAsString(key, valueAsString); + } + } + } + } + }, CommonFields.VALUES, ObjectParser.ValueType.OBJECT_ARRAY); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java new file mode 100644 index 00000000000..de3e81e966c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles; + +public abstract class ParsedPercentileRanks extends AbstractParsedPercentiles implements PercentileRanks { + + @Override + public double percent(double value) { + return getPercentile(value); + } + + @Override + public String percentAsString(double value) { + return getPercentileAsString(value); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java new file mode 100644 index 00000000000..31fc8f88cfc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; + +import java.io.IOException; + +public class ParsedHDRPercentileRanks extends ParsedPercentileRanks { + + @Override + protected String getType() { + return InternalHDRPercentileRanks.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedHDRPercentileRanks.class.getSimpleName(), true, ParsedHDRPercentileRanks::new); + static { + AbstractParsedPercentiles.declarePercentilesFields(PARSER); + } + + public static ParsedHDRPercentileRanks fromXContent(XContentParser parser, String name) throws IOException { + ParsedHDRPercentileRanks aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java new file mode 100644 index 00000000000..57f3df04115 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; + +import java.io.IOException; + +public class ParsedTDigestPercentileRanks extends ParsedPercentileRanks { + + @Override + protected String getType() { + return InternalTDigestPercentileRanks.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedTDigestPercentileRanks.class.getSimpleName(), true, ParsedTDigestPercentileRanks::new); + static { + AbstractParsedPercentiles.declarePercentilesFields(PARSER); + } + + public static ParsedTDigestPercentileRanks fromXContent(XContentParser parser, String name) throws IOException { + ParsedTDigestPercentileRanks aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 1f0e9d2d74f..d6b50f69bc9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -19,27 +19,61 @@ package org.elasticsearch.search.aggregations; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.Matchers.containsString; public abstract class InternalAggregationTestCase extends AbstractWireSerializingTestCase { + private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables()); + private final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(getNamedXContents()); + + static List getNamedXContents() { + Map> namedXContents = new HashMap<>(); + namedXContents.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c)); + namedXContents.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c)); + namedXContents.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c)); + + return namedXContents.entrySet().stream() + .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) + .collect(Collectors.toList()); + } + protected abstract T createTestInstance(String name, List pipelineAggregators, Map metaData); /** Return an instance on an unmapped field. */ @@ -125,4 +159,54 @@ public abstract class InternalAggregationTestCase protected NamedWriteableRegistry getNamedWriteableRegistry() { return namedWriteableRegistry; } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return namedXContentRegistry; + } + + public final void testFromXContent() throws IOException { + final NamedXContentRegistry xContentRegistry = xContentRegistry(); + final T aggregation = createTestInstance(); + + final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); + final boolean humanReadable = randomBoolean(); + final XContentType xContentType = randomFrom(XContentType.values()); + final BytesReference originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable); + + Aggregation parsedAggregation; + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, originalBytes)) { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + + String currentName = parser.currentName(); + int i = currentName.indexOf(InternalAggregation.TYPED_KEYS_DELIMITER); + String aggType = currentName.substring(0, i); + String aggName = currentName.substring(i + 1); + + parsedAggregation = parser.namedObject(Aggregation.class, aggType, aggName); + + assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken()); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + + assertEquals(aggregation.getName(), parsedAggregation.getName()); + assertEquals(aggregation.getMetaData(), parsedAggregation.getMetaData()); + + assertTrue(parsedAggregation instanceof ParsedAggregation); + assertEquals(aggregation.getType(), ((ParsedAggregation) parsedAggregation).getType()); + + final BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); + assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); + assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation); + + } catch (NamedXContentRegistry.UnknownNamedObjectException e) { + //norelease Remove this catch block when all aggregations can be parsed back. + assertThat(e.getMessage(), containsString("Unknown Aggregation")); + } + } + + //norelease TODO make abstract + protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java index 83b9dfbe46e..efa5438ae3c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java @@ -19,33 +19,21 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; - public class InternalCardinalityTests extends InternalAggregationTestCase { private static List algos; private static int p; @@ -86,39 +74,13 @@ public class InternalCardinalityTests extends InternalAggregationTestCase ParsedCardinality.fromXContent(parser, (String) name)); - return new NamedXContentRegistry(Collections.singletonList(entry)); + protected void assertFromXContent(InternalCardinality aggregation, ParsedAggregation parsedAggregation) { + assertTrue(parsedAggregation instanceof ParsedCardinality); + ParsedCardinality parsed = (ParsedCardinality) parsedAggregation; + + assertEquals(aggregation.getValue(), parsed.getValue(), Double.MIN_VALUE); + assertEquals(aggregation.getValueAsString(), parsed.getValueAsString()); } @After diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java new file mode 100644 index 00000000000..23acfcd779a --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.List; +import java.util.Map; + +public abstract class InternalPercentilesRanksTestCase extends InternalAggregationTestCase { + + @Override + protected final T createTestInstance(String name, List pipelineAggregators, Map metaData) { + final boolean keyed = randomBoolean(); + final DocValueFormat format = randomFrom(DocValueFormat.RAW, new DocValueFormat.Decimal("###.##")); + List randomCdfValues = randomSubsetOf(randomIntBetween(1, 5), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); + double[] cdfValues = new double[randomCdfValues.size()]; + for (int i = 0; i < randomCdfValues.size(); i++) { + cdfValues[i] = randomCdfValues.get(i); + } + return createTestInstance(name, pipelineAggregators, metaData, cdfValues, keyed, format); + } + + protected abstract T createTestInstance(String name, List aggregators, Map metadata, + double[] cdfValues, boolean keyed, DocValueFormat format); + + @Override + protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + assertTrue(aggregation instanceof PercentileRanks); + PercentileRanks percentileRanks = (PercentileRanks) aggregation; + + assertTrue(parsedAggregation instanceof PercentileRanks); + PercentileRanks parsedPercentileRanks = (PercentileRanks) parsedAggregation; + + for (Percentile percentile : percentileRanks) { + Double value = percentile.getValue(); + assertEquals(percentileRanks.percent(value), parsedPercentileRanks.percent(value), 0); + assertEquals(percentileRanks.percentAsString(value), parsedPercentileRanks.percentAsString(value)); + } + + Class parsedClass = parsedParsedPercentileRanksClass(); + assertNotNull(parsedClass); + assertTrue(parsedClass.isInstance(parsedAggregation)); + } + + protected abstract Class parsedParsedPercentileRanksClass(); +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java index 15207e1d314..a3fe4d62d14 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java @@ -22,28 +22,20 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; -import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; import java.util.Map; -public class InternalHDRPercentilesRanksTests extends InternalAggregationTestCase { +public class InternalHDRPercentilesRanksTests extends InternalPercentilesRanksTestCase { @Override - protected InternalHDRPercentileRanks createTestInstance(String name, List pipelineAggregators, - Map metaData) { - double[] cdfValues = new double[] { 0.5 }; - int numberOfSignificantValueDigits = 3; - DoubleHistogram state = new DoubleHistogram(numberOfSignificantValueDigits); - int numValues = randomInt(100); - for (int i = 0; i < numValues; ++i) { - state.recordValue(randomDouble()); - } - boolean keyed = false; - DocValueFormat format = DocValueFormat.RAW; - return new InternalHDRPercentileRanks(name, cdfValues, state, keyed, format, pipelineAggregators, metaData); + protected InternalHDRPercentileRanks createTestInstance(String name, List aggregators, Map metadata, + double[] cdfValues, boolean keyed, DocValueFormat format) { + DoubleHistogram state = new DoubleHistogram(3); + return new InternalHDRPercentileRanks(name, cdfValues, state, keyed, format, aggregators, metadata); } @Override @@ -61,4 +53,8 @@ public class InternalHDRPercentilesRanksTests extends InternalAggregationTestCas return InternalHDRPercentileRanks::new; } + @Override + protected Class parsedParsedPercentileRanksClass() { + return ParsedHDRPercentileRanks.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java index bc1df930f6a..30d416763c1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java @@ -21,26 +21,25 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; import java.util.Map; -public class InternalTDigestPercentilesRanksTests extends InternalAggregationTestCase { +public class InternalTDigestPercentilesRanksTests extends InternalPercentilesRanksTestCase { @Override - protected InternalTDigestPercentileRanks createTestInstance(String name, List pipelineAggregators, - Map metaData) { - double[] cdfValues = new double[] { 0.5 }; + protected InternalTDigestPercentileRanks createTestInstance(String name, List aggregators, + Map metadata, + double[] cdfValues, boolean keyed, DocValueFormat format) { TDigestState state = new TDigestState(100); int numValues = randomInt(100); for (int i = 0; i < numValues; ++i) { state.add(randomDouble()); } - boolean keyed = false; - DocValueFormat format = DocValueFormat.RAW; - return new InternalTDigestPercentileRanks(name, cdfValues, state, keyed, format, pipelineAggregators, metaData); + return new InternalTDigestPercentileRanks(name, cdfValues, state, keyed, format, aggregators, metadata); } @Override @@ -71,4 +70,8 @@ public class InternalTDigestPercentilesRanksTests extends InternalAggregationTes return InternalTDigestPercentileRanks::new; } + @Override + protected Class parsedParsedPercentileRanksClass() { + return ParsedTDigestPercentileRanks.class; + } } From c1ba6997ff64008e8a843fda77905a54ff33bcec Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 18 Apr 2017 16:54:17 +0200 Subject: [PATCH 10/62] AbstractParsedPercentiles should use Percentile class (#24160) Now the Percentile interface has been merged with the InternalPercentile class in core (#24154) the AbstractParsedPercentiles should use it. This commit also changes InternalPercentilesRanksTestCase so that it now tests the iterator obtained from parsed percentiles ranks aggregations. Adding this new test raised an issue in the iterators where key and value are "swapped" in internal implementations when building the iterators (see InternalTDigestPercentileRanks.Iter constructor that accepts the `keys` as the first parameter named `values`, each key being mapped to the `value` field of Percentile class). This is because percentiles ranks aggs inverts percentiles/values compared to the percentiles aggs. * Add assume in InternalAggregationTestCase * Update after Luca review --- .../AbstractParsedPercentiles.java | 2 +- .../hdr/ParsedHDRPercentileRanks.java | 19 +++++++++ .../tdigest/ParsedTDigestPercentileRanks.java | 19 +++++++++ .../InternalAggregationTestCase.java | 9 ++-- .../InternalPercentilesRanksTestCase.java | 42 +++++++++++++++++++ 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java index f48e7257b1c..a45d089cc79 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java @@ -81,7 +81,7 @@ public abstract class AbstractParsedPercentiles extends ParsedAggregation implem @Override public Percentile next() { Map.Entry next = iterator.next(); - return new InternalPercentile(next.getKey(), next.getValue()); + return new Percentile(next.getKey(), next.getValue()); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java index 31fc8f88cfc..79749df1176 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java @@ -23,8 +23,10 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import java.io.IOException; +import java.util.Iterator; public class ParsedHDRPercentileRanks extends ParsedPercentileRanks { @@ -33,6 +35,23 @@ public class ParsedHDRPercentileRanks extends ParsedPercentileRanks { return InternalHDRPercentileRanks.NAME; } + @Override + public Iterator iterator() { + final Iterator iterator = super.iterator(); + return new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Percentile next() { + Percentile percentile = iterator.next(); + return new Percentile(percentile.getValue(), percentile.getPercent()); + } + }; + } + private static ObjectParser PARSER = new ObjectParser<>(ParsedHDRPercentileRanks.class.getSimpleName(), true, ParsedHDRPercentileRanks::new); static { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java index 57f3df04115..3d51e98d622 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java @@ -23,8 +23,10 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import java.io.IOException; +import java.util.Iterator; public class ParsedTDigestPercentileRanks extends ParsedPercentileRanks { @@ -33,6 +35,23 @@ public class ParsedTDigestPercentileRanks extends ParsedPercentileRanks { return InternalTDigestPercentileRanks.NAME; } + @Override + public Iterator iterator() { + final Iterator iterator = super.iterator(); + return new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Percentile next() { + Percentile percentile = iterator.next(); + return new Percentile(percentile.getValue(), percentile.getPercent()); + } + }; + } + private static ObjectParser PARSER = new ObjectParser<>(ParsedTDigestPercentileRanks.class.getSimpleName(), true, ParsedTDigestPercentileRanks::new); static { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index d6b50f69bc9..e6c5098761c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -54,7 +54,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; -import static org.hamcrest.Matchers.containsString; public abstract class InternalAggregationTestCase extends AbstractWireSerializingTestCase { @@ -169,6 +168,10 @@ public abstract class InternalAggregationTestCase final NamedXContentRegistry xContentRegistry = xContentRegistry(); final T aggregation = createTestInstance(); + //norelease Remove this assumption when all aggregations can be parsed back. + assumeTrue("This test does not support the aggregation type yet", + getNamedXContents().stream().filter(entry -> entry.name.match(aggregation.getType())).count() > 0); + final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); final boolean humanReadable = randomBoolean(); final XContentType xContentType = randomFrom(XContentType.values()); @@ -199,10 +202,6 @@ public abstract class InternalAggregationTestCase final BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation); - - } catch (NamedXContentRegistry.UnknownNamedObjectException e) { - //norelease Remove this catch block when all aggregations can be parsed back. - assertThat(e.getMessage(), containsString("Unknown Aggregation")); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java index 23acfcd779a..be7d55e5447 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java @@ -19,15 +19,26 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import java.io.IOException; +import java.util.Iterator; import java.util.List; import java.util.Map; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; + public abstract class InternalPercentilesRanksTestCase extends InternalAggregationTestCase { @Override @@ -64,5 +75,36 @@ public abstract class InternalPercentilesRanksTestCase it = ((PercentileRanks) aggregation).iterator(); + final Iterator parsedIt = ((PercentileRanks) parsedAggregation).iterator(); + while (it.hasNext()) { + assertEquals(it.next(), parsedIt.next()); + } + } + protected abstract Class parsedParsedPercentileRanksClass(); } From 75fdc9449fd1d544cb56ae7e101f6ce323fc331d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Apr 2017 17:32:09 +0200 Subject: [PATCH 11/62] Adding parsing for InternalMax and InternalMin --- .../common/xcontent/ObjectParser.java | 1 + .../aggregations/ParsedAggregation.java | 2 +- ...dSingleValueNumericMetricsAggregation.java | 72 +++++++++++++++++++ .../cardinality/ParsedCardinality.java | 2 +- .../aggregations/metrics/max/ParsedMax.java | 62 ++++++++++++++++ .../aggregations/metrics/min/ParsedMin.java | 62 ++++++++++++++++ .../AbstractParsedPercentiles.java | 2 +- .../InternalAggregationTestCase.java | 6 ++ .../aggregations/ParsedAggregationTests.java | 2 +- .../metrics/InternalMaxTests.java | 22 +++++- .../metrics/min/InternalMinTests.java | 20 +++++- 11 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index 0a7e71c6f06..a3a886dadfd 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -448,6 +448,7 @@ public final class ObjectParser extends AbstractObjectParser objectParser) { + protected static void declareAggregationFields(ObjectParser objectParser) { objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata), (parser, context) -> parser.map(), InternalAggregation.CommonFields.META); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java new file mode 100644 index 00000000000..11da4c09f57 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; + +public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedAggregation implements NumericMetricsAggregation.SingleValue { + + protected double value; + protected String valueAsString; + + @Override + public String getValueAsString() { + if (valueAsString != null) { + return valueAsString; + } else { + return Double.toString(value); + } + } + + @Override + public double value() { + return value; + } + + protected void setValue(double value) { + this.value = value; + } + + protected void setValueAsString(String valueAsString) { + this.valueAsString = valueAsString; + } + + protected static double parseValue(XContentParser parser, double defaultNullValue) throws IOException { + Token currentToken = parser.currentToken(); + if (currentToken == XContentParser.Token.VALUE_NUMBER || currentToken == XContentParser.Token.VALUE_STRING) { + return parser.doubleValue(); + } else { + return defaultNullValue; + } + } + + protected static void declareSingeValueFields(ObjectParser objectParser, + double defaultNullValue) { + declareAggregationFields(objectParser); + objectParser.declareField(ParsedSingleValueNumericMetricsAggregation::setValue, + (parser, context) -> parseValue(parser, defaultNullValue), CommonFields.VALUE, ValueType.DOUBLE_OR_NULL); + objectParser.declareString(ParsedSingleValueNumericMetricsAggregation::setValueAsString, CommonFields.VALUE_AS_STRING); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java index cac367cc7a2..5c7a1bad427 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java @@ -62,7 +62,7 @@ public class ParsedCardinality extends ParsedAggregation implements Cardinality CardinalityAggregationBuilder.NAME, true, ParsedCardinality::new); static { - declareCommonFields(PARSER); + declareAggregationFields(PARSER); PARSER.declareLong(ParsedCardinality::setValue, CommonFields.VALUE); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java new file mode 100644 index 00000000000..f3a4ff85f80 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.max; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; + +public class ParsedMax extends ParsedSingleValueNumericMetricsAggregation implements Max { + + @Override + public double getValue() { + return value(); + } + + @Override + protected String getType() { + return MaxAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + boolean hasValue = !Double.isInfinite(value); + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedMax.class.getSimpleName(), true, ParsedMax::new); + + static { + declareSingeValueFields(PARSER, Double.NEGATIVE_INFINITY); + } + + public static ParsedMax fromXContent(XContentParser parser, final String name) { + ParsedMax max = PARSER.apply(parser, null); + max.setName(name); + return max; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java new file mode 100644 index 00000000000..5be0444e478 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.min; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; + +public class ParsedMin extends ParsedSingleValueNumericMetricsAggregation implements Min { + + @Override + public double getValue() { + return value(); + } + + @Override + protected String getType() { + return MinAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + boolean hasValue = !Double.isInfinite(value); + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedMin.class.getSimpleName(), true, ParsedMin::new); + + static { + declareSingeValueFields(PARSER, Double.POSITIVE_INFINITY); + } + + public static ParsedMin fromXContent(XContentParser parser, final String name) { + ParsedMin min = PARSER.apply(parser, null); + min.setName(name); + return min; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java index a45d089cc79..e3152bac3a3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java @@ -120,7 +120,7 @@ public abstract class AbstractParsedPercentiles extends ParsedAggregation implem } protected static void declarePercentilesFields(ObjectParser objectParser) { - ParsedAggregation.declareCommonFields(objectParser); + ParsedAggregation.declareAggregationFields(objectParser); objectParser.declareField((parser, aggregation, context) -> { XContentParser.Token token = parser.currentToken(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index e6c5098761c..0db3277f0b1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -35,6 +35,10 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.max.ParsedMax; +import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.min.ParsedMin; import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks; @@ -67,6 +71,8 @@ public abstract class InternalAggregationTestCase namedXContents.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c)); namedXContents.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c)); namedXContents.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c)); + namedXContents.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)); + namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java index a79ccf5c91f..f56ae175a86 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsedAggregationTests.java @@ -90,7 +90,7 @@ public class ParsedAggregationTests extends ESTestCase { private static ObjectParser PARSER = new ObjectParser<>("testAggParser", TestParsedAggregation::new); static { - ParsedAggregation.declareCommonFields(PARSER); + ParsedAggregation.declareAggregationFields(PARSER); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java index de045ff533e..ec178feab71 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java @@ -22,18 +22,21 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; +import org.elasticsearch.search.aggregations.metrics.max.ParsedMax; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; import java.util.Map; public class InternalMaxTests extends InternalAggregationTestCase { + @Override protected InternalMax createTestInstance(String name, List pipelineAggregators, Map metaData) { - return new InternalMax(name, randomDouble(), - randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), pipelineAggregators, - metaData); + double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY }); + DocValueFormat formatter = randomFrom(new DocValueFormat.Decimal("###.##"), DocValueFormat.BOOLEAN, DocValueFormat.RAW); + return new InternalMax(name, value, formatter, pipelineAggregators, metaData); } @Override @@ -45,4 +48,17 @@ public class InternalMaxTests extends InternalAggregationTestCase { protected void assertReduced(InternalMax reduced, List inputs) { assertEquals(inputs.stream().mapToDouble(InternalMax::value).max().getAsDouble(), reduced.value(), 0); } + + @Override + protected void assertFromXContent(InternalMax max, ParsedAggregation parsedAggregation) { + ParsedMax parsed = ((ParsedMax) parsedAggregation); + if (Double.isInfinite(max.getValue()) == false) { + assertEquals(max.getValue(), parsed.getValue(), Double.MIN_VALUE); + assertEquals(max.getValueAsString(), parsed.getValueAsString()); + } else { + // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we + // cannot differentiate between them. Also we cannot recreate the exact String representation + assertEquals(parsed.getValue(), Double.NEGATIVE_INFINITY, 0); + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java index f93e7c5c81e..82ba4dec5ef 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.metrics.min; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -30,9 +31,9 @@ import java.util.Map; public class InternalMinTests extends InternalAggregationTestCase { @Override protected InternalMin createTestInstance(String name, List pipelineAggregators, Map metaData) { - return new InternalMin(name, randomDouble(), - randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), pipelineAggregators, - metaData); + double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY }); + DocValueFormat formatter = randomFrom(new DocValueFormat.Decimal("###.##"), DocValueFormat.BOOLEAN, DocValueFormat.RAW); + return new InternalMin(name, value, formatter, pipelineAggregators, metaData); } @Override @@ -44,4 +45,17 @@ public class InternalMinTests extends InternalAggregationTestCase { protected void assertReduced(InternalMin reduced, List inputs) { assertEquals(inputs.stream().mapToDouble(InternalMin::value).min().getAsDouble(), reduced.value(), 0); } + + @Override + protected void assertFromXContent(InternalMin min, ParsedAggregation parsedAggregation) { + ParsedMin parsed = ((ParsedMin) parsedAggregation); + if (Double.isInfinite(min.getValue()) == false) { + assertEquals(min.getValue(), parsed.getValue(), Double.MIN_VALUE); + assertEquals(min.getValueAsString(), parsed.getValueAsString()); + } else { + // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we + // cannot differentiate between them. Also we cannot recreate the exact String representation + assertEquals(parsed.getValue(), Double.POSITIVE_INFINITY, 0); + } + } } From 695b2858f4f22509b264020fc535ca808433f2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Apr 2017 17:41:44 +0200 Subject: [PATCH 12/62] Adding parsing for InternalSum --- .../aggregations/metrics/sum/ParsedSum.java | 61 +++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../metrics/sum/InternalSumTests.java | 12 +++- 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java new file mode 100644 index 00000000000..15a85aa3f87 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.sum; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; + +public class ParsedSum extends ParsedSingleValueNumericMetricsAggregation implements Sum { + + @Override + public double getValue() { + return value(); + } + + @Override + protected String getType() { + return SumAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(CommonFields.VALUE.getPreferredName(), value); + if (valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedSum.class.getSimpleName(), true, ParsedSum::new); + + static { + declareSingeValueFields(PARSER, Double.NEGATIVE_INFINITY); + } + + public static ParsedSum fromXContent(XContentParser parser, final String name) { + ParsedSum sum = PARSER.apply(parser, null); + sum.setName(name); + return sum; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 0db3277f0b1..e1d3fe5bcde 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -43,6 +43,8 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDR import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -73,6 +75,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c)); namedXContents.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)); namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); + namedXContents.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java index 44e5ea940ad..6fb61257a9e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSumTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.metrics.sum; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -30,7 +31,9 @@ public class InternalSumTests extends InternalAggregationTestCase { @Override protected InternalSum createTestInstance(String name, List pipelineAggregators, Map metaData) { - return new InternalSum(name, randomDouble(), DocValueFormat.RAW, pipelineAggregators, metaData); + double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY }); + DocValueFormat formatter = randomFrom(new DocValueFormat.Decimal("###.##"), DocValueFormat.BOOLEAN, DocValueFormat.RAW); + return new InternalSum(name, value, formatter, pipelineAggregators, metaData); } @Override @@ -43,4 +46,11 @@ public class InternalSumTests extends InternalAggregationTestCase { double expectedSum = inputs.stream().mapToDouble(InternalSum::getValue).sum(); assertEquals(expectedSum, reduced.getValue(), 0.0001d); } + + @Override + protected void assertFromXContent(InternalSum sum, ParsedAggregation parsedAggregation) { + ParsedSum parsed = ((ParsedSum) parsedAggregation); + assertEquals(sum.getValue(), parsed.getValue(), Double.MIN_VALUE); + assertEquals(sum.getValueAsString(), parsed.getValueAsString()); + } } From 5f96972b042a6a99689601c8e9a873419d341c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Apr 2017 17:48:11 +0200 Subject: [PATCH 13/62] Adding parsing for InternalAvg --- .../aggregations/metrics/avg/ParsedAvg.java | 64 +++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../metrics/avg/InternalAvgTests.java | 17 ++++- 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java new file mode 100644 index 00000000000..ed4b9416908 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.avg; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; + +public class ParsedAvg extends ParsedSingleValueNumericMetricsAggregation implements Avg { + + @Override + public double getValue() { + return value(); + } + + @Override + protected String getType() { + return AvgAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + // InternalAvg renders value only if the avg normalizer (count) is not 0. + // We parse back `null` as Double.POSITIVE_INFINITY so we check for that value here to get the same xContent output + boolean hasValue = value != Double.POSITIVE_INFINITY; + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedAvg.class.getSimpleName(), true, ParsedAvg::new); + + static { + declareSingeValueFields(PARSER, Double.POSITIVE_INFINITY); + } + + public static ParsedAvg fromXContent(XContentParser parser, final String name) { + ParsedAvg avg = PARSER.apply(parser, null); + avg.setName(name); + return avg; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index e1d3fe5bcde..bfa17179851 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -33,6 +33,8 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; @@ -76,6 +78,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)); namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); namedXContents.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)); + namedXContents.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java index 0600d7299b4..bfdf37d5775 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvgTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.metrics.avg; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -31,9 +32,9 @@ public class InternalAvgTests extends InternalAggregationTestCase { @Override protected InternalAvg createTestInstance(String name, List pipelineAggregators, Map metaData) { - return new InternalAvg(name, randomDoubleBetween(0, 100000, true), randomNonNegativeLong() % 100000, - randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), pipelineAggregators, - metaData); + DocValueFormat formatter = randomFrom(new DocValueFormat.Decimal("###.##"), DocValueFormat.BOOLEAN, DocValueFormat.RAW); + long count = frequently() ? randomNonNegativeLong() % 100000 : 0; + return new InternalAvg(name, randomDoubleBetween(0, 100000, true), count, formatter, pipelineAggregators, metaData); } @Override @@ -53,4 +54,14 @@ public class InternalAvgTests extends InternalAggregationTestCase { assertEquals(sum, reduced.getSum(), 0.0000001); assertEquals(sum / counts, reduced.value(), 0.0000001); } + + @Override + protected void assertFromXContent(InternalAvg avg, ParsedAggregation parsedAggregation) { + ParsedAvg parsed = ((ParsedAvg) parsedAggregation); + assertEquals(avg.getValue(), parsed.getValue(), Double.MIN_VALUE); + // we don't print out VALUE_AS_STRING for avg.getCount() == 0, so we cannot get the exact same value back + if (avg.getCount() != 0) { + assertEquals(avg.getValueAsString(), parsed.getValueAsString()); + } + } } From bc646cf7ad74998be73febf21fd8a258e6ab113a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Apr 2017 17:56:16 +0200 Subject: [PATCH 14/62] Adding parsing for InternalValueCount --- .../metrics/valuecount/ParsedValueCount.java | 74 +++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../valuecount/InternalValueCountTests.java | 12 ++- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java new file mode 100644 index 00000000000..012323f0041 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ParsedValueCount.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.valuecount; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; + +public class ParsedValueCount extends ParsedAggregation implements ValueCount { + + private long valueCount; + + @Override + public double value() { + return getValue(); + } + + @Override + public long getValue() { + return valueCount; + } + + @Override + public String getValueAsString() { + // InternalValueCount doesn't print "value_as_string", but you can get a formatted value using + // getValueAsString() using the raw formatter and converting the value to double + return Double.toString(valueCount); + } + + @Override + protected String getType() { + return ValueCountAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(CommonFields.VALUE.getPreferredName(), valueCount); + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedValueCount.class.getSimpleName(), true, + ParsedValueCount::new); + + static { + declareAggregationFields(PARSER); + PARSER.declareLong((agg, value) -> agg.valueCount = value, CommonFields.VALUE); + } + + public static ParsedValueCount fromXContent(XContentParser parser, final String name) { + ParsedValueCount sum = PARSER.apply(parser, null); + sum.setName(name); + return sum; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index bfa17179851..04c0ed58f39 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -47,6 +47,8 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.Interna import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; +import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -79,6 +81,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); namedXContents.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)); namedXContents.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c)); + namedXContents.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java index 3de87898778..17df4b89dc6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.metrics.valuecount; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -29,9 +30,8 @@ import java.util.Map; public class InternalValueCountTests extends InternalAggregationTestCase { @Override - protected InternalValueCount createTestInstance(String name, - List pipelineAggregators, - Map metaData) { + protected InternalValueCount createTestInstance(String name, List pipelineAggregators, + Map metaData) { return new InternalValueCount(name, randomIntBetween(0, 100), pipelineAggregators, metaData); } @@ -44,4 +44,10 @@ public class InternalValueCountTests extends InternalAggregationTestCase instanceReader() { return InternalValueCount::new; } + + @Override + protected void assertFromXContent(InternalValueCount valueCount, ParsedAggregation parsedAggregation) { + assertEquals(valueCount.getValue(), ((ParsedValueCount) parsedAggregation).getValue(), 0); + assertEquals(valueCount.getValueAsString(), ((ParsedValueCount) parsedAggregation).getValueAsString()); + } } From 210e101f6d7677e103b24beb33726a8ed8672c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Apr 2017 17:56:31 +0200 Subject: [PATCH 15/62] Minor changes in ParsedCardinality --- .../metrics/cardinality/ParsedCardinality.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java index 5c7a1bad427..77356db5197 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/ParsedCardinality.java @@ -49,21 +49,17 @@ public class ParsedCardinality extends ParsedAggregation implements Cardinality return cardinalityValue; } - private void setValue(long cardinalityValue) { - this.cardinalityValue = cardinalityValue; - } - @Override protected String getType() { return CardinalityAggregationBuilder.NAME; } private static final ObjectParser PARSER = new ObjectParser<>( - CardinalityAggregationBuilder.NAME, true, ParsedCardinality::new); + ParsedCardinality.class.getSimpleName(), true, ParsedCardinality::new); static { declareAggregationFields(PARSER); - PARSER.declareLong(ParsedCardinality::setValue, CommonFields.VALUE); + PARSER.declareLong((agg, value) -> agg.cardinalityValue = value, CommonFields.VALUE); } public static ParsedCardinality fromXContent(XContentParser parser, final String name) { From bf5cfabe040ae131d94fe3e627b1479ea098f7ac Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 19 Apr 2017 10:23:25 +0200 Subject: [PATCH 16/62] Fix checkstyle violation --- .../metrics/ParsedSingleValueNumericMetricsAggregation.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java index 11da4c09f57..9c7972e638f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java @@ -26,7 +26,8 @@ import org.elasticsearch.search.aggregations.ParsedAggregation; import java.io.IOException; -public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedAggregation implements NumericMetricsAggregation.SingleValue { +public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedAggregation + implements NumericMetricsAggregation.SingleValue { protected double value; protected String valueAsString; From 4562c8a345d76024f6ac1812249ab3e187a971ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 19 Apr 2017 12:58:26 +0200 Subject: [PATCH 17/62] Add parsing for InternalSimpleValue and InternalDerivative (#24162) --- ...dSingleValueNumericMetricsAggregation.java | 2 +- .../aggregations/metrics/avg/ParsedAvg.java | 2 +- .../aggregations/metrics/max/ParsedMax.java | 2 +- .../aggregations/metrics/min/ParsedMin.java | 2 +- .../aggregations/metrics/sum/ParsedSum.java | 2 +- .../pipeline/ParsedSimpleValue.java | 58 ++++++++++++++ .../pipeline/derivative/ParsedDerivative.java | 79 +++++++++++++++++++ .../InternalAggregationTestCase.java | 6 ++ .../pipeline/InternalSimpleValueTests.java | 23 ++++-- .../derivative/InternalDerivativeTests.java | 21 ++++- 10 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java index 9c7972e638f..5eb0a7223a1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java @@ -63,7 +63,7 @@ public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedA } } - protected static void declareSingeValueFields(ObjectParser objectParser, + protected static void declareSingleValueFields(ObjectParser objectParser, double defaultNullValue) { declareAggregationFields(objectParser); objectParser.declareField(ParsedSingleValueNumericMetricsAggregation::setValue, diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java index ed4b9416908..d77fd2bb3ad 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/ParsedAvg.java @@ -53,7 +53,7 @@ public class ParsedAvg extends ParsedSingleValueNumericMetricsAggregation implem private static final ObjectParser PARSER = new ObjectParser<>(ParsedAvg.class.getSimpleName(), true, ParsedAvg::new); static { - declareSingeValueFields(PARSER, Double.POSITIVE_INFINITY); + declareSingleValueFields(PARSER, Double.POSITIVE_INFINITY); } public static ParsedAvg fromXContent(XContentParser parser, final String name) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java index f3a4ff85f80..86eab0025ac 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/ParsedMax.java @@ -51,7 +51,7 @@ public class ParsedMax extends ParsedSingleValueNumericMetricsAggregation implem private static final ObjectParser PARSER = new ObjectParser<>(ParsedMax.class.getSimpleName(), true, ParsedMax::new); static { - declareSingeValueFields(PARSER, Double.NEGATIVE_INFINITY); + declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY); } public static ParsedMax fromXContent(XContentParser parser, final String name) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java index 5be0444e478..c50cd5c3615 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/ParsedMin.java @@ -51,7 +51,7 @@ public class ParsedMin extends ParsedSingleValueNumericMetricsAggregation implem private static final ObjectParser PARSER = new ObjectParser<>(ParsedMin.class.getSimpleName(), true, ParsedMin::new); static { - declareSingeValueFields(PARSER, Double.POSITIVE_INFINITY); + declareSingleValueFields(PARSER, Double.POSITIVE_INFINITY); } public static ParsedMin fromXContent(XContentParser parser, final String name) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java index 15a85aa3f87..86d3e1c4295 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/ParsedSum.java @@ -50,7 +50,7 @@ public class ParsedSum extends ParsedSingleValueNumericMetricsAggregation implem private static final ObjectParser PARSER = new ObjectParser<>(ParsedSum.class.getSimpleName(), true, ParsedSum::new); static { - declareSingeValueFields(PARSER, Double.NEGATIVE_INFINITY); + declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY); } public static ParsedSum fromXContent(XContentParser parser, final String name) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java new file mode 100644 index 00000000000..7d1bcec3bc7 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedSimpleValue.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; + +public class ParsedSimpleValue extends ParsedSingleValueNumericMetricsAggregation implements SimpleValue { + + @Override + protected String getType() { + return InternalSimpleValue.NAME; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedSimpleValue.class.getSimpleName(), true, + ParsedSimpleValue::new); + + static { + declareSingleValueFields(PARSER, Double.NaN); + } + + public static ParsedSimpleValue fromXContent(XContentParser parser, final String name) { + ParsedSimpleValue simpleValue = PARSER.apply(parser, null); + simpleValue.setName(name); + return simpleValue; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + boolean hasValue = Double.isNaN(value) == false; + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + return builder; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java new file mode 100644 index 00000000000..ed463239b24 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.derivative; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; + +import java.io.IOException; + +public class ParsedDerivative extends ParsedSimpleValue implements Derivative { + + private double normalizedValue; + private String normalizedAsString; + private boolean hasNormalizationFactor; + private static final ParseField NORMALIZED_AS_STRING = new ParseField("normalized_value_as_string"); + private static final ParseField NORMALIZED = new ParseField("normalized_value"); + + @Override + public double normalizedValue() { + return this.normalizedValue; + } + + @Override + protected String getType() { + return DerivativePipelineAggregationBuilder.NAME; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedDerivative.class.getSimpleName(), true, + ParsedDerivative::new); + + static { + declareSingleValueFields(PARSER, Double.NaN); + PARSER.declareField((agg, normalized) -> { + agg.normalizedValue = normalized; + agg.hasNormalizationFactor = true; + }, (parser, context) -> parseValue(parser, Double.NaN), NORMALIZED, ValueType.DOUBLE_OR_NULL); + PARSER.declareString((agg, normalAsString) -> agg.normalizedAsString = normalAsString, NORMALIZED_AS_STRING); + } + + public static ParsedDerivative fromXContent(XContentParser parser, final String name) { + ParsedDerivative derivative = PARSER.apply(parser, null); + derivative.setName(name); + return derivative; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + super.doXContentBody(builder, params); + if (hasNormalizationFactor) { + boolean hasValue = Double.isNaN(normalizedValue) == false; + builder.field(NORMALIZED.getPreferredName(), hasValue ? normalizedValue : null); + if (hasValue && normalizedAsString != null) { + builder.field(NORMALIZED_AS_STRING.getPreferredName(), normalizedAsString); + } + } + return builder; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 53984332f37..e3584f5a85c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -51,7 +51,11 @@ import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; +import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; @@ -85,6 +89,8 @@ public abstract class InternalAggregationTestCase namedXContents.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)); namedXContents.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c)); namedXContents.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c)); + namedXContents.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c)); + namedXContents.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java index 4dae0a8d653..ffd5fd1bf8b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValueTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.pipeline; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import java.util.Collections; import java.util.List; @@ -30,18 +31,18 @@ import java.util.Map; public class InternalSimpleValueTests extends InternalAggregationTestCase{ @Override - protected InternalSimpleValue createTestInstance(String name, - List pipelineAggregators, Map metaData) { + protected InternalSimpleValue createTestInstance(String name, List pipelineAggregators, + Map metaData) { DocValueFormat formatter = randomNumericDocValueFormat(); - double value = randomDoubleBetween(0, 100000, true); + double value = frequently() ? randomDoubleBetween(0, 100000, true) + : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN }); return new InternalSimpleValue(name, value, formatter, pipelineAggregators, metaData); } @Override public void testReduceRandom() { expectThrows(UnsupportedOperationException.class, - () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, - null)); + () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, null)); } @Override @@ -54,4 +55,16 @@ public class InternalSimpleValueTests extends InternalAggregationTestCase pipelineAggregators, Map metaData) { DocValueFormat formatter = randomNumericDocValueFormat(); - double value = randomDoubleBetween(0, 100000, true); - double normalizationFactor = randomDoubleBetween(0, 100000, true); + double value = frequently() ? randomDoubleBetween(-100000, 100000, true) + : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN }); + double normalizationFactor = frequently() ? randomDoubleBetween(0, 100000, true) : 0; return new InternalDerivative(name, value, normalizationFactor, formatter, pipelineAggregators, metaData); } @Override public void testReduceRandom() { expectThrows(UnsupportedOperationException.class, - () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, - null)); + () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, null)); } @Override @@ -56,4 +57,16 @@ public class InternalDerivativeTests extends InternalAggregationTestCase Date: Thu, 20 Apr 2017 10:16:51 +0200 Subject: [PATCH 18/62] Add parsing methods for Percentiles aggregations (#24183) --- .../percentiles/ParsedPercentileRanks.java | 2 +- ...ercentiles.java => ParsedPercentiles.java} | 8 +- .../hdr/ParsedHDRPercentileRanks.java | 4 +- .../percentiles/hdr/ParsedHDRPercentiles.java | 57 +++++++++++++ .../tdigest/ParsedTDigestPercentileRanks.java | 4 +- .../tdigest/ParsedTDigestPercentiles.java | 57 +++++++++++++ .../InternalAggregationTestCase.java | 44 +++++++--- .../AbstractPercentilesTestCase.java | 84 +++++++++++++++++++ .../InternalPercentilesRanksTestCase.java | 82 ++---------------- .../InternalPercentilesTestCase.java | 44 +++------- .../hdr/InternalHDRPercentilesRanksTests.java | 14 ++-- .../hdr/InternalHDRPercentilesTests.java | 6 ++ .../InternalTDigestPercentilesRanksTests.java | 18 ++-- .../InternalTDigestPercentilesTests.java | 6 ++ 14 files changed, 289 insertions(+), 141 deletions(-) rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/{AbstractParsedPercentiles.java => ParsedPercentiles.java} (96%) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java index de3e81e966c..2c80d0328dd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentileRanks.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; -public abstract class ParsedPercentileRanks extends AbstractParsedPercentiles implements PercentileRanks { +public abstract class ParsedPercentileRanks extends ParsedPercentiles implements PercentileRanks { @Override public double percent(double value) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java similarity index 96% rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java index e3152bac3a3..eee058fc2f8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractParsedPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java @@ -31,7 +31,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -public abstract class AbstractParsedPercentiles extends ParsedAggregation implements Iterable { +public abstract class ParsedPercentiles extends ParsedAggregation implements Iterable { private final Map percentiles = new LinkedHashMap<>(); private final Map percentilesAsString = new HashMap<>(); @@ -46,14 +46,14 @@ public abstract class AbstractParsedPercentiles extends ParsedAggregation implem percentilesAsString.put(key, valueAsString); } - Double getPercentile(double percent) { + protected Double getPercentile(double percent) { if (percentiles.isEmpty()) { return Double.NaN; } return percentiles.get(percent); } - String getPercentileAsString(double percent) { + protected String getPercentileAsString(double percent) { String valueAsString = percentilesAsString.get(percent); if (valueAsString != null) { return valueAsString; @@ -119,7 +119,7 @@ public abstract class AbstractParsedPercentiles extends ParsedAggregation implem return builder; } - protected static void declarePercentilesFields(ObjectParser objectParser) { + protected static void declarePercentilesFields(ObjectParser objectParser) { ParsedAggregation.declareAggregationFields(objectParser); objectParser.declareField((parser, aggregation, context) -> { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java index 79749df1176..5d00d8dc8e3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentileRanks.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; @@ -55,7 +55,7 @@ public class ParsedHDRPercentileRanks extends ParsedPercentileRanks { private static ObjectParser PARSER = new ObjectParser<>(ParsedHDRPercentileRanks.class.getSimpleName(), true, ParsedHDRPercentileRanks::new); static { - AbstractParsedPercentiles.declarePercentilesFields(PARSER); + ParsedPercentiles.declarePercentilesFields(PARSER); } public static ParsedHDRPercentileRanks fromXContent(XContentParser parser, String name) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java new file mode 100644 index 00000000000..0b6da9f00e1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/ParsedHDRPercentiles.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; + +import java.io.IOException; + +public class ParsedHDRPercentiles extends ParsedPercentiles implements Percentiles { + + @Override + protected String getType() { + return InternalHDRPercentiles.NAME; + } + + @Override + public double percentile(double percent) { + return getPercentile(percent); + } + + @Override + public String percentileAsString(double percent) { + return getPercentileAsString(percent); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedHDRPercentiles.class.getSimpleName(), true, ParsedHDRPercentiles::new); + static { + ParsedPercentiles.declarePercentilesFields(PARSER); + } + + public static ParsedHDRPercentiles fromXContent(XContentParser parser, String name) throws IOException { + ParsedHDRPercentiles aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java index 3d51e98d622..34fd92ad4fb 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentileRanks.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.aggregations.metrics.percentiles.AbstractParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; @@ -55,7 +55,7 @@ public class ParsedTDigestPercentileRanks extends ParsedPercentileRanks { private static ObjectParser PARSER = new ObjectParser<>(ParsedTDigestPercentileRanks.class.getSimpleName(), true, ParsedTDigestPercentileRanks::new); static { - AbstractParsedPercentiles.declarePercentilesFields(PARSER); + ParsedPercentiles.declarePercentilesFields(PARSER); } public static ParsedTDigestPercentileRanks fromXContent(XContentParser parser, String name) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java new file mode 100644 index 00000000000..bdd2d6b810a --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/ParsedTDigestPercentiles.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; + +import java.io.IOException; + +public class ParsedTDigestPercentiles extends ParsedPercentiles implements Percentiles { + + @Override + protected String getType() { + return InternalTDigestPercentiles.NAME; + } + + @Override + public double percentile(double percent) { + return getPercentile(percent); + } + + @Override + public String percentileAsString(double percent) { + return getPercentileAsString(percent); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedTDigestPercentiles.class.getSimpleName(), true, ParsedTDigestPercentiles::new); + static { + ParsedPercentiles.declarePercentilesFields(PARSER); + } + + public static ParsedTDigestPercentiles fromXContent(XContentParser parser, String name) throws IOException { + ParsedTDigestPercentiles aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index e3584f5a85c..c2b547b2d93 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -33,7 +33,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; @@ -44,9 +43,13 @@ import org.elasticsearch.search.aggregations.metrics.max.ParsedMax; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.min.ParsedMin; import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; @@ -82,7 +85,9 @@ public abstract class InternalAggregationTestCase static List getNamedXContents() { Map> namedXContents = new HashMap<>(); namedXContents.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c)); + namedXContents.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c)); namedXContents.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c)); + namedXContents.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c)); namedXContents.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c)); namedXContents.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)); namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); @@ -189,7 +194,6 @@ public abstract class InternalAggregationTestCase } public final void testFromXContent() throws IOException { - final NamedXContentRegistry xContentRegistry = xContentRegistry(); final T aggregation = createTestInstance(); //norelease Remove this assumption when all aggregations can be parsed back. @@ -201,8 +205,33 @@ public abstract class InternalAggregationTestCase final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable); + final Aggregation parsedAggregation = parse(aggregation, xContentType, humanReadable, randomBoolean()); + + final BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); + assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); + assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation); + } + + //norelease TODO make abstract + protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + } + + @SuppressWarnings("unchecked") + protected

P parse(final InternalAggregation aggregation, + final XContentType xContentType, + final boolean humanReadable, + final boolean shuffled) throws IOException { + + final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); + final BytesReference originalBytes; + if (shuffled) { + originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable); + } else { + originalBytes = toXContent(aggregation, xContentType, params, humanReadable); + } + Aggregation parsedAggregation; - try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, originalBytes)) { + try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) { assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); @@ -222,15 +251,8 @@ public abstract class InternalAggregationTestCase assertTrue(parsedAggregation instanceof ParsedAggregation); assertEquals(aggregation.getType(), ((ParsedAggregation) parsedAggregation).getType()); - - final BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); - assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); - assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation); } - } - - //norelease TODO make abstract - protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + return (P) parsedAggregation; } /** diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java new file mode 100644 index 00000000000..0859a511177 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.percentiles; + +import com.carrotsearch.randomizedtesting.annotations.Repeat; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.junit.Before; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public abstract class AbstractPercentilesTestCase> + extends InternalAggregationTestCase { + + private double[] percents; + private boolean keyed; + private DocValueFormat docValueFormat; + + @Before + public void init() { + percents = randomPercents(); + keyed = randomBoolean(); + docValueFormat = randomNumericDocValueFormat(); + } + + @Override + protected T createTestInstance(String name, List pipelineAggregators, Map metaData) { + int numValues = randomInt(100); + double[] values = new double[numValues]; + for (int i = 0; i < numValues; ++i) { + values[i] = randomDouble(); + } + return createTestInstance(name, pipelineAggregators, metaData, keyed, docValueFormat, percents, values); + } + + protected abstract T createTestInstance(String name, List pipelineAggregators, Map metaData, + boolean keyed, DocValueFormat format, double[] percents, double[] values); + + protected abstract Class implementationClass(); + + @Repeat(iterations = 1000) + public void testPercentilesIterators() throws IOException { + final T aggregation = createTestInstance(); + final Iterable parsedAggregation = parse(aggregation, randomFrom(XContentType.values()), randomBoolean(), false); + + Iterator it = aggregation.iterator(); + Iterator parsedIt = parsedAggregation.iterator(); + while (it.hasNext()) { + assertEquals(it.next(), parsedIt.next()); + } + } + + private static double[] randomPercents() { + List randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); + double[] percents = new double[randomCdfValues.size()]; + for (int i = 0; i < randomCdfValues.size(); i++) { + percents[i] = randomCdfValues.get(i); + } + return percents; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java index be7d55e5447..f45b7cce51e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesRanksTestCase.java @@ -19,92 +19,24 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.ParsedAggregation; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.singletonMap; -import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; - -public abstract class InternalPercentilesRanksTestCase extends InternalAggregationTestCase { - - @Override - protected final T createTestInstance(String name, List pipelineAggregators, Map metaData) { - final boolean keyed = randomBoolean(); - final DocValueFormat format = randomFrom(DocValueFormat.RAW, new DocValueFormat.Decimal("###.##")); - List randomCdfValues = randomSubsetOf(randomIntBetween(1, 5), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); - double[] cdfValues = new double[randomCdfValues.size()]; - for (int i = 0; i < randomCdfValues.size(); i++) { - cdfValues[i] = randomCdfValues.get(i); - } - return createTestInstance(name, pipelineAggregators, metaData, cdfValues, keyed, format); - } - - protected abstract T createTestInstance(String name, List aggregators, Map metadata, - double[] cdfValues, boolean keyed, DocValueFormat format); +public abstract class InternalPercentilesRanksTestCase + extends AbstractPercentilesTestCase { @Override protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { - assertTrue(aggregation instanceof PercentileRanks); - PercentileRanks percentileRanks = (PercentileRanks) aggregation; - assertTrue(parsedAggregation instanceof PercentileRanks); PercentileRanks parsedPercentileRanks = (PercentileRanks) parsedAggregation; - for (Percentile percentile : percentileRanks) { + for (Percentile percentile : aggregation) { Double value = percentile.getValue(); - assertEquals(percentileRanks.percent(value), parsedPercentileRanks.percent(value), 0); - assertEquals(percentileRanks.percentAsString(value), parsedPercentileRanks.percentAsString(value)); + assertEquals(aggregation.percent(value), parsedPercentileRanks.percent(value), 0); + assertEquals(aggregation.percentAsString(value), parsedPercentileRanks.percentAsString(value)); } - Class parsedClass = parsedParsedPercentileRanksClass(); - assertNotNull(parsedClass); - assertTrue(parsedClass.isInstance(parsedAggregation)); + Class parsedClass = implementationClass(); + assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation)); } - - public void testPercentilesRanksIterators() throws IOException { - final T aggregation = createTestInstance(); - - final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); - final XContentType xContentType = randomFrom(XContentType.values()); - final BytesReference originalBytes = toXContent(aggregation, xContentType, params, randomBoolean()); - - Aggregation parsedAggregation; - try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry(), originalBytes)) { - assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); - assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); - - String currentName = parser.currentName(); - int i = currentName.indexOf(InternalAggregation.TYPED_KEYS_DELIMITER); - String aggType = currentName.substring(0, i); - String aggName = currentName.substring(i + 1); - - parsedAggregation = parser.namedObject(Aggregation.class, aggType, aggName); - - assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken()); - assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); - assertNull(parser.nextToken()); - } - - final Iterator it = ((PercentileRanks) aggregation).iterator(); - final Iterator parsedIt = ((PercentileRanks) parsedAggregation).iterator(); - while (it.hasNext()) { - assertEquals(it.next(), parsedIt.next()); - } - } - - protected abstract Class parsedParsedPercentileRanksClass(); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java index 0cfa07538e4..404c033f11a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentilesTestCase.java @@ -19,43 +19,23 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.junit.Before; +import org.elasticsearch.search.aggregations.ParsedAggregation; -import java.util.List; -import java.util.Map; - -public abstract class InternalPercentilesTestCase extends InternalAggregationTestCase { - - private double[] percents; - - @Before - public void init() { - percents = randomPercents(); - } +public abstract class InternalPercentilesTestCase extends AbstractPercentilesTestCase { @Override - protected T createTestInstance(String name, List pipelineAggregators, Map metaData) { - int numValues = randomInt(100); - double[] values = new double[numValues]; - for (int i = 0; i < numValues; ++i) { - values[i] = randomDouble(); - } - return createTestInstance(name, pipelineAggregators, metaData, randomBoolean(), DocValueFormat.RAW, percents, values); - } + protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + assertTrue(parsedAggregation instanceof Percentiles); + Percentiles parsedPercentiles = (Percentiles) parsedAggregation; - protected abstract T createTestInstance(String name, List pipelineAggregators, Map metaData, - boolean keyed, DocValueFormat format, double[] percents, double[] values); - - private static double[] randomPercents() { - List randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); - double[] percents = new double[randomCdfValues.size()]; - for (int i = 0; i < randomCdfValues.size(); i++) { - percents[i] = randomCdfValues.get(i); + for (Percentile percentile : aggregation) { + Double percent = percentile.getPercent(); + assertEquals(aggregation.percentile(percent), parsedPercentiles.percentile(percent), 0); + assertEquals(aggregation.percentileAsString(percent), parsedPercentiles.percentileAsString(percent)); } - return percents; + + Class parsedClass = implementationClass(); + assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation)); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java index a3fe4d62d14..d9379edefef 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesRanksTests.java @@ -23,9 +23,10 @@ import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase; -import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -33,9 +34,12 @@ public class InternalHDRPercentilesRanksTests extends InternalPercentilesRanksTe @Override protected InternalHDRPercentileRanks createTestInstance(String name, List aggregators, Map metadata, - double[] cdfValues, boolean keyed, DocValueFormat format) { - DoubleHistogram state = new DoubleHistogram(3); - return new InternalHDRPercentileRanks(name, cdfValues, state, keyed, format, aggregators, metadata); + boolean keyed, DocValueFormat format, double[] percents, double[] values) { + + final DoubleHistogram state = new DoubleHistogram(3); + Arrays.stream(values).forEach(state::recordValue); + + return new InternalHDRPercentileRanks(name, percents, state, keyed, format, aggregators, metadata); } @Override @@ -54,7 +58,7 @@ public class InternalHDRPercentilesRanksTests extends InternalPercentilesRanksTe } @Override - protected Class parsedParsedPercentileRanksClass() { + protected Class implementationClass() { return ParsedHDRPercentileRanks.class; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java index bff026d5cf4..5a24e0dfbf2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java @@ -23,6 +23,7 @@ import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.Arrays; @@ -57,4 +58,9 @@ public class InternalHDRPercentilesTests extends InternalPercentilesTestCase instanceReader() { return InternalHDRPercentiles::new; } + + @Override + protected Class implementationClass() { + return ParsedHDRPercentiles.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java index 30d416763c1..f8698fda2cb 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesRanksTests.java @@ -22,9 +22,10 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesRanksTestCase; -import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -33,13 +34,12 @@ public class InternalTDigestPercentilesRanksTests extends InternalPercentilesRan @Override protected InternalTDigestPercentileRanks createTestInstance(String name, List aggregators, Map metadata, - double[] cdfValues, boolean keyed, DocValueFormat format) { - TDigestState state = new TDigestState(100); - int numValues = randomInt(100); - for (int i = 0; i < numValues; ++i) { - state.add(randomDouble()); - } - return new InternalTDigestPercentileRanks(name, cdfValues, state, keyed, format, aggregators, metadata); + boolean keyed, DocValueFormat format, double[] percents, double[] values) { + final TDigestState state = new TDigestState(100); + Arrays.stream(values).forEach(state::add); + + assertEquals(state.centroidCount(), values.length); + return new InternalTDigestPercentileRanks(name, percents, state, keyed, format, aggregators, metadata); } @Override @@ -71,7 +71,7 @@ public class InternalTDigestPercentilesRanksTests extends InternalPercentilesRan } @Override - protected Class parsedParsedPercentileRanksClass() { + protected Class implementationClass() { return ParsedTDigestPercentileRanks.class; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java index f2db4a48530..867469592e9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.Arrays; @@ -64,4 +65,9 @@ public class InternalTDigestPercentilesTests extends InternalPercentilesTestCase protected Writeable.Reader instanceReader() { return InternalTDigestPercentiles::new; } + + @Override + protected Class implementationClass() { + return ParsedTDigestPercentiles.class; + } } From 2b14db9b70a16349d8541e11bbc14497a087dea1 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 20 Apr 2017 10:44:12 +0200 Subject: [PATCH 19/62] Remove @Repeat(iterations = 1000) in tests --- .../metrics/percentiles/AbstractPercentilesTestCase.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java index 0859a511177..b8648f5199c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -61,7 +60,6 @@ public abstract class AbstractPercentilesTestCase implementationClass(); - @Repeat(iterations = 1000) public void testPercentilesIterators() throws IOException { final T aggregation = createTestInstance(); final Iterable parsedAggregation = parse(aggregation, randomFrom(XContentType.values()), randomBoolean(), false); From 6e22a1e9bad79a47d4877382afe05249a46d4b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 20 Apr 2017 11:49:09 +0200 Subject: [PATCH 20/62] Add parsing for InternalBucketMetricValue (#24182) --- .../InternalBucketMetricValue.java | 19 ++++- .../ParsedBucketMetricValue.java | 73 ++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../InternalBucketMetricValueTests.java | 75 +++++++++++++++++++ 4 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java index 9c9da2f26bd..95b3782bd54 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -28,11 +29,14 @@ import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggre import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; public class InternalBucketMetricValue extends InternalNumericMetricsAggregation.SingleValue implements BucketMetricValue { public static final String NAME = "bucket_metric_value"; + static final ParseField KEYS_FIELD = new ParseField("keys"); private double value; private String[] keys; @@ -88,7 +92,7 @@ public class InternalBucketMetricValue extends InternalNumericMetricsAggregation return this; } else if (path.size() == 1 && "value".equals(path.get(0))) { return value(); - } else if (path.size() == 1 && "keys".equals(path.get(0))) { + } else if (path.size() == 1 && KEYS_FIELD.getPreferredName().equals(path.get(0))) { return keys(); } else { throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path); @@ -102,7 +106,7 @@ public class InternalBucketMetricValue extends InternalNumericMetricsAggregation if (hasValue && format != DocValueFormat.RAW) { builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value)); } - builder.startArray("keys"); + builder.startArray(KEYS_FIELD.getPreferredName()); for (String key : keys) { builder.value(key); } @@ -110,4 +114,15 @@ public class InternalBucketMetricValue extends InternalNumericMetricsAggregation return builder; } + @Override + protected int doHashCode() { + return Objects.hash(value, Arrays.hashCode(keys)); + } + + @Override + protected boolean doEquals(Object obj) { + InternalBucketMetricValue other = (InternalBucketMetricValue) obj; + return Objects.equals(value, other.value) + && Arrays.equals(keys, other.keys); + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java new file mode 100644 index 00000000000..9008c56c5ca --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ParsedBucketMetricValue.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class ParsedBucketMetricValue extends ParsedSingleValueNumericMetricsAggregation implements BucketMetricValue { + + private List keys = Collections.emptyList(); + + @Override + public String[] keys() { + return this.keys.toArray(new String[keys.size()]); + } + + @Override + protected String getType() { + return InternalBucketMetricValue.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + boolean hasValue = !Double.isInfinite(value); + builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); + if (hasValue && valueAsString != null) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString); + } + builder.startArray(InternalBucketMetricValue.KEYS_FIELD.getPreferredName()); + for (String key : keys) { + builder.value(key); + } + builder.endArray(); + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>( + ParsedBucketMetricValue.class.getSimpleName(), true, ParsedBucketMetricValue::new); + + static { + declareSingleValueFields(PARSER, Double.NEGATIVE_INFINITY); + PARSER.declareStringArray((agg, value) -> agg.keys = value, InternalBucketMetricValue.KEYS_FIELD); + } + + public static ParsedBucketMetricValue fromXContent(XContentParser parser, final String name) { + ParsedBucketMetricValue bucketMetricValue = PARSER.apply(parser, null); + bucketMetricValue.setName(name); + return bucketMetricValue; + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index c2b547b2d93..09b05dab84b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -57,6 +57,8 @@ import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggreg import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -96,6 +98,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c)); namedXContents.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c)); namedXContents.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); + namedXContents.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java new file mode 100644 index 00000000000..8de1700141b --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValueTests.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class InternalBucketMetricValueTests extends InternalAggregationTestCase { + + @Override + protected InternalBucketMetricValue createTestInstance(String name, List pipelineAggregators, + Map metaData) { + double value = frequently() ? randomDoubleBetween(-10000, 100000, true) + : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN }); + String[] keys = new String[randomIntBetween(0, 5)]; + for (int i = 0; i < keys.length; i++) { + keys[i] = randomAlphaOfLength(10); + } + return new InternalBucketMetricValue(name, keys, value, randomNumericDocValueFormat(), pipelineAggregators, metaData); + } + + @Override + public void testReduceRandom() { + expectThrows(UnsupportedOperationException.class, + () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, + null)); + } + + @Override + protected void assertReduced(InternalBucketMetricValue reduced, List inputs) { + // no test since reduce operation is unsupported + } + + @Override + protected Reader instanceReader() { + return InternalBucketMetricValue::new; + } + + @Override + protected void assertFromXContent(InternalBucketMetricValue bucketMetricValue, ParsedAggregation parsedAggregation) { + BucketMetricValue parsed = ((BucketMetricValue) parsedAggregation); + assertArrayEquals(bucketMetricValue.keys(), parsed.keys()); + if (Double.isInfinite(bucketMetricValue.value()) == false) { + assertEquals(bucketMetricValue.value(), parsed.value(), 0); + assertEquals(bucketMetricValue.getValueAsString(), parsed.getValueAsString()); + } else { + // we write Double.NEGATIVE_INFINITY and Double.POSITIVE_INFINITY to xContent as 'null', so we + // cannot differentiate between them. Also we cannot recreate the exact String representation + assertEquals(parsed.value(), Double.NEGATIVE_INFINITY, 0); + } + } +} From d0df1ed193a86f1ddb6662b976e8cb89ae6fe9c8 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 20 Apr 2017 13:22:50 +0200 Subject: [PATCH 21/62] [Test] Always check the XContent equivalent when parsing aggregations (#24208) In InternalAggregationTestCase, we can check that the internal aggregation and the parsed aggregation always produce the same XContent even if the original internal aggregation has been shuffled or not. --- .../InternalAggregationTestCase.java | 23 ++++++++----------- .../AbstractPercentilesTestCase.java | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 09b05dab84b..fd6b9ecdc44 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -203,15 +203,7 @@ public abstract class InternalAggregationTestCase assumeTrue("This test does not support the aggregation type yet", getNamedXContents().stream().filter(entry -> entry.name.match(aggregation.getType())).count() > 0); - final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); - final boolean humanReadable = randomBoolean(); - final XContentType xContentType = randomFrom(XContentType.values()); - final BytesReference originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable); - - final Aggregation parsedAggregation = parse(aggregation, xContentType, humanReadable, randomBoolean()); - - final BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); - assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); + final Aggregation parsedAggregation = parseAndAssert(aggregation, randomBoolean()); assertFromXContent(aggregation, (ParsedAggregation) parsedAggregation); } @@ -220,12 +212,13 @@ public abstract class InternalAggregationTestCase } @SuppressWarnings("unchecked") - protected

P parse(final InternalAggregation aggregation, - final XContentType xContentType, - final boolean humanReadable, - final boolean shuffled) throws IOException { + protected

P parseAndAssert(final InternalAggregation aggregation, + final boolean shuffled) throws IOException { final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); + final XContentType xContentType = randomFrom(XContentType.values()); + final boolean humanReadable = randomBoolean(); + final BytesReference originalBytes; if (shuffled) { originalBytes = toShuffledXContent(aggregation, xContentType, params, humanReadable); @@ -255,6 +248,10 @@ public abstract class InternalAggregationTestCase assertTrue(parsedAggregation instanceof ParsedAggregation); assertEquals(aggregation.getType(), ((ParsedAggregation) parsedAggregation).getType()); } + + BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); + assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); + return (P) parsedAggregation; } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java index b8648f5199c..510620d12aa 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java @@ -62,7 +62,7 @@ public abstract class AbstractPercentilesTestCase parsedAggregation = parse(aggregation, randomFrom(XContentType.values()), randomBoolean(), false); + final Iterable parsedAggregation = parseAndAssert(aggregation, false); Iterator it = aggregation.iterator(); Iterator parsedIt = parsedAggregation.iterator(); From c8fc30a999b748417988fecf806bf024baf7a5fc Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 20 Apr 2017 13:36:16 +0200 Subject: [PATCH 22/62] [Test] Expose AbstractPercentilesTestCase.randomPercents() --- .../metrics/percentiles/AbstractPercentilesTestCase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java index 510620d12aa..0223db03599 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; @@ -71,7 +70,7 @@ public abstract class AbstractPercentilesTestCase randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); double[] percents = new double[randomCdfValues.size()]; for (int i = 0; i < randomCdfValues.size(); i++) { From 768420db554479d7853a3d63548b2cb8cd507f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 25 Apr 2017 13:42:15 +0200 Subject: [PATCH 23/62] Add parsing for InternalStats (#24239) --- .../aggregations/ParsedAggregation.java | 15 ++ ...dSingleValueNumericMetricsAggregation.java | 53 +++---- .../metrics/stats/InternalStats.java | 29 ++-- .../metrics/stats/ParsedStats.java | 147 ++++++++++++++++++ .../stats/extended/InternalExtendedStats.java | 2 +- .../pipeline/derivative/ParsedDerivative.java | 2 +- .../InternalAggregationTestCase.java | 3 + .../metrics/InternalStatsTests.java | 27 +++- 8 files changed, 230 insertions(+), 48 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index 452fe9dcb08..6942b6aec5d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -22,6 +22,8 @@ package org.elasticsearch.search.aggregations; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import java.io.IOException; import java.util.Collections; @@ -77,4 +79,17 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { } protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException; + + /** + * Parse a token of type XContentParser.Token.VALUE_NUMBER or XContentParser.Token.STRING to a double. + * In other cases the default value is returned instead. + */ + protected static double parseDouble(XContentParser parser, double defaultNullValue) throws IOException { + Token currentToken = parser.currentToken(); + if (currentToken == XContentParser.Token.VALUE_NUMBER || currentToken == XContentParser.Token.VALUE_STRING) { + return parser.doubleValue(); + } else { + return defaultNullValue; + } + } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java index 5eb0a7223a1..3105a785e19 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedSingleValueNumericMetricsAggregation.java @@ -20,54 +20,41 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.ParsedAggregation; -import java.io.IOException; - public abstract class ParsedSingleValueNumericMetricsAggregation extends ParsedAggregation implements NumericMetricsAggregation.SingleValue { - protected double value; - protected String valueAsString; + protected double value; + protected String valueAsString; - @Override - public String getValueAsString() { - if (valueAsString != null) { - return valueAsString; - } else { - return Double.toString(value); - } + @Override + public String getValueAsString() { + if (valueAsString != null) { + return valueAsString; + } else { + return Double.toString(value); } + } - @Override - public double value() { - return value; - } + @Override + public double value() { + return value; + } - protected void setValue(double value) { - this.value = value; - } + protected void setValue(double value) { + this.value = value; + } - protected void setValueAsString(String valueAsString) { - this.valueAsString = valueAsString; - } - - protected static double parseValue(XContentParser parser, double defaultNullValue) throws IOException { - Token currentToken = parser.currentToken(); - if (currentToken == XContentParser.Token.VALUE_NUMBER || currentToken == XContentParser.Token.VALUE_STRING) { - return parser.doubleValue(); - } else { - return defaultNullValue; - } - } + protected void setValueAsString(String valueAsString) { + this.valueAsString = valueAsString; + } protected static void declareSingleValueFields(ObjectParser objectParser, double defaultNullValue) { declareAggregationFields(objectParser); objectParser.declareField(ParsedSingleValueNumericMetricsAggregation::setValue, - (parser, context) -> parseValue(parser, defaultNullValue), CommonFields.VALUE, ValueType.DOUBLE_OR_NULL); + (parser, context) -> parseDouble(parser, defaultNullValue), CommonFields.VALUE, ValueType.DOUBLE_OR_NULL); objectParser.declareString(ParsedSingleValueNumericMetricsAggregation::setValueAsString, CommonFields.VALUE_AS_STRING); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java index b0b2ea73d3c..a29754ea559 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java @@ -177,21 +177,28 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.COUNT, count); - builder.field(Fields.MIN, count != 0 ? min : null); - builder.field(Fields.MAX, count != 0 ? max : null); - builder.field(Fields.AVG, count != 0 ? getAvg() : null); - builder.field(Fields.SUM, count != 0 ? sum : null); - if (count != 0 && format != DocValueFormat.RAW) { - builder.field(Fields.MIN_AS_STRING, format.format(min)); - builder.field(Fields.MAX_AS_STRING, format.format(max)); - builder.field(Fields.AVG_AS_STRING, format.format(getAvg())); - builder.field(Fields.SUM_AS_STRING, format.format(sum)); + if (count != 0) { + builder.field(Fields.MIN, min); + builder.field(Fields.MAX, max); + builder.field(Fields.AVG, getAvg()); + builder.field(Fields.SUM, sum); + if (format != DocValueFormat.RAW) { + builder.field(Fields.MIN_AS_STRING, format.format(min)); + builder.field(Fields.MAX_AS_STRING, format.format(max)); + builder.field(Fields.AVG_AS_STRING, format.format(getAvg())); + builder.field(Fields.SUM_AS_STRING, format.format(sum)); + } + } else { + builder.nullField(Fields.MIN); + builder.nullField(Fields.MAX); + builder.nullField(Fields.AVG); + builder.nullField(Fields.SUM); } - otherStatsToXCotent(builder, params); + otherStatsToXContent(builder, params); return builder; } - protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params params) throws IOException { + protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException { return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java new file mode 100644 index 00000000000..28ca3418a99 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.stats; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.metrics.stats.InternalStats.Fields; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class ParsedStats extends ParsedAggregation implements Stats { + + protected long count; + protected double min; + protected double max; + protected double sum; + protected double avg; + + protected final Map valueAsString = new HashMap<>(); + + @Override + public long getCount() { + return count; + } + + @Override + public double getMin() { + return min; + } + + @Override + public double getMax() { + return max; + } + + @Override + public double getAvg() { + return avg; + } + + @Override + public double getSum() { + return sum; + } + + @Override + public String getMinAsString() { + return valueAsString.getOrDefault(Fields.MIN_AS_STRING, Double.toString(min)); + } + + @Override + public String getMaxAsString() { + return valueAsString.getOrDefault(Fields.MAX_AS_STRING, Double.toString(max)); + } + + @Override + public String getAvgAsString() { + return valueAsString.getOrDefault(Fields.AVG_AS_STRING, Double.toString(avg)); + } + + @Override + public String getSumAsString() { + return valueAsString.getOrDefault(Fields.SUM_AS_STRING, Double.toString(sum)); + } + + @Override + protected String getType() { + return StatsAggregationBuilder.NAME; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(Fields.COUNT, count); + if (count != 0) { + builder.field(Fields.MIN, min); + builder.field(Fields.MAX, max); + builder.field(Fields.AVG, avg); + builder.field(Fields.SUM, sum); + if (valueAsString.get(Fields.MIN_AS_STRING) != null) { + builder.field(Fields.MIN_AS_STRING, getMinAsString()); + builder.field(Fields.MAX_AS_STRING, getMaxAsString()); + builder.field(Fields.AVG_AS_STRING, getAvgAsString()); + builder.field(Fields.SUM_AS_STRING, getSumAsString()); + } + } else { + builder.nullField(Fields.MIN); + builder.nullField(Fields.MAX); + builder.nullField(Fields.AVG); + builder.nullField(Fields.SUM); + } + otherStatsToXContent(builder, params); + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedStats.class.getSimpleName(), true, + ParsedStats::new); + + static { + declareAggregationFields(PARSER); + PARSER.declareLong((agg, value) -> agg.count = value, new ParseField(Fields.COUNT)); + PARSER.declareField((agg, value) -> agg.min = value, (parser, context) -> parseDouble(parser, Double.POSITIVE_INFINITY), + new ParseField(Fields.MIN), ValueType.DOUBLE_OR_NULL); + PARSER.declareField((agg, value) -> agg.max = value, (parser, context) -> parseDouble(parser, Double.NEGATIVE_INFINITY), + new ParseField(Fields.MAX), ValueType.DOUBLE_OR_NULL); + PARSER.declareField((agg, value) -> agg.avg = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.AVG), + ValueType.DOUBLE_OR_NULL); + PARSER.declareField((agg, value) -> agg.sum = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.SUM), + ValueType.DOUBLE_OR_NULL); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.MIN_AS_STRING, value), new ParseField(Fields.MIN_AS_STRING)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.MAX_AS_STRING, value), new ParseField(Fields.MAX_AS_STRING)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.AVG_AS_STRING, value), new ParseField(Fields.AVG_AS_STRING)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_AS_STRING, value), new ParseField(Fields.SUM_AS_STRING)); + } + + public static ParsedStats fromXContent(XContentParser parser, final String name) { + ParsedStats parsedStats = PARSER.apply(parser, null); + parsedStats.setName(name); + return parsedStats; + } + + protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java index 370399bfbb8..b769850b817 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java @@ -169,7 +169,7 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat } @Override - protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params params) throws IOException { + protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.SUM_OF_SQRS, count != 0 ? sumOfSqrs : null); builder.field(Fields.VARIANCE, count != 0 ? getVariance() : null); builder.field(Fields.STD_DEVIATION, count != 0 ? getStdDeviation() : null); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java index ed463239b24..5469b8b65cf 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/ParsedDerivative.java @@ -54,7 +54,7 @@ public class ParsedDerivative extends ParsedSimpleValue implements Derivative { PARSER.declareField((agg, normalized) -> { agg.normalizedValue = normalized; agg.hasNormalizationFactor = true; - }, (parser, context) -> parseValue(parser, Double.NaN), NORMALIZED, ValueType.DOUBLE_OR_NULL); + }, (parser, context) -> parseDouble(parser, Double.NaN), NORMALIZED, ValueType.DOUBLE_OR_NULL); PARSER.declareString((agg, normalAsString) -> agg.normalizedAsString = normalAsString, NORMALIZED_AS_STRING); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index fd6b9ecdc44..63211f3db22 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -50,6 +50,8 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.Interna import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; +import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; @@ -99,6 +101,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c)); namedXContents.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); namedXContents.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c)); + namedXContents.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java index c99d208581c..c0c54d3a8a0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java @@ -21,7 +21,9 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.metrics.stats.InternalStats; +import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.Collections; @@ -29,9 +31,9 @@ import java.util.List; import java.util.Map; public class InternalStatsTests extends InternalAggregationTestCase { + @Override - protected InternalStats createTestInstance(String name, List pipelineAggregators, - Map metaData) { + protected InternalStats createTestInstance(String name, List pipelineAggregators, Map metaData) { long count = frequently() ? randomIntBetween(1, Integer.MAX_VALUE) : 0; double min = randomDoubleBetween(-1000000, 1000000, true); double max = randomDoubleBetween(-1000000, 1000000, true); @@ -62,8 +64,29 @@ public class InternalStatsTests extends InternalAggregationTestCase 0 ? aggregation.getMin() : Double.POSITIVE_INFINITY , parsed.getMin(), 0); + assertEquals(count > 0 ? aggregation.getMax() : Double.NEGATIVE_INFINITY, parsed.getMax(), 0); + assertEquals(count > 0 ? aggregation.getSum() : 0, parsed.getSum(), 0); + assertEquals(count > 0 ? aggregation.getAvg() : 0, parsed.getAvg(), 0); + // also as_string values are only rendered for count != 0 + if (count > 0) { + assertEquals(aggregation.getMinAsString(), parsed.getMinAsString()); + assertEquals(aggregation.getMaxAsString(), parsed.getMaxAsString()); + assertEquals(aggregation.getSumAsString(), parsed.getSumAsString()); + assertEquals(aggregation.getAvgAsString(), parsed.getAvgAsString()); + } + } + @Override protected Writeable.Reader instanceReader() { return InternalStats::new; } } + From 0e31a21db2951e5d803269d31a541cf50e0b98b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 25 Apr 2017 23:55:24 +0200 Subject: [PATCH 24/62] Add parsing for InternalExtendedStats (#24284) --- .../metrics/stats/ParsedStats.java | 28 ++- .../stats/extended/InternalExtendedStats.java | 49 +++-- .../stats/extended/ParsedExtendedStats.java | 186 ++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../metrics/InternalExtendedStatsTests.java | 48 +++-- .../metrics/InternalStatsTests.java | 7 +- 6 files changed, 279 insertions(+), 42 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java index 28ca3418a99..1711e4ee15d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/ParsedStats.java @@ -119,20 +119,28 @@ public class ParsedStats extends ParsedAggregation implements Stats { ParsedStats::new); static { - declareAggregationFields(PARSER); - PARSER.declareLong((agg, value) -> agg.count = value, new ParseField(Fields.COUNT)); - PARSER.declareField((agg, value) -> agg.min = value, (parser, context) -> parseDouble(parser, Double.POSITIVE_INFINITY), + declareStatsFields(PARSER); + } + + protected static void declareStatsFields(ObjectParser objectParser) { + declareAggregationFields(objectParser); + objectParser.declareLong((agg, value) -> agg.count = value, new ParseField(Fields.COUNT)); + objectParser.declareField((agg, value) -> agg.min = value, (parser, context) -> parseDouble(parser, Double.POSITIVE_INFINITY), new ParseField(Fields.MIN), ValueType.DOUBLE_OR_NULL); - PARSER.declareField((agg, value) -> agg.max = value, (parser, context) -> parseDouble(parser, Double.NEGATIVE_INFINITY), + objectParser.declareField((agg, value) -> agg.max = value, (parser, context) -> parseDouble(parser, Double.NEGATIVE_INFINITY), new ParseField(Fields.MAX), ValueType.DOUBLE_OR_NULL); - PARSER.declareField((agg, value) -> agg.avg = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.AVG), + objectParser.declareField((agg, value) -> agg.avg = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.AVG), ValueType.DOUBLE_OR_NULL); - PARSER.declareField((agg, value) -> agg.sum = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.SUM), + objectParser.declareField((agg, value) -> agg.sum = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.SUM), ValueType.DOUBLE_OR_NULL); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.MIN_AS_STRING, value), new ParseField(Fields.MIN_AS_STRING)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.MAX_AS_STRING, value), new ParseField(Fields.MAX_AS_STRING)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.AVG_AS_STRING, value), new ParseField(Fields.AVG_AS_STRING)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_AS_STRING, value), new ParseField(Fields.SUM_AS_STRING)); + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.MIN_AS_STRING, value), + new ParseField(Fields.MIN_AS_STRING)); + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.MAX_AS_STRING, value), + new ParseField(Fields.MAX_AS_STRING)); + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.AVG_AS_STRING, value), + new ParseField(Fields.AVG_AS_STRING)); + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_AS_STRING, value), + new ParseField(Fields.SUM_AS_STRING)); } public static ParsedStats fromXContent(XContentParser parser, final String name) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java index b769850b817..6e06a88cccd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java @@ -170,24 +170,37 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat @Override protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.SUM_OF_SQRS, count != 0 ? sumOfSqrs : null); - builder.field(Fields.VARIANCE, count != 0 ? getVariance() : null); - builder.field(Fields.STD_DEVIATION, count != 0 ? getStdDeviation() : null); - builder.startObject(Fields.STD_DEVIATION_BOUNDS) - .field(Fields.UPPER, count != 0 ? getStdDeviationBound(Bounds.UPPER) : null) - .field(Fields.LOWER, count != 0 ? getStdDeviationBound(Bounds.LOWER) : null) - .endObject(); - - if (count != 0 && format != DocValueFormat.RAW) { - builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs)); - builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance())); - builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString()); - - builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING) - .field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER)) - .field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER)) - .endObject(); - + if (count != 0) { + builder.field(Fields.SUM_OF_SQRS, sumOfSqrs); + builder.field(Fields.VARIANCE, getVariance()); + builder.field(Fields.STD_DEVIATION, getStdDeviation()); + builder.startObject(Fields.STD_DEVIATION_BOUNDS); + { + builder.field(Fields.UPPER, getStdDeviationBound(Bounds.UPPER)); + builder.field(Fields.LOWER, getStdDeviationBound(Bounds.LOWER)); + } + builder.endObject(); + if (format != DocValueFormat.RAW) { + builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs)); + builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance())); + builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString()); + builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING); + { + builder.field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER)); + builder.field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER)); + } + builder.endObject(); + } + } else { + builder.nullField(Fields.SUM_OF_SQRS); + builder.nullField(Fields.VARIANCE); + builder.nullField(Fields.STD_DEVIATION); + builder.startObject(Fields.STD_DEVIATION_BOUNDS); + { + builder.nullField(Fields.UPPER); + builder.nullField(Fields.LOWER); + } + builder.endObject(); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java new file mode 100644 index 00000000000..8947f4c0aac --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java @@ -0,0 +1,186 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.stats.extended; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; +import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats.Fields; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { + + protected double sumOfSquares; + protected double variance; + protected double stdDeviation; + protected double stdDeviationBoundUpper; + protected double stdDeviationBoundLower; + protected double sum; + protected double avg; + + @Override + protected String getType() { + return ExtendedStatsAggregationBuilder.NAME; + } + + @Override + public double getSumOfSquares() { + return sumOfSquares; + } + + @Override + public double getVariance() { + return variance; + } + + @Override + public double getStdDeviation() { + return stdDeviation; + } + + private void setStdDeviationBounds(Tuple bounds) { + this.stdDeviationBoundLower = bounds.v1(); + this.stdDeviationBoundUpper = bounds.v2(); + } + + @Override + public double getStdDeviationBound(Bounds bound) { + return (bound.equals(Bounds.LOWER)) ? stdDeviationBoundLower : stdDeviationBoundUpper; + } + + @Override + public String getStdDeviationAsString() { + return valueAsString.getOrDefault(Fields.STD_DEVIATION_AS_STRING, Double.toString(stdDeviation)); + } + + private void setStdDeviationBoundsAsString(Tuple boundsAsString) { + this.valueAsString.put(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_lower", boundsAsString.v1()); + this.valueAsString.put(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_upper", boundsAsString.v2()); + } + + @Override + public String getStdDeviationBoundAsString(Bounds bound) { + if (bound.equals(Bounds.LOWER)) { + return valueAsString.getOrDefault(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_lower", Double.toString(stdDeviationBoundLower)); + } else { + return valueAsString.getOrDefault(Fields.STD_DEVIATION_BOUNDS_AS_STRING + "_upper", Double.toString(stdDeviationBoundUpper)); + } + } + + @Override + public String getSumOfSquaresAsString() { + return valueAsString.getOrDefault(Fields.SUM_OF_SQRS_AS_STRING, Double.toString(sumOfSquares)); + } + + @Override + public String getVarianceAsString() { + return valueAsString.getOrDefault(Fields.VARIANCE_AS_STRING, Double.toString(variance)); + } + + @Override + protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params params) throws IOException { + if (count != 0) { + builder.field(Fields.SUM_OF_SQRS, sumOfSquares); + builder.field(Fields.VARIANCE, getVariance()); + builder.field(Fields.STD_DEVIATION, getStdDeviation()); + builder.startObject(Fields.STD_DEVIATION_BOUNDS); + { + builder.field(Fields.UPPER, getStdDeviationBound(Bounds.UPPER)); + builder.field(Fields.LOWER, getStdDeviationBound(Bounds.LOWER)); + } + builder.endObject(); + if (valueAsString.containsKey(Fields.SUM_OF_SQRS_AS_STRING)) { + builder.field(Fields.SUM_OF_SQRS_AS_STRING, getSumOfSquaresAsString()); + builder.field(Fields.VARIANCE_AS_STRING, getVarianceAsString()); + builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString()); + builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING); + { + builder.field(Fields.UPPER, getStdDeviationBoundAsString(Bounds.UPPER)); + builder.field(Fields.LOWER, getStdDeviationBoundAsString(Bounds.LOWER)); + } + builder.endObject(); + } + } else { + builder.nullField(Fields.SUM_OF_SQRS); + builder.nullField(Fields.VARIANCE); + builder.nullField(Fields.STD_DEVIATION); + builder.startObject(Fields.STD_DEVIATION_BOUNDS); + { + builder.nullField(Fields.UPPER); + builder.nullField(Fields.LOWER); + } + builder.endObject(); + } + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(), true, + ParsedExtendedStats::new); + + private static final ConstructingObjectParser, Void> STD_BOUNDS_PARSER = new ConstructingObjectParser<>( + ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS", true, args -> new Tuple<>((Double) args[0], (Double) args[1])); + static { + STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0), + new ParseField(Fields.LOWER), ValueType.DOUBLE_OR_NULL); + STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0), + new ParseField(Fields.UPPER), ValueType.DOUBLE_OR_NULL); + } + + private static final ConstructingObjectParser, Void> STD_BOUNDS_AS_STRING_PARSER = new ConstructingObjectParser<>( + ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true, args -> new Tuple<>((String) args[0], (String) args[1])); + static { + STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.LOWER)); + STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.UPPER)); + } + + static { + declareAggregationFields(PARSER); + declareStatsFields(PARSER); + PARSER.declareField((agg, value) -> agg.sumOfSquares = value, (parser, context) -> parseDouble(parser, 0), + new ParseField(Fields.SUM_OF_SQRS), ValueType.DOUBLE_OR_NULL); + PARSER.declareField((agg, value) -> agg.variance = value, (parser, context) -> parseDouble(parser, 0), + new ParseField(Fields.VARIANCE), ValueType.DOUBLE_OR_NULL); + PARSER.declareField((agg, value) -> agg.stdDeviation = value, (parser, context) -> parseDouble(parser, 0), + new ParseField(Fields.STD_DEVIATION), ValueType.DOUBLE_OR_NULL); + PARSER.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER, new ParseField(Fields.STD_DEVIATION_BOUNDS)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_OF_SQRS_AS_STRING, value), + new ParseField(Fields.SUM_OF_SQRS_AS_STRING)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.VARIANCE_AS_STRING, value), + new ParseField(Fields.VARIANCE_AS_STRING)); + PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.STD_DEVIATION_AS_STRING, value), + new ParseField(Fields.STD_DEVIATION_AS_STRING)); + PARSER.declareObject(ParsedExtendedStats::setStdDeviationBoundsAsString, STD_BOUNDS_AS_STRING_PARSER, + new ParseField(Fields.STD_DEVIATION_BOUNDS_AS_STRING)); + } + + public static ParsedExtendedStats fromXContent(XContentParser parser, final String name) { + ParsedExtendedStats parsedStats = PARSER.apply(parser, null); + parsedStats.setName(name); + return parsedStats; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 63211f3db22..1a8a674a531 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -52,6 +52,8 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedT import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats; import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; @@ -102,6 +104,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); namedXContents.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c)); namedXContents.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)); + namedXContents.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java index 83e1815f398..b6d9fd3dd7b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java @@ -22,11 +22,13 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats.Bounds; import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats; +import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.Before; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -40,15 +42,14 @@ public class InternalExtendedStatsTests extends InternalAggregationTestCase pipelineAggregators, - Map metaData) { - long count = randomIntBetween(1, 50); - double[] minMax = new double[2]; - minMax[0] = randomDouble(); - minMax[0] = randomDouble(); - double sum = randomDoubleBetween(0, 100, true); - return new InternalExtendedStats(name, count, sum, minMax[0], minMax[1], - randomDouble(), sigma, DocValueFormat.RAW, - pipelineAggregators, Collections.emptyMap()); + Map metaData) { + long count = frequently() ? randomIntBetween(1, Integer.MAX_VALUE) : 0; + double min = randomDoubleBetween(-1000000, 1000000, true); + double max = randomDoubleBetween(-1000000, 1000000, true); + double sum = randomDoubleBetween(-1000000, 1000000, true); + DocValueFormat format = randomNumericDocValueFormat(); + return new InternalExtendedStats(name, count, sum, min, max, randomDoubleBetween(0, 1000000, true), sigma, format, + pipelineAggregators, metaData); } @Override @@ -72,10 +73,33 @@ public class InternalExtendedStatsTests extends InternalAggregationTestCase 0 ? aggregation.getSumOfSquares() : 0 , parsed.getSumOfSquares(), 0); + assertEquals(count > 0 ? aggregation.getVariance() : 0 , parsed.getVariance(), 0); + assertEquals(count > 0 ? aggregation.getStdDeviation() : 0 , parsed.getStdDeviation(), 0); + assertEquals(count > 0 ? aggregation.getStdDeviationBound(Bounds.LOWER) : 0 , parsed.getStdDeviationBound(Bounds.LOWER), 0); + assertEquals(count > 0 ? aggregation.getStdDeviationBound(Bounds.UPPER) : 0 , parsed.getStdDeviationBound(Bounds.UPPER), 0); + // also as_string values are only rendered for count != 0 + if (count > 0) { + assertEquals(aggregation.getSumOfSquaresAsString(), parsed.getSumOfSquaresAsString()); + assertEquals(aggregation.getVarianceAsString(), parsed.getVarianceAsString()); + assertEquals(aggregation.getStdDeviationAsString(), parsed.getStdDeviationAsString()); + assertEquals(aggregation.getStdDeviationBoundAsString(Bounds.LOWER), parsed.getStdDeviationBoundAsString(Bounds.LOWER)); + assertEquals(aggregation.getStdDeviationBoundAsString(Bounds.UPPER), parsed.getStdDeviationBoundAsString(Bounds.UPPER)); + } } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java index c0c54d3a8a0..354470114a9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.InternalStats; import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -39,7 +38,7 @@ public class InternalStatsTests extends InternalAggregationTestCase Date: Wed, 26 Apr 2017 11:04:33 +0200 Subject: [PATCH 25/62] Correcting 140 character line length --- .../metrics/stats/extended/ParsedExtendedStats.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java index 8947f4c0aac..2c17376e887 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java @@ -139,8 +139,8 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { return builder; } - private static final ObjectParser PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(), true, - ParsedExtendedStats::new); + private static final ObjectParser PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(), + true, ParsedExtendedStats::new); private static final ConstructingObjectParser, Void> STD_BOUNDS_PARSER = new ConstructingObjectParser<>( ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS", true, args -> new Tuple<>((Double) args[0], (Double) args[1])); @@ -152,7 +152,8 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { } private static final ConstructingObjectParser, Void> STD_BOUNDS_AS_STRING_PARSER = new ConstructingObjectParser<>( - ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true, args -> new Tuple<>((String) args[0], (String) args[1])); + ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true, + args -> new Tuple<>((String) args[0], (String) args[1])); static { STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.LOWER)); STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.UPPER)); From 774368d2acdb522fab7c0d887ef5a80942c225d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 26 Apr 2017 22:32:30 +0200 Subject: [PATCH 26/62] Add parsing for InternalStatsBucket and InternalExtendedStatsBucket (#24312) --- .../stats/extended/ParsedExtendedStats.java | 27 ++++---- .../stats/ParsedStatsBucket.java | 46 ++++++++++++++ .../extended/ParsedExtendedStatsBucket.java | 46 ++++++++++++++ .../InternalAggregationTestCase.java | 6 ++ .../metrics/InternalExtendedStatsTests.java | 10 ++- .../metrics/InternalStatsBucketTests.java | 63 +++++++++++++++++++ .../metrics/InternalStatsTests.java | 7 ++- .../InternalExtendedStatsBucketTests.java | 63 +++++++++++++++++++ 8 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java index 2c17376e887..299b9ce65a4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java @@ -142,6 +142,11 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { private static final ObjectParser PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(), true, ParsedExtendedStats::new); + static { + declareExtendedStatsFields(PARSER); + } + + private static final ConstructingObjectParser, Void> STD_BOUNDS_PARSER = new ConstructingObjectParser<>( ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS", true, args -> new Tuple<>((Double) args[0], (Double) args[1])); static { @@ -159,23 +164,23 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.UPPER)); } - static { - declareAggregationFields(PARSER); - declareStatsFields(PARSER); - PARSER.declareField((agg, value) -> agg.sumOfSquares = value, (parser, context) -> parseDouble(parser, 0), + protected static void declareExtendedStatsFields(ObjectParser objectParser) { + declareAggregationFields(objectParser); + declareStatsFields(objectParser); + objectParser.declareField((agg, value) -> agg.sumOfSquares = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.SUM_OF_SQRS), ValueType.DOUBLE_OR_NULL); - PARSER.declareField((agg, value) -> agg.variance = value, (parser, context) -> parseDouble(parser, 0), + objectParser.declareField((agg, value) -> agg.variance = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.VARIANCE), ValueType.DOUBLE_OR_NULL); - PARSER.declareField((agg, value) -> agg.stdDeviation = value, (parser, context) -> parseDouble(parser, 0), + objectParser.declareField((agg, value) -> agg.stdDeviation = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.STD_DEVIATION), ValueType.DOUBLE_OR_NULL); - PARSER.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER, new ParseField(Fields.STD_DEVIATION_BOUNDS)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_OF_SQRS_AS_STRING, value), + objectParser.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER, new ParseField(Fields.STD_DEVIATION_BOUNDS)); + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_OF_SQRS_AS_STRING, value), new ParseField(Fields.SUM_OF_SQRS_AS_STRING)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.VARIANCE_AS_STRING, value), + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.VARIANCE_AS_STRING, value), new ParseField(Fields.VARIANCE_AS_STRING)); - PARSER.declareString((agg, value) -> agg.valueAsString.put(Fields.STD_DEVIATION_AS_STRING, value), + objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.STD_DEVIATION_AS_STRING, value), new ParseField(Fields.STD_DEVIATION_AS_STRING)); - PARSER.declareObject(ParsedExtendedStats::setStdDeviationBoundsAsString, STD_BOUNDS_AS_STRING_PARSER, + objectParser.declareObject(ParsedExtendedStats::setStdDeviationBoundsAsString, STD_BOUNDS_AS_STRING_PARSER, new ParseField(Fields.STD_DEVIATION_BOUNDS_AS_STRING)); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java new file mode 100644 index 00000000000..4738ad7d9da --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/ParsedStatsBucket.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; + + +public class ParsedStatsBucket extends ParsedStats implements StatsBucket { + + @Override + protected String getType() { + return StatsBucketPipelineAggregationBuilder.NAME; + } + + private static final ObjectParser PARSER = new ObjectParser<>( + ParsedStatsBucket.class.getSimpleName(), true, ParsedStatsBucket::new); + + static { + declareStatsFields(PARSER); + } + + public static ParsedStatsBucket fromXContent(XContentParser parser, final String name) { + ParsedStatsBucket parsedStatsBucket = PARSER.apply(parser, null); + parsedStatsBucket.setName(name); + return parsedStatsBucket; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java new file mode 100644 index 00000000000..08bfbfe587b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ParsedExtendedStatsBucket.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats; + + +public class ParsedExtendedStatsBucket extends ParsedExtendedStats implements ExtendedStatsBucket { + + @Override + protected String getType() { + return ExtendedStatsBucketPipelineAggregationBuilder.NAME; + } + + private static final ObjectParser PARSER = new ObjectParser<>( + ParsedExtendedStatsBucket.class.getSimpleName(), true, ParsedExtendedStatsBucket::new); + + static { + declareExtendedStatsFields(PARSER); + } + + public static ParsedExtendedStatsBucket fromXContent(XContentParser parser, final String name) { + ParsedExtendedStatsBucket parsedStatsBucket = PARSER.apply(parser, null); + parsedStatsBucket.setName(name); + return parsedStatsBucket; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 1a8a674a531..05a4f0eef82 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -63,6 +63,10 @@ import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ParsedExtendedStatsBucket; import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -104,7 +108,9 @@ public abstract class InternalAggregationTestCase namedXContents.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); namedXContents.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c)); namedXContents.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)); + namedXContents.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); namedXContents.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); + namedXContents.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java index b6d9fd3dd7b..d9ce145fd39 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; public class InternalExtendedStatsTests extends InternalAggregationTestCase { - private double sigma; + protected double sigma; @Before public void randomSigma() { @@ -48,8 +48,12 @@ public class InternalExtendedStatsTests extends InternalAggregationTestCase pipelineAggregators, Map metaData) { + return new InternalExtendedStats(name, count, sum, min, max, sumOfSqrs, sigma, formatter, pipelineAggregators, metaData); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java new file mode 100644 index 00000000000..cbb097a7282 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsBucketTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.metrics.stats.InternalStats; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.InternalStatsBucket; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class InternalStatsBucketTests extends InternalStatsTests { + + @Override + protected InternalStatsBucket createInstance(String name, long count, double sum, double min, double max, + DocValueFormat formatter, List pipelineAggregators, Map metaData) { + return new InternalStatsBucket(name, count, sum, min, max, formatter, pipelineAggregators, metaData); + } + + @Override + public void testReduceRandom() { + expectThrows(UnsupportedOperationException.class, + () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, null)); + } + + @Override + protected void assertReduced(InternalStats reduced, List inputs) { + // no test since reduce operation is unsupported + } + + @Override + protected Writeable.Reader instanceReader() { + return InternalStatsBucket::new; + } + + @Override + protected void assertFromXContent(InternalStats aggregation, ParsedAggregation parsedAggregation) { + super.assertFromXContent(aggregation, parsedAggregation); + assertTrue(parsedAggregation instanceof ParsedStatsBucket); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java index 354470114a9..6bfcfdd1ee5 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java @@ -38,7 +38,12 @@ public class InternalStatsTests extends InternalAggregationTestCase pipelineAggregators, Map metaData) { + return new InternalStats(name, count, sum, min, max, formatter, pipelineAggregators, metaData); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java new file mode 100644 index 00000000000..5261c686174 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucketTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests; +import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class InternalExtendedStatsBucketTests extends InternalExtendedStatsTests { + + @Override + protected InternalExtendedStatsBucket createInstance(String name, long count, double sum, double min, double max, double sumOfSqrs, + double sigma, DocValueFormat formatter, List pipelineAggregators, Map metaData) { + return new InternalExtendedStatsBucket(name, count, sum, min, max, sumOfSqrs, sigma, formatter, pipelineAggregators, + Collections.emptyMap()); + } + + @Override + public void testReduceRandom() { + expectThrows(UnsupportedOperationException.class, + () -> createTestInstance("name", Collections.emptyList(), null).reduce(null, null)); + } + + @Override + protected void assertReduced(InternalExtendedStats reduced, List inputs) { + // no test since reduce operation is unsupported + } + + @Override + protected Writeable.Reader instanceReader() { + return InternalExtendedStatsBucket::new; + } + + @Override + protected void assertFromXContent(InternalExtendedStats aggregation, ParsedAggregation parsedAggregation) { + super.assertFromXContent(aggregation, parsedAggregation); + assertTrue(parsedAggregation instanceof ParsedExtendedStatsBucket); + } +} From e2b95f9758d1cd15688e2c466b4840006715e2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 26 Apr 2017 23:07:49 +0200 Subject: [PATCH 27/62] Fix order of static field declarations for parser --- .../stats/extended/ParsedExtendedStats.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java index 299b9ce65a4..f8a219fa6b7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java @@ -142,26 +142,21 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { private static final ObjectParser PARSER = new ObjectParser<>(ParsedExtendedStats.class.getSimpleName(), true, ParsedExtendedStats::new); - static { - declareExtendedStatsFields(PARSER); - } - - private static final ConstructingObjectParser, Void> STD_BOUNDS_PARSER = new ConstructingObjectParser<>( ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS", true, args -> new Tuple<>((Double) args[0], (Double) args[1])); + + private static final ConstructingObjectParser, Void> STD_BOUNDS_AS_STRING_PARSER = new ConstructingObjectParser<>( + ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true, + args -> new Tuple<>((String) args[0], (String) args[1])); + static { STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.LOWER), ValueType.DOUBLE_OR_NULL); STD_BOUNDS_PARSER.declareField(constructorArg(), (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.UPPER), ValueType.DOUBLE_OR_NULL); - } - - private static final ConstructingObjectParser, Void> STD_BOUNDS_AS_STRING_PARSER = new ConstructingObjectParser<>( - ParsedExtendedStats.class.getSimpleName() + "_STD_BOUNDS_AS_STRING", true, - args -> new Tuple<>((String) args[0], (String) args[1])); - static { STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.LOWER)); STD_BOUNDS_AS_STRING_PARSER.declareString(constructorArg(), new ParseField(Fields.UPPER)); + declareExtendedStatsFields(PARSER); } protected static void declareExtendedStatsFields(ObjectParser objectParser) { From b46c39ea2e82bf917a4605c3105623a63532fcbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 27 Apr 2017 00:02:08 +0200 Subject: [PATCH 28/62] Fixed another sneaky LineLength violation --- .../metrics/stats/extended/ParsedExtendedStats.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java index f8a219fa6b7..0eb0bdb215d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ParsedExtendedStats.java @@ -168,7 +168,8 @@ public class ParsedExtendedStats extends ParsedStats implements ExtendedStats { new ParseField(Fields.VARIANCE), ValueType.DOUBLE_OR_NULL); objectParser.declareField((agg, value) -> agg.stdDeviation = value, (parser, context) -> parseDouble(parser, 0), new ParseField(Fields.STD_DEVIATION), ValueType.DOUBLE_OR_NULL); - objectParser.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER, new ParseField(Fields.STD_DEVIATION_BOUNDS)); + objectParser.declareObject(ParsedExtendedStats::setStdDeviationBounds, STD_BOUNDS_PARSER, + new ParseField(Fields.STD_DEVIATION_BOUNDS)); objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.SUM_OF_SQRS_AS_STRING, value), new ParseField(Fields.SUM_OF_SQRS_AS_STRING)); objectParser.declareString((agg, value) -> agg.valueAsString.put(Fields.VARIANCE_AS_STRING, value), From 93415cdfb42f5ecfc40e1bf8a9ecc5369f480b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 27 Apr 2017 11:04:57 +0200 Subject: [PATCH 29/62] Add parsing for InternalPercentilesBucket (#24330) --- .../percentiles/ParsedPercentiles.java | 9 +- .../percentile/ParsedPercentilesBucket.java | 88 +++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../AbstractPercentilesTestCase.java | 8 +- .../hdr/InternalHDRPercentilesTests.java | 4 +- .../InternalPercentilesBucketTests.java | 28 ++++++ 6 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java index eee058fc2f8..a96fb2cd13c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java @@ -33,8 +33,8 @@ import java.util.Map; public abstract class ParsedPercentiles extends ParsedAggregation implements Iterable { - private final Map percentiles = new LinkedHashMap<>(); - private final Map percentilesAsString = new HashMap<>(); + protected final Map percentiles = new LinkedHashMap<>(); + protected final Map percentilesAsString = new HashMap<>(); private boolean keyed; @@ -130,7 +130,6 @@ public abstract class ParsedPercentiles extends ParsedAggregation implements Ite if (token.isValue()) { if (token == XContentParser.Token.VALUE_NUMBER) { aggregation.addPercentile(Double.valueOf(parser.currentName()), parser.doubleValue()); - } else if (token == XContentParser.Token.VALUE_STRING) { int i = parser.currentName().indexOf("_as_string"); if (i > 0) { @@ -140,6 +139,8 @@ public abstract class ParsedPercentiles extends ParsedAggregation implements Ite aggregation.addPercentile(Double.valueOf(parser.currentName()), Double.valueOf(parser.text())); } } + } else if (token == XContentParser.Token.VALUE_NULL) { + aggregation.addPercentile(Double.valueOf(parser.currentName()), Double.NaN); } } } else if (token == XContentParser.Token.START_ARRAY) { @@ -162,6 +163,8 @@ public abstract class ParsedPercentiles extends ParsedAggregation implements Ite } else if (CommonFields.VALUE_AS_STRING.getPreferredName().equals(currentFieldName)) { valueAsString = parser.text(); } + } else if (token == XContentParser.Token.VALUE_NULL) { + value = Double.NaN; } } if (key != null) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java new file mode 100644 index 00000000000..eebe296e531 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/ParsedPercentilesBucket.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; + +import java.io.IOException; +import java.util.Map.Entry; + +public class ParsedPercentilesBucket extends ParsedPercentiles implements Percentiles { + + @Override + public String getType() { + return PercentilesBucketPipelineAggregationBuilder.NAME; + } + + @Override + public double percentile(double percent) throws IllegalArgumentException { + Double value = percentiles.get(percent); + if (value == null) { + throw new IllegalArgumentException("Percent requested [" + String.valueOf(percent) + "] was not" + + " one of the computed percentiles. Available keys are: " + percentiles.keySet()); + } + return value; + } + + @Override + public String percentileAsString(double percent) { + double value = percentile(percent); // check availability as unformatted value + String valueAsString = percentilesAsString.get(percent); + if (valueAsString != null) { + return valueAsString; + } else { + return Double.toString(value); + } + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.startObject("values"); + for (Entry percent : percentiles.entrySet()) { + double value = percent.getValue(); + boolean hasValue = !(Double.isNaN(value)); + Double key = percent.getKey(); + builder.field(Double.toString(key), hasValue ? value : null); + String valueAsString = percentilesAsString.get(key); + if (hasValue && valueAsString != null) { + builder.field(key + "_as_string", valueAsString); + } + } + builder.endObject(); + return builder; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedPercentilesBucket.class.getSimpleName(), true, ParsedPercentilesBucket::new); + + static { + ParsedPercentiles.declarePercentilesFields(PARSER); + } + + public static ParsedPercentilesBucket fromXContent(XContentParser parser, String name) throws IOException { + ParsedPercentilesBucket aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 05a4f0eef82..d4c293a0e8b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -63,6 +63,8 @@ import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.ParsedPercentilesBucket; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder; @@ -99,6 +101,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c)); namedXContents.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c)); namedXContents.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c)); + namedXContents.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c)); namedXContents.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)); namedXContents.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)); namedXContents.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java index 0223db03599..72167dac651 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.Before; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,7 +40,7 @@ public abstract class AbstractPercentilesTestCase randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d); double[] percents = new double[randomCdfValues.size()]; for (int i = 0; i < randomCdfValues.size(); i++) { percents[i] = randomCdfValues.get(i); } + if (sorted) { + Arrays.sort(percents); + } return percents; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java index 908a1db18c6..07c74d78b08 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentilesTests.java @@ -23,8 +23,8 @@ import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentilesTestCase; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.percentiles.ParsedPercentiles; +import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.Arrays; @@ -70,7 +70,7 @@ public class InternalHDRPercentilesTests extends InternalPercentilesTestCase parsedAggregation = parseAndAssert(aggregation, false); + Iterator it = aggregation.iterator(); + Iterator parsedIt = parsedAggregation.iterator(); + while (it.hasNext()) { + assertEquals(it.next(), parsedIt.next()); + } + } } From a6c38b8f8a9c95f81c10bd97318d9ab66142108e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 28 Apr 2017 12:10:16 +0200 Subject: [PATCH 30/62] Add parsing for InternalGeoBounds (#24365) --- .../metrics/geobounds/InternalGeoBounds.java | 23 ++-- .../metrics/geobounds/ParsedGeoBounds.java | 105 ++++++++++++++++++ .../InternalAggregationTestCase.java | 6 +- .../geobounds/InternalGeoBoundsTests.java | 34 ++++-- 4 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java index 2a3d03e43e6..37297475a28 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.geobounds; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -32,6 +33,14 @@ import java.util.Map; import java.util.Objects; public class InternalGeoBounds extends InternalAggregation implements GeoBounds { + + final static ParseField BOUNDS_FIELD = new ParseField("bounds"); + final static ParseField TOP_LEFT_FIELD = new ParseField("top_left"); + final static ParseField BOTTOM_RIGHT_FIELD = new ParseField("bottom_right"); + final static ParseField LAT_FIELD = new ParseField("lat"); + final static ParseField LON_FIELD = new ParseField("lon"); + + final double top; final double bottom; final double posLeft; @@ -170,14 +179,14 @@ public class InternalGeoBounds extends InternalAggregation implements GeoBounds GeoPoint topLeft = topLeft(); GeoPoint bottomRight = bottomRight(); if (topLeft != null) { - builder.startObject("bounds"); - builder.startObject("top_left"); - builder.field("lat", topLeft.lat()); - builder.field("lon", topLeft.lon()); + builder.startObject(BOUNDS_FIELD.getPreferredName()); + builder.startObject(TOP_LEFT_FIELD.getPreferredName()); + builder.field(LAT_FIELD.getPreferredName(), topLeft.lat()); + builder.field(LON_FIELD.getPreferredName(), topLeft.lon()); builder.endObject(); - builder.startObject("bottom_right"); - builder.field("lat", bottomRight.lat()); - builder.field("lon", bottomRight.lon()); + builder.startObject(BOTTOM_RIGHT_FIELD.getPreferredName()); + builder.field(LAT_FIELD.getPreferredName(), bottomRight.lat()); + builder.field(LON_FIELD.getPreferredName(), bottomRight.lon()); builder.endObject(); builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java new file mode 100644 index 00000000000..04d2b2448d2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/ParsedGeoBounds.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geobounds; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.BOTTOM_RIGHT_FIELD; +import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.BOUNDS_FIELD; +import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.LAT_FIELD; +import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.LON_FIELD; +import static org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds.TOP_LEFT_FIELD; + +public class ParsedGeoBounds extends ParsedAggregation implements GeoBounds { + private GeoPoint topLeft; + private GeoPoint bottomRight; + + @Override + public String getType() { + return GeoBoundsAggregationBuilder.NAME; + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (topLeft != null) { + builder.startObject("bounds"); + builder.startObject("top_left"); + builder.field("lat", topLeft.getLat()); + builder.field("lon", topLeft.getLon()); + builder.endObject(); + builder.startObject("bottom_right"); + builder.field("lat", bottomRight.getLat()); + builder.field("lon", bottomRight.getLon()); + builder.endObject(); + builder.endObject(); + } + return builder; + } + + @Override + public GeoPoint topLeft() { + return topLeft; + } + + @Override + public GeoPoint bottomRight() { + return bottomRight; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedGeoBounds.class.getSimpleName(), true, + ParsedGeoBounds::new); + + private static final ConstructingObjectParser, Void> BOUNDS_PARSER = + new ConstructingObjectParser<>(ParsedGeoBounds.class.getSimpleName() + "_BOUNDS", true, + args -> new Tuple<>((GeoPoint) args[0], (GeoPoint) args[1])); + + private static final ObjectParser GEO_POINT_PARSER = new ObjectParser<>( + ParsedGeoBounds.class.getSimpleName() + "_POINT", true, GeoPoint::new); + + static { + declareAggregationFields(PARSER); + PARSER.declareObject((agg, bbox) -> { + agg.topLeft = bbox.v1(); + agg.bottomRight = bbox.v2(); + }, BOUNDS_PARSER, BOUNDS_FIELD); + + BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, TOP_LEFT_FIELD); + BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, BOTTOM_RIGHT_FIELD); + + GEO_POINT_PARSER.declareDouble(GeoPoint::resetLat, LAT_FIELD); + GEO_POINT_PARSER.declareDouble(GeoPoint::resetLon, LON_FIELD); + } + + public static ParsedGeoBounds fromXContent(XContentParser parser, final String name) { + ParsedGeoBounds geoBounds = PARSER.apply(parser, null); + geoBounds.setName(name); + return geoBounds; + } + +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index d4c293a0e8b..664a2b756df 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -38,6 +38,8 @@ import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.geobounds.ParsedGeoBounds; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.ParsedMax; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; @@ -113,7 +115,9 @@ public abstract class InternalAggregationTestCase namedXContents.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)); namedXContents.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); namedXContents.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); - namedXContents.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); + namedXContents.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, + (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); + namedXContents.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java index cd5d4d43d17..611178d1bf2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBoundsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.metrics.geobounds; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.Collections; @@ -35,8 +36,10 @@ public class InternalGeoBoundsTests extends InternalAggregationTestCase pipelineAggregators, Map metaData) { + // we occasionally want to test top = Double.NEGATIVE_INFINITY since this triggers empty xContent object + double top = frequently() ? randomDouble() : Double.NEGATIVE_INFINITY; InternalGeoBounds geo = new InternalGeoBounds(name, - randomDouble(), randomDouble(), randomDouble(), randomDouble(), + top, randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomBoolean(), pipelineAggregators, Collections.emptyMap()); return geo; @@ -70,12 +73,29 @@ public class InternalGeoBoundsTests extends InternalAggregationTestCase Date: Fri, 28 Apr 2017 14:19:00 +0200 Subject: [PATCH 31/62] Fixing checktyle error for modifier order --- .../metrics/geobounds/InternalGeoBounds.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java index 37297475a28..a64115447be 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java @@ -34,11 +34,11 @@ import java.util.Objects; public class InternalGeoBounds extends InternalAggregation implements GeoBounds { - final static ParseField BOUNDS_FIELD = new ParseField("bounds"); - final static ParseField TOP_LEFT_FIELD = new ParseField("top_left"); - final static ParseField BOTTOM_RIGHT_FIELD = new ParseField("bottom_right"); - final static ParseField LAT_FIELD = new ParseField("lat"); - final static ParseField LON_FIELD = new ParseField("lon"); + static final ParseField BOUNDS_FIELD = new ParseField("bounds"); + static final ParseField TOP_LEFT_FIELD = new ParseField("top_left"); + static final ParseField BOTTOM_RIGHT_FIELD = new ParseField("bottom_right"); + static final ParseField LAT_FIELD = new ParseField("lat"); + static final ParseField LON_FIELD = new ParseField("lon"); final double top; From 4d14143ac637bf11764723319bb37299b2e29ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 28 Apr 2017 17:08:48 +0200 Subject: [PATCH 32/62] Add parsing for InternalGeoCentroid (#24371) --- .../geocentroid/InternalGeoCentroid.java | 6 +- .../geocentroid/ParsedGeoCentroid.java | 87 +++++++++++++++++++ .../InternalAggregationTestCase.java | 3 + .../geocentroid/InternalGeoCentroidTests.java | 10 +++ 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java index c5578813c84..b8d317ff787 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -149,17 +149,13 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr static class Fields { static final ParseField CENTROID = new ParseField("location"); + static final ParseField COUNT = new ParseField("count"); static final ParseField CENTROID_LAT = new ParseField("lat"); static final ParseField CENTROID_LON = new ParseField("lon"); - static final ParseField COUNT = new ParseField("count"); } @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - return renderXContent(builder, params, centroid, count); - } - - static XContentBuilder renderXContent(XContentBuilder builder, Params params, GeoPoint centroid, long count) throws IOException { if (centroid != null) { builder.startObject(Fields.CENTROID.getPreferredName()); { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java new file mode 100644 index 00000000000..ed09c281868 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/ParsedGeoCentroid.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid.Fields; + +import java.io.IOException; + +/** + * Serialization and merge logic for {@link GeoCentroidAggregator}. + */ +public class ParsedGeoCentroid extends ParsedAggregation implements GeoCentroid { + private GeoPoint centroid; + private long count; + + @Override + public GeoPoint centroid() { + return centroid; + } + + @Override + public long count() { + return count; + } + + @Override + protected String getType() { + return GeoCentroidAggregationBuilder.NAME; + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (centroid != null) { + builder.startObject(Fields.CENTROID.getPreferredName()); + { + builder.field(Fields.CENTROID_LAT.getPreferredName(), centroid.lat()); + builder.field(Fields.CENTROID_LON.getPreferredName(), centroid.lon()); + } + builder.endObject(); + } + builder.field(Fields.COUNT.getPreferredName(), count); + return builder; + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedGeoCentroid.class.getSimpleName(), true, + ParsedGeoCentroid::new); + + private static final ObjectParser GEO_POINT_PARSER = new ObjectParser<>( + ParsedGeoCentroid.class.getSimpleName() + "_POINT", true, GeoPoint::new); + + static { + declareAggregationFields(PARSER); + PARSER.declareObject((agg, centroid) -> agg.centroid = centroid, GEO_POINT_PARSER, Fields.CENTROID); + PARSER.declareLong((agg, count) -> agg.count = count, Fields.COUNT); + + GEO_POINT_PARSER.declareDouble(GeoPoint::resetLat, Fields.CENTROID_LAT); + GEO_POINT_PARSER.declareDouble(GeoPoint::resetLon, Fields.CENTROID_LON); + } + + public static ParsedGeoCentroid fromXContent(XContentParser parser, final String name) { + ParsedGeoCentroid geoCentroid = PARSER.apply(parser, null); + geoCentroid.setName(name); + return geoCentroid; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 664a2b756df..e99ad8da131 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -40,6 +40,8 @@ import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggr import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.geobounds.ParsedGeoBounds; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.geocentroid.ParsedGeoCentroid; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.ParsedMax; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; @@ -118,6 +120,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); namedXContents.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); + namedXContents.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java index 3bbe1a1b462..31da19bb915 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroidTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.geo.RandomGeoGenerator; @@ -70,4 +71,13 @@ public class InternalGeoCentroidTests extends InternalAggregationTestCase Date: Wed, 3 May 2017 10:45:31 +0200 Subject: [PATCH 33/62] Introduce SearchResponseSections base class (#24442) SearchResponseSections is the common part extracted from InternalSearchResponse that can be shared between high level REST and elasticsearch. The only bits left out are around serialization which is not supported. This way it can accept Aggregations as a constructor argument, without requiring InternalAggregations, as the high level REST client uses its own objects for aggs parsing rather than internal ones. This change also makes Aggregations implement ToXContent, and Aggregation extend ToXContent. Especially the latter is suboptimal but the best solution that allows to share as much code as possible between core and the client, that doesn't require messing with generics and making the api complicated. Also it doesn't have downsides as all of the current implementations of Aggregation do implement ToXContent already. --- .../action/search/SearchResponse.java | 7 +- .../action/search/SearchResponseSections.java | 122 ++++++++++++++++ .../search/aggregations/Aggregation.java | 5 +- .../search/aggregations/Aggregations.java | 43 +++++- .../aggregations/InternalAggregations.java | 23 +-- .../internal/InternalSearchResponse.java | 136 +++--------------- 6 files changed, 190 insertions(+), 146 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 54d8eab99e7..3e83996c7c6 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -39,14 +39,13 @@ import java.io.IOException; import java.util.Map; import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure; -import static org.elasticsearch.search.internal.InternalSearchResponse.readInternalSearchResponse; /** * A response of a search request. */ public class SearchResponse extends ActionResponse implements StatusToXContentObject { - private InternalSearchResponse internalResponse; + private SearchResponseSections internalResponse; private String scrollId; @@ -61,7 +60,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb public SearchResponse() { } - public SearchResponse(InternalSearchResponse internalResponse, String scrollId, int totalShards, int successfulShards, + public SearchResponse(SearchResponseSections internalResponse, String scrollId, int totalShards, int successfulShards, long tookInMillis, ShardSearchFailure[] shardFailures) { this.internalResponse = internalResponse; this.scrollId = scrollId; @@ -209,7 +208,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - internalResponse = readInternalSearchResponse(in); + internalResponse = new InternalSearchResponse(in); totalShards = in.readVInt(); successfulShards = in.readVInt(); int size = in.readVInt(); diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java new file mode 100644 index 00000000000..1757acbfd6d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.profile.ProfileShardResult; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.suggest.Suggest; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Base class that holds the various sections which a search response is + * composed of (hits, aggs, suggestions etc.) and allows to retrieve them. + * + * The reason why this class exists is that the high level REST client uses its own classes + * to parse aggregations into, which are not serializable. This is the common part that can be + * shared between core and client. + */ +public class SearchResponseSections implements ToXContent { + + protected final SearchHits hits; + protected final Aggregations aggregations; + protected final Suggest suggest; + protected final SearchProfileShardResults profileResults; + protected final boolean timedOut; + protected final Boolean terminatedEarly; + protected final int numReducePhases; + + public SearchResponseSections(SearchHits hits, Aggregations aggregations, Suggest suggest, boolean timedOut, Boolean terminatedEarly, + SearchProfileShardResults profileResults, int numReducePhases) { + this.hits = hits; + this.aggregations = aggregations; + this.suggest = suggest; + this.profileResults = profileResults; + this.timedOut = timedOut; + this.terminatedEarly = terminatedEarly; + this.numReducePhases = numReducePhases; + } + + public final boolean timedOut() { + return this.timedOut; + } + + public final Boolean terminatedEarly() { + return this.terminatedEarly; + } + + public final SearchHits hits() { + return hits; + } + + public final Aggregations aggregations() { + return aggregations; + } + + public final Suggest suggest() { + return suggest; + } + + /** + * Returns the number of reduce phases applied to obtain this search response + */ + public final int getNumReducePhases() { + return numReducePhases; + } + + /** + * Returns the profile results for this search response (including all shards). + * An empty map is returned if profiling was not enabled + * + * @return Profile results + */ + public final Map profile() { + if (profileResults == null) { + return Collections.emptyMap(); + } + return profileResults.getShardResults(); + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + hits.toXContent(builder, params); + if (aggregations != null) { + aggregations.toXContent(builder, params); + } + if (suggest != null) { + suggest.toXContent(builder, params); + } + if (profileResults != null) { + profileResults.toXContent(builder, params); + } + return builder; + } + + protected void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java index 991243ce0d4..eb55f1e7abc 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregation.java @@ -19,13 +19,14 @@ package org.elasticsearch.search.aggregations; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContent; import java.util.Map; /** - * An aggregation + * An aggregation. Extends {@link ToXContent} as it makes it easier to print out its content. */ -public interface Aggregation { +public interface Aggregation extends ToXContent { /** * Delimiter used when prefixing aggregation names with their type diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java index ef84edfb729..c7f84fe6577 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java @@ -18,6 +18,13 @@ */ package org.elasticsearch.search.aggregations; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -30,7 +37,9 @@ import static java.util.Collections.unmodifiableMap; /** * Represents a set of {@link Aggregation}s */ -public abstract class Aggregations implements Iterable { +public class Aggregations implements Iterable, ToXContent { + + public static final String AGGREGATIONS_FIELD = "aggregations"; protected List aggregations = Collections.emptyList(); protected Map aggregationsAsMap; @@ -98,4 +107,36 @@ public abstract class Aggregations implements Iterable { public final int hashCode() { return Objects.hash(getClass(), aggregations); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (aggregations.isEmpty()) { + return builder; + } + builder.startObject(AGGREGATIONS_FIELD); + toXContentInternal(builder, params); + return builder.endObject(); + } + + /** + * Directly write all the aggregations without their bounding object. Used by sub-aggregations (non top level aggs) + */ + public XContentBuilder toXContentInternal(XContentBuilder builder, Params params) throws IOException { + for (Aggregation aggregation : aggregations) { + aggregation.toXContent(builder, params); + } + return builder; + } + + //TODO add tests for this method + public static Aggregations fromXContent(XContentParser parser) throws IOException { + final List aggregations = new ArrayList<>(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + return new Aggregations(aggregations); + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java index 64c9e4794c3..846dc1bb3b0 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import java.io.IOException; @@ -32,6 +31,7 @@ import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; + /** * An internal implementation of {@link Aggregations}. */ @@ -80,27 +80,6 @@ public final class InternalAggregations extends Aggregations implements ToXConte return new InternalAggregations(reducedAggregations); } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (aggregations.isEmpty()) { - return builder; - } - builder.startObject("aggregations"); - toXContentInternal(builder, params); - return builder.endObject(); - } - - /** - * Directly write all the aggregations without their bounding object. Used by sub-aggregations (non top level aggs) - */ - public XContentBuilder toXContentInternal(XContentBuilder builder, Params params) throws IOException { - for (Aggregation aggregation : aggregations) { - ((InternalAggregation)aggregation).toXContent(builder, params); - } - return builder; - } - - public static InternalAggregations readAggregations(StreamInput in) throws IOException { InternalAggregations result = new InternalAggregations(); result.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java index 391f6efe18b..baa153e3470 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java +++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java @@ -19,148 +19,50 @@ package org.elasticsearch.search.internal; +import org.elasticsearch.action.search.SearchResponseSections; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.suggest.Suggest; import java.io.IOException; -import java.util.Collections; -import java.util.Map; -public class InternalSearchResponse implements Streamable, ToXContent { +/** + * {@link SearchResponseSections} subclass that can be serialized over the wire. + */ +public class InternalSearchResponse extends SearchResponseSections implements Writeable, ToXContent { public static InternalSearchResponse empty() { return new InternalSearchResponse(SearchHits.empty(), null, null, null, false, null, 1); } - private SearchHits hits; - - private InternalAggregations aggregations; - - private Suggest suggest; - - private SearchProfileShardResults profileResults; - - private boolean timedOut; - - private Boolean terminatedEarly = null; - - private int numReducePhases = 1; - - private InternalSearchResponse() { - } - public InternalSearchResponse(SearchHits hits, InternalAggregations aggregations, Suggest suggest, SearchProfileShardResults profileResults, boolean timedOut, Boolean terminatedEarly, int numReducePhases) { - this.hits = hits; - this.aggregations = aggregations; - this.suggest = suggest; - this.profileResults = profileResults; - this.timedOut = timedOut; - this.terminatedEarly = terminatedEarly; - this.numReducePhases = numReducePhases; + super(hits, aggregations, suggest, timedOut, terminatedEarly, profileResults, numReducePhases); } - public boolean timedOut() { - return this.timedOut; - } - - public Boolean terminatedEarly() { - return this.terminatedEarly; - } - - public SearchHits hits() { - return hits; - } - - public Aggregations aggregations() { - return aggregations; - } - - public Suggest suggest() { - return suggest; - } - - /** - * Returns the number of reduce phases applied to obtain this search response - */ - public int getNumReducePhases() { - return numReducePhases; - } - - /** - * Returns the profile results for this search response (including all shards). - * An empty map is returned if profiling was not enabled - * - * @return Profile results - */ - public Map profile() { - if (profileResults == null) { - return Collections.emptyMap(); - } - return profileResults.getShardResults(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - hits.toXContent(builder, params); - if (aggregations != null) { - aggregations.toXContent(builder, params); - } - if (suggest != null) { - suggest.toXContent(builder, params); - } - if (profileResults != null) { - profileResults.toXContent(builder, params); - } - return builder; - } - - public static InternalSearchResponse readInternalSearchResponse(StreamInput in) throws IOException { - InternalSearchResponse response = new InternalSearchResponse(); - response.readFrom(in); - return response; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - hits = SearchHits.readSearchHits(in); - if (in.readBoolean()) { - aggregations = InternalAggregations.readAggregations(in); - } - if (in.readBoolean()) { - suggest = Suggest.readSuggest(in); - } - timedOut = in.readBoolean(); - terminatedEarly = in.readOptionalBoolean(); - profileResults = in.readOptionalWriteable(SearchProfileShardResults::new); - numReducePhases = in.readVInt(); + public InternalSearchResponse(StreamInput in) throws IOException { + super( + SearchHits.readSearchHits(in), + in.readBoolean() ? InternalAggregations.readAggregations(in) : null, + in.readBoolean() ? Suggest.readSuggest(in) : null, + in.readBoolean(), + in.readOptionalBoolean(), + in.readOptionalWriteable(SearchProfileShardResults::new), + in.readVInt() + ); } @Override public void writeTo(StreamOutput out) throws IOException { hits.writeTo(out); - if (aggregations == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - aggregations.writeTo(out); - } - if (suggest == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - suggest.writeTo(out); - } + out.writeOptionalStreamable((InternalAggregations)aggregations); + out.writeOptionalStreamable(suggest); out.writeBoolean(timedOut); out.writeOptionalBoolean(terminatedEarly); out.writeOptionalWriteable(profileResults); From 2ac90b3de96b375e4fb61035eb6679917e7c017d Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 3 May 2017 13:27:20 +0200 Subject: [PATCH 34/62] Add parsing methods for InternalDateHistogram and InternalHistogram (#24213) --- .../common/xcontent/XContentParserUtils.java | 8 +- .../ParsedMultiBucketAggregation.java | 181 ++++++++++++++++++ .../bucket/histogram/ParsedDateHistogram.java | 80 ++++++++ .../bucket/histogram/ParsedHistogram.java | 73 +++++++ .../elasticsearch/search/suggest/Suggest.java | 1 + .../xcontent/XContentParserUtilsTests.java | 8 +- .../InternalAggregationTestCase.java | 6 + ...nternalMultiBucketAggregationTestCase.java | 142 ++++++++++++++ .../histogram/InternalDateHistogramTests.java | 33 +++- .../histogram/InternalHistogramTests.java | 32 +++- 10 files changed, 536 insertions(+), 28 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java index 169202e40d7..30199afa98c 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java @@ -111,10 +111,9 @@ public final class XContentParserUtils { } /** - * This method expects that the current token is a {@code XContentParser.Token.FIELD_NAME} and - * that the current field name is the concatenation of a type, delimiter and name (ex: terms#foo - * where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry}, "#" is - * the delimiter and "foo" the name of the object to parse). + * This method expects that the current field name is the concatenation of a type, a delimiter and a name + * (ex: terms#foo where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry}, + * "#" is the delimiter and "foo" the name of the object to parse). * * The method splits the field's name to extract the type and name and then parses the object * using the {@link XContentParser#namedObject(Class, String, Object)} method. @@ -128,7 +127,6 @@ public final class XContentParserUtils { * from the field's name */ public static T parseTypedKeysObject(XContentParser parser, String delimiter, Class objectClass) throws IOException { - ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); String currentFieldName = parser.currentName(); if (Strings.hasLength(currentFieldName)) { int position = currentFieldName.indexOf(delimiter); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java new file mode 100644 index 00000000000..b2823669a75 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java @@ -0,0 +1,181 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +public abstract class ParsedMultiBucketAggregation extends ParsedAggregation implements MultiBucketsAggregation { + + protected final List> buckets = new ArrayList<>(); + protected boolean keyed; + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (keyed) { + builder.startObject(CommonFields.BUCKETS.getPreferredName()); + } else { + builder.startArray(CommonFields.BUCKETS.getPreferredName()); + } + for (ParsedBucket bucket : buckets) { + bucket.toXContent(builder, params); + } + if (keyed) { + builder.endObject(); + } else { + builder.endArray(); + } + return builder; + } + + protected static void declareMultiBucketAggregationFields(final ObjectParser objectParser, + final CheckedFunction, IOException> bucketParser, + final CheckedFunction, IOException> keyedBucketParser) { + declareAggregationFields(objectParser); + objectParser.declareField((parser, aggregation, context) -> { + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + aggregation.keyed = true; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + aggregation.buckets.add(keyedBucketParser.apply(parser)); + } + } else if (token == XContentParser.Token.START_ARRAY) { + aggregation.keyed = false; + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + aggregation.buckets.add(bucketParser.apply(parser)); + } + } + }, CommonFields.BUCKETS, ObjectParser.ValueType.OBJECT_ARRAY); + } + + public static class ParsedBucket implements MultiBucketsAggregation.Bucket { + + private Aggregations aggregations; + private T key; + private String keyAsString; + private long docCount; + private boolean keyed; + + protected void setKey(T key) { + this.key = key; + } + + @Override + public Object getKey() { + return key; + } + + protected void setKeyAsString(String keyAsString) { + this.keyAsString = keyAsString; + } + + @Override + public String getKeyAsString() { + return keyAsString; + } + + protected void setDocCount(long docCount) { + this.docCount = docCount; + } + + @Override + public long getDocCount() { + return docCount; + } + + public void setKeyed(boolean keyed) { + this.keyed = keyed; + } + + protected void setAggregations(Aggregations aggregations) { + this.aggregations = aggregations; + } + + @Override + public Aggregations getAggregations() { + return aggregations; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (keyed) { + // Subclasses can override the getKeyAsString method to handle specific cases like + // keyed bucket with RAW doc value format where the key_as_string field is not printed + // out but we still need to have a string version of the key to use as the bucket's name. + builder.startObject(getKeyAsString()); + } else { + builder.startObject(); + } + if (keyAsString != null) { + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); + } + builder.field(CommonFields.KEY.getPreferredName(), key); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + aggregations.toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + protected static > B parseXContent(final XContentParser parser, + final boolean keyed, + final Supplier bucketSupplier, + final CheckedFunction keyParser) + throws IOException { + final B bucket = bucketSupplier.get(); + bucket.setKeyed(keyed); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (keyed) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + } + + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + bucket.setKey(keyParser.apply(parser)); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + bucket.setAggregations(new Aggregations(aggregations)); + return bucket; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java new file mode 100644 index 00000000000..27ba2c029d2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.histogram; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements Histogram { + + @Override + protected String getType() { + return DateHistogramAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets.stream().map(bucket -> (Histogram.Bucket) bucket).collect(Collectors.toList()); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedDateHistogram.class.getSimpleName(), true, ParsedDateHistogram::new); + static { + declareMultiBucketAggregationFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedDateHistogram fromXContent(XContentParser parser, String name) throws IOException { + ParsedDateHistogram aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + + @Override + public Object getKey() { + return new DateTime(super.getKey(), DateTimeZone.UTC); + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } else { + return DocValueFormat.RAW.format((Long) super.getKey()); + } + } + + static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { + return parseXContent(parser, keyed, ParsedBucket::new, XContentParser::longValue); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java new file mode 100644 index 00000000000..2b6730df2cc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.histogram; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +public class ParsedHistogram extends ParsedMultiBucketAggregation implements Histogram { + + @Override + protected String getType() { + return HistogramAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets.stream().map(bucket -> (Histogram.Bucket) bucket).collect(Collectors.toList()); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedHistogram.class.getSimpleName(), true, ParsedHistogram::new); + static { + declareMultiBucketAggregationFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedHistogram fromXContent(XContentParser parser, String name) throws IOException { + ParsedHistogram aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } else { + return DocValueFormat.RAW.format((Double) getKey()); + } + } + + static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { + return parseXContent(parser, keyed, ParsedBucket::new, XContentParser::doubleValue); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java index 73cad2310f8..4af794ab42c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java @@ -386,6 +386,7 @@ public class Suggest implements Iterable> fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); return XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Suggestion.class); } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java index d0426e1e104..4b90016eaa4 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java @@ -65,12 +65,10 @@ public class XContentParserUtilsTests extends ESTestCase { BytesReference bytes = toXContent((builder, params) -> builder.field("test", 0), xContentType, randomBoolean()); try (XContentParser parser = xContentType.xContent().createParser(namedXContentRegistry, bytes)) { - parser.nextToken(); - ParsingException e = expectThrows(ParsingException.class, () -> parseTypedKeysObject(parser, delimiter, Boolean.class)); - assertEquals("Failed to parse object: expecting token of type [FIELD_NAME] but found [START_OBJECT]", e.getMessage()); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation); - parser.nextToken(); - e = expectThrows(ParsingException.class, () -> parseTypedKeysObject(parser, delimiter, Boolean.class)); + ParsingException e = expectThrows(ParsingException.class, () -> parseTypedKeysObject(parser, delimiter, Boolean.class)); assertEquals("Cannot parse object of class [Boolean] without type information. Set [typed_keys] parameter " + "on the request to ensure the type information is added to the response output", e.getMessage()); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index e99ad8da131..782e98e8885 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -34,6 +34,10 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; @@ -121,6 +125,8 @@ public abstract class InternalAggregationTestCase (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); namedXContents.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); namedXContents.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); + namedXContents.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); + namedXContents.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java new file mode 100644 index 00000000000..ae293d5ac31 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyMap; + +public abstract class InternalMultiBucketAggregationTestCase + extends InternalAggregationTestCase { + + private boolean hasSubAggregations; + + @Before + public void initHasSubAggregations() { + hasSubAggregations = randomBoolean(); + } + + @Override + protected final T createTestInstance(String name, List pipelineAggregators, Map metaData) { + List internal = new ArrayList<>(); + if (hasSubAggregations) { + final int numAggregations = randomIntBetween(1, 3); + for (int i = 0; i pipelineAggregators, + Map metaData, InternalAggregations aggregations); + + protected abstract Class implementationClass(); + + @Override + protected final void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + assertMultiBucketsAggregation(aggregation, parsedAggregation, false); + } + + public void testIterators() throws IOException { + final T aggregation = createTestInstance(); + assertMultiBucketsAggregation(aggregation, parseAndAssert(aggregation, false), true); + } + + private void assertMultiBucketsAggregation(Aggregation expected, Aggregation actual, boolean checkOrder) { + assertTrue(expected instanceof MultiBucketsAggregation); + MultiBucketsAggregation expectedMultiBucketsAggregation = (MultiBucketsAggregation) expected; + + assertTrue(actual instanceof MultiBucketsAggregation); + MultiBucketsAggregation actualMultiBucketsAggregation = (MultiBucketsAggregation) actual; + + Class parsedClass = implementationClass(); + assertTrue(parsedClass != null && parsedClass.isInstance(actual)); + + assertTrue(expected instanceof InternalAggregation && actual instanceof ParsedAggregation); + assertEquals(expected.getName(), actual.getName()); + assertEquals(expected.getMetaData(), actual.getMetaData()); + assertEquals(((InternalAggregation) expected).getType(), ((ParsedAggregation) actual).getType()); + + List expectedBuckets = expectedMultiBucketsAggregation.getBuckets(); + List actualBuckets = actualMultiBucketsAggregation.getBuckets(); + assertEquals(expectedBuckets.size(), actualBuckets.size()); + + if (checkOrder) { + Iterator expectedIt = expectedBuckets.iterator(); + Iterator actualIt = actualBuckets.iterator(); + while (expectedIt.hasNext()) { + MultiBucketsAggregation.Bucket expectedBucket = expectedIt.next(); + MultiBucketsAggregation.Bucket actualBucket = actualIt.next(); + assertBucket(expectedBucket, actualBucket, true); + } + } else { + for (MultiBucketsAggregation.Bucket expectedBucket : expectedBuckets) { + boolean found = false; + for (MultiBucketsAggregation.Bucket actualBucket : actualBuckets) { + if (actualBucket.getKey().equals(expectedBucket.getKey())) { + found = true; + assertBucket(expectedBucket, actualBucket, false); + break; + } + } + assertTrue("Failed to find bucket with key [" + expectedBucket.getKey() + "]", found); + } + } + } + + private void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) { + assertTrue(expected instanceof InternalMultiBucketAggregation.InternalBucket); + assertTrue(actual instanceof ParsedMultiBucketAggregation.ParsedBucket); + + assertEquals(expected.getKey(), actual.getKey()); + assertEquals(expected.getKeyAsString(), actual.getKeyAsString()); + assertEquals(expected.getDocCount(), actual.getDocCount()); + + Aggregations expectedAggregations = expected.getAggregations(); + Aggregations actualAggregations = actual.getAggregations(); + assertEquals(expectedAggregations.asList().size(), actualAggregations.asList().size()); + + if (checkOrder) { + Iterator expectedIt = expectedAggregations.iterator(); + Iterator actualIt = actualAggregations.iterator(); + + while (expectedIt.hasNext()) { + Aggregation expectedAggregation = expectedIt.next(); + Aggregation actualAggregation = actualIt.next(); + assertMultiBucketsAggregation(expectedAggregation, actualAggregation, true); + } + } else { + for (Aggregation expectedAggregation : expectedAggregations) { + Aggregation actualAggregation = actualAggregations.get(expectedAggregation.getName()); + assertNotNull(actualAggregation); + assertMultiBucketsAggregation(expectedAggregation, actualAggregation, false); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java index 40f268e6556..c4410713ac8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java @@ -19,14 +19,14 @@ package org.elasticsearch.search.aggregations.bucket.histogram; -import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.joda.time.DateTime; +import org.junit.Before; import java.util.ArrayList; import java.util.List; @@ -37,14 +37,22 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueHours; import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -public class InternalDateHistogramTests extends InternalAggregationTestCase { +public class InternalDateHistogramTests extends InternalMultiBucketAggregationTestCase { + + private boolean keyed; + private DocValueFormat format; + + @Before + public void init() { + keyed = randomBoolean(); + format = randomNumericDocValueFormat(); + } @Override - protected InternalDateHistogram createTestInstance(String name, List pipelineAggregators, - Map metaData) { - - boolean keyed = randomBoolean(); - DocValueFormat format = DocValueFormat.RAW; + protected InternalDateHistogram createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { int nbBuckets = randomInt(10); List buckets = new ArrayList<>(nbBuckets); long startingDate = System.currentTimeMillis(); @@ -54,7 +62,7 @@ public class InternalDateHistogramTests extends InternalAggregationTestCase instanceReader() { return InternalDateHistogram::new; } + + @Override + protected Class implementationClass() { + return ParsedDateHistogram.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java index 093496738fd..bfcfe78ff7d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java @@ -24,30 +24,42 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.junit.Before; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; -public class InternalHistogramTests extends InternalAggregationTestCase { +public class InternalHistogramTests extends InternalMultiBucketAggregationTestCase { + + private boolean keyed; + private DocValueFormat format; + + @Before + public void init() { + keyed = randomBoolean(); + format = randomNumericDocValueFormat(); + } @Override - protected InternalHistogram createTestInstance(String name, List pipelineAggregators, - Map metaData) { - final boolean keyed = randomBoolean(); - final DocValueFormat format = DocValueFormat.RAW; + protected InternalHistogram createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { final int base = randomInt(50) - 30; final int numBuckets = randomInt(10); final int interval = randomIntBetween(1, 3); List buckets = new ArrayList<>(); for (int i = 0; i < numBuckets; ++i) { final int docCount = TestUtil.nextInt(random(), 1, 50); - buckets.add(new InternalHistogram.Bucket(base + i * interval, docCount, keyed, format, InternalAggregations.EMPTY)); + buckets.add(new InternalHistogram.Bucket(base + i * interval, docCount, keyed, format, aggregations)); } - return new InternalHistogram(name, buckets, (InternalOrder) InternalHistogram.Order.KEY_ASC, - 1, null, format, keyed, pipelineAggregators, metaData); + InternalOrder order = (InternalOrder) randomFrom(InternalHistogram.Order.KEY_ASC, InternalHistogram.Order.KEY_DESC); + return new InternalHistogram(name, buckets, order, 1, null, format, keyed, pipelineAggregators, metaData); } @Override @@ -72,4 +84,8 @@ public class InternalHistogramTests extends InternalAggregationTestCase implementationClass() { + return ParsedHistogram.class; + } } From c5bdbecc649d1af8ef2d08dbd66d36882802b5a4 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 9 May 2017 21:08:10 +0200 Subject: [PATCH 35/62] [TEST] Add test for Aggregations#fromXContent (#24524) AggregationsTests#testFromXContent verifies that parsing of aggregations works by combining multiple aggs at the same level, and also adding sub-aggregations to multi bucket and single bucket aggs, up to a maximum depth of 5. --- .../search/aggregations/Aggregations.java | 1 - .../aggregations/AggregationsTests.java | 156 ++++++++++++++++++ ...nternalMultiBucketAggregationTestCase.java | 32 ++-- ...ternalSingleBucketAggregationTestCase.java | 28 +++- .../histogram/InternalDateHistogramTests.java | 6 +- .../histogram/InternalHistogramTests.java | 7 +- .../metrics/InternalMaxTests.java | 2 +- .../cardinality/InternalCardinalityTests.java | 22 +-- .../geocentroid/InternalGeoCentroidTests.java | 2 +- .../AbstractPercentilesTestCase.java | 6 +- 10 files changed, 218 insertions(+), 44 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java index c7f84fe6577..465cef30877 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java @@ -128,7 +128,6 @@ public class Aggregations implements Iterable, ToXContent { return builder; } - //TODO add tests for this method public static Aggregations fromXContent(XContentParser parser) throws IOException { final List aggregations = new ArrayList<>(); XContentParser.Token token; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java new file mode 100644 index 00000000000..84d11db3dbe --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -0,0 +1,156 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests; +import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests; +import org.elasticsearch.search.aggregations.metrics.InternalMaxTests; +import org.elasticsearch.search.aggregations.metrics.InternalStatsBucketTests; +import org.elasticsearch.search.aggregations.metrics.InternalStatsTests; +import org.elasticsearch.search.aggregations.metrics.avg.InternalAvgTests; +import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinalityTests; +import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBoundsTests; +import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroidTests; +import org.elasticsearch.search.aggregations.metrics.min.InternalMinTests; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentilesRanksTests; +import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentilesTests; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesRanksTests; +import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesTests; +import org.elasticsearch.search.aggregations.metrics.sum.InternalSumTests; +import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCountTests; +import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValueTests; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.InternalPercentilesBucketTests; +import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.InternalExtendedStatsBucketTests; +import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivativeTests; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.singletonMap; + +/** + * This class tests that aggregations parsing works properly. It checks that we can parse + * different aggregations and adds sub-aggregations where applicable. + * + */ +public class AggregationsTests extends ESTestCase { + + private static final List aggsTests = getAggsTests(); + + private static List getAggsTests() { + List aggsTests = new ArrayList<>(); + aggsTests.add(new InternalCardinalityTests()); + aggsTests.add(new InternalTDigestPercentilesTests()); + aggsTests.add(new InternalTDigestPercentilesRanksTests()); + aggsTests.add(new InternalHDRPercentilesTests()); + aggsTests.add(new InternalHDRPercentilesRanksTests()); + aggsTests.add(new InternalPercentilesBucketTests()); + aggsTests.add(new InternalMinTests()); + aggsTests.add(new InternalMaxTests()); + aggsTests.add(new InternalAvgTests()); + aggsTests.add(new InternalSumTests()); + aggsTests.add(new InternalValueCountTests()); + aggsTests.add(new InternalSimpleValueTests()); + aggsTests.add(new InternalDerivativeTests()); + aggsTests.add(new InternalBucketMetricValueTests()); + aggsTests.add(new InternalStatsTests()); + aggsTests.add(new InternalStatsBucketTests()); + aggsTests.add(new InternalExtendedStatsTests()); + aggsTests.add(new InternalExtendedStatsBucketTests()); + aggsTests.add(new InternalGeoBoundsTests()); + aggsTests.add(new InternalGeoCentroidTests()); + aggsTests.add(new InternalHistogramTests()); + aggsTests.add(new InternalDateHistogramTests()); + return Collections.unmodifiableList(aggsTests); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(InternalAggregationTestCase.getNamedXContents()); + } + + @Before + public void init() throws Exception { + for (InternalAggregationTestCase aggsTest : aggsTests) { + aggsTest.setUp(); + } + } + + @After + public void cleanUp() throws Exception { + for (InternalAggregationTestCase aggsTest : aggsTests) { + aggsTest.tearDown(); + } + } + + public void testFromXContent() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); + Aggregations aggregations = createTestInstance(); + BytesReference originalBytes = toShuffledXContent(aggregations, xContentType, params, randomBoolean()); + try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertEquals(Aggregations.AGGREGATIONS_FIELD, parser.currentName()); + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + Aggregations parsedAggregations = Aggregations.fromXContent(parser); + BytesReference parsedBytes = XContentHelper.toXContent(parsedAggregations, xContentType, randomBoolean()); + ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); + } + } + + private static InternalAggregations createTestInstance() { + return createTestInstance(0, 5); + } + + private static InternalAggregations createTestInstance(final int currentDepth, final int maxDepth) { + int numAggs = randomIntBetween(1, 5); + List aggs = new ArrayList<>(numAggs); + for (int i = 0; i < numAggs; i++) { + InternalAggregationTestCase testCase = randomFrom(aggsTests); + if (testCase instanceof InternalMultiBucketAggregationTestCase && currentDepth < maxDepth) { + InternalMultiBucketAggregationTestCase multiBucketAggTestCase = (InternalMultiBucketAggregationTestCase) testCase; + multiBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(currentDepth + 1, maxDepth); + } + if (testCase instanceof InternalSingleBucketAggregationTestCase && currentDepth < maxDepth) { + InternalSingleBucketAggregationTestCase singleBucketAggTestCase = (InternalSingleBucketAggregationTestCase) testCase; + singleBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(currentDepth + 1, maxDepth); + } + aggs.add(testCase.createTestInstance()); + } + return new InternalAggregations(aggs); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java index ae293d5ac31..64eee02dcb6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java @@ -21,36 +21,42 @@ package org.elasticsearch.search.aggregations; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.junit.Before; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; +import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; public abstract class InternalMultiBucketAggregationTestCase extends InternalAggregationTestCase { - private boolean hasSubAggregations; + Supplier subAggregationsSupplier; - @Before - public void initHasSubAggregations() { - hasSubAggregations = randomBoolean(); + @Override + public void setUp() throws Exception { + super.setUp(); + if (randomBoolean()) { + subAggregationsSupplier = () -> InternalAggregations.EMPTY; + } else { + subAggregationsSupplier = () -> { + final int numAggregations = randomIntBetween(1, 3); + List aggs = new ArrayList<>(); + for (int i = 0; i < numAggregations; i++) { + aggs.add(createTestInstance(randomAlphaOfLength(5), emptyList(), emptyMap(), InternalAggregations.EMPTY)); + } + return new InternalAggregations(aggs); + }; + } } @Override protected final T createTestInstance(String name, List pipelineAggregators, Map metaData) { - List internal = new ArrayList<>(); - if (hasSubAggregations) { - final int numAggregations = randomIntBetween(1, 3); - for (int i = 0; i pipelineAggregators, diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java index 5d2e8affe78..a31b28e7fdb 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java @@ -29,31 +29,43 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; public abstract class InternalSingleBucketAggregationTestCase extends InternalAggregationTestCase { + private final boolean hasInternalMax = randomBoolean(); private final boolean hasInternalMin = randomBoolean(); + public Supplier subAggregationsSupplier; + + @Override + public void setUp() throws Exception { + super.setUp(); + subAggregationsSupplier = () -> { + List aggs = new ArrayList<>(); + if (hasInternalMax) { + aggs.add(new InternalMax("max", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap())); + } + if (hasInternalMin) { + aggs.add(new InternalMin("min", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap())); + } + return new InternalAggregations(aggs); + }; + } + protected abstract T createTestInstance(String name, long docCount, InternalAggregations aggregations, List pipelineAggregators, Map metaData); protected abstract void extraAssertReduced(T reduced, List inputs); @Override protected final T createTestInstance(String name, List pipelineAggregators, Map metaData) { - List internal = new ArrayList<>(); - if (hasInternalMax) { - internal.add(new InternalMax("max", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap())); - } - if (hasInternalMin) { - internal.add(new InternalMin("min", randomDouble(), randomNumericDocValueFormat(), emptyList(), emptyMap())); - } // we shouldn't use the full long range here since we sum doc count on reduce, and don't want to overflow the long range there long docCount = between(0, Integer.MAX_VALUE); - return createTestInstance(name, docCount, new InternalAggregations(internal), pipelineAggregators, metaData); + return createTestInstance(name, docCount, subAggregationsSupplier.get(), pipelineAggregators, metaData); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java index c4410713ac8..0b71d138a57 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestC import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.joda.time.DateTime; -import org.junit.Before; import java.util.ArrayList; import java.util.List; @@ -42,8 +41,9 @@ public class InternalDateHistogramTests extends InternalMultiBucketAggregationTe private boolean keyed; private DocValueFormat format; - @Before - public void init() { + @Override + public void setUp() throws Exception { + super.setUp(); keyed = randomBoolean(); format = randomNumericDocValueFormat(); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java index bfcfe78ff7d..4f0fea87d63 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java @@ -22,12 +22,10 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.junit.Before; import java.util.ArrayList; import java.util.List; @@ -39,8 +37,9 @@ public class InternalHistogramTests extends InternalMultiBucketAggregationTestCa private boolean keyed; private DocValueFormat format; - @Before - public void init() { + @Override + public void setUp() throws Exception{ + super.setUp(); keyed = randomBoolean(); format = randomNumericDocValueFormat(); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java index 53483be55c0..1d3a8b45a11 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java @@ -34,7 +34,7 @@ public class InternalMaxTests extends InternalAggregationTestCase { @Override protected InternalMax createTestInstance(String name, List pipelineAggregators, Map metaData) { - double value = frequently() ? randomDouble() : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY }); + double value = frequently() ? randomDouble() : randomFrom(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); DocValueFormat formatter = randomNumericDocValueFormat(); return new InternalMax(name, value, formatter, pipelineAggregators, metaData); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java index efa5438ae3c..60f0179d9a1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.InternalAggregationTestCase; import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.After; -import org.junit.Before; import java.util.ArrayList; import java.util.List; @@ -38,12 +37,22 @@ public class InternalCardinalityTests extends InternalAggregationTestCase algos; private static int p; - @Before - public void setup() { + @Override + public void setUp() throws Exception { + super.setUp(); algos = new ArrayList<>(); p = randomIntBetween(HyperLogLogPlusPlus.MIN_PRECISION, HyperLogLogPlusPlus.MAX_PRECISION); } + @After //we force @After to have it run before ESTestCase#after otherwise it fails + @Override + public void tearDown() throws Exception { + super.tearDown(); + Releasables.close(algos); + algos.clear(); + algos = null; + } + @Override protected InternalCardinality createTestInstance(String name, List pipelineAggregators, Map metaData) { @@ -82,11 +91,4 @@ public class InternalCardinalityTests extends InternalAggregationTestCase Date: Tue, 9 May 2017 21:23:00 +0200 Subject: [PATCH 36/62] [TEST] add test to verify that no aggs have been forgotten in AggregationsTests --- .../search/aggregations/AggregationsTests.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 84d11db3dbe..f986f614fa1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -58,6 +58,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static java.util.Collections.singletonMap; @@ -116,6 +118,14 @@ public class AggregationsTests extends ESTestCase { } } + public void testAllAggsAreBeingTested() { + assertEquals(InternalAggregationTestCase.getNamedXContents().size(), aggsTests.size()); + Set aggs = aggsTests.stream().map((testCase) -> testCase.createTestInstance().getType()).collect(Collectors.toSet()); + for (NamedXContentRegistry.Entry entry : InternalAggregationTestCase.getNamedXContents()) { + assertTrue(aggs.contains(entry.name.getPreferredName())); + } + } + public void testFromXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); From 3c66ac06aee2401db8f6554a7a6a0a6282b7f187 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 10 May 2017 10:17:25 +0200 Subject: [PATCH 37/62] Add parsing for String/Long/Double Terms aggregations (#24521) --- .../search/aggregations/Aggregations.java | 2 +- .../ParsedMultiBucketAggregation.java | 42 +++-- .../bucket/histogram/ParsedDateHistogram.java | 29 ++-- .../bucket/histogram/ParsedHistogram.java | 23 ++- .../bucket/terms/ParsedDoubleTerms.java | 85 ++++++++++ .../bucket/terms/ParsedLongTerms.java | 85 ++++++++++ .../bucket/terms/ParsedStringTerms.java | 85 ++++++++++ .../bucket/terms/ParsedTerms.java | 151 ++++++++++++++++++ .../percentiles/ParsedPercentiles.java | 3 +- .../InternalAggregationTestCase.java | 19 ++- .../bucket/terms/DoubleTermsTests.java | 23 +-- .../bucket/terms/InternalTermsTestCase.java | 32 +++- .../bucket/terms/LongTermsTests.java | 22 +-- .../bucket/terms/StringTermsTests.java | 20 ++- 14 files changed, 542 insertions(+), 79 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java index 465cef30877..05521e48831 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregations.java @@ -47,7 +47,7 @@ public class Aggregations implements Iterable, ToXContent { protected Aggregations() { } - protected Aggregations(List aggregations) { + public Aggregations(List aggregations) { this.aggregations = aggregations; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java index b2823669a75..02e29a1746f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -33,10 +34,11 @@ import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -public abstract class ParsedMultiBucketAggregation extends ParsedAggregation implements MultiBucketsAggregation { +public abstract class ParsedMultiBucketAggregation + extends ParsedAggregation implements MultiBucketsAggregation { - protected final List> buckets = new ArrayList<>(); - protected boolean keyed; + protected final List buckets = new ArrayList<>(); + protected boolean keyed = false; @Override protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { @@ -45,7 +47,7 @@ public abstract class ParsedMultiBucketAggregation extends ParsedAggregation imp } else { builder.startArray(CommonFields.BUCKETS.getPreferredName()); } - for (ParsedBucket bucket : buckets) { + for (B bucket : buckets) { bucket.toXContent(builder, params); } if (keyed) { @@ -57,8 +59,8 @@ public abstract class ParsedMultiBucketAggregation extends ParsedAggregation imp } protected static void declareMultiBucketAggregationFields(final ObjectParser objectParser, - final CheckedFunction, IOException> bucketParser, - final CheckedFunction, IOException> keyedBucketParser) { + final CheckedFunction bucketParser, + final CheckedFunction keyedBucketParser) { declareAggregationFields(objectParser); objectParser.declareField((parser, aggregation, context) -> { XContentParser.Token token = parser.currentToken(); @@ -76,23 +78,13 @@ public abstract class ParsedMultiBucketAggregation extends ParsedAggregation imp }, CommonFields.BUCKETS, ObjectParser.ValueType.OBJECT_ARRAY); } - public static class ParsedBucket implements MultiBucketsAggregation.Bucket { + public static abstract class ParsedBucket implements MultiBucketsAggregation.Bucket { private Aggregations aggregations; - private T key; private String keyAsString; private long docCount; private boolean keyed; - protected void setKey(T key) { - this.key = key; - } - - @Override - public Object getKey() { - return key; - } - protected void setKeyAsString(String keyAsString) { this.keyAsString = keyAsString; } @@ -137,17 +129,21 @@ public abstract class ParsedMultiBucketAggregation extends ParsedAggregation imp if (keyAsString != null) { builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); } - builder.field(CommonFields.KEY.getPreferredName(), key); + keyToXContent(builder); builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); aggregations.toXContentInternal(builder, params); builder.endObject(); return builder; } - protected static > B parseXContent(final XContentParser parser, - final boolean keyed, - final Supplier bucketSupplier, - final CheckedFunction keyParser) + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), getKey()); + } + + protected static B parseXContent(final XContentParser parser, + final boolean keyed, + final Supplier bucketSupplier, + final CheckedBiConsumer keyConsumer) throws IOException { final B bucket = bucketSupplier.get(); bucket.setKeyed(keyed); @@ -166,7 +162,7 @@ public abstract class ParsedMultiBucketAggregation extends ParsedAggregation imp if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { bucket.setKeyAsString(parser.text()); } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { - bucket.setKey(keyParser.apply(parser)); + keyConsumer.accept(parser, bucket); } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { bucket.setDocCount(parser.longValue()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java index 27ba2c029d2..9efc478aff1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java @@ -20,17 +20,16 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.List; -import java.util.stream.Collectors; -public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements Histogram { +public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements Histogram { @Override protected String getType() { @@ -39,7 +38,7 @@ public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements @Override public List getBuckets() { - return buckets.stream().map(bucket -> (Histogram.Bucket) bucket).collect(Collectors.toList()); + return buckets; } private static ObjectParser PARSER = @@ -56,11 +55,16 @@ public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements return aggregation; } - public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + + private Long key; @Override public Object getKey() { - return new DateTime(super.getKey(), DateTimeZone.UTC); + if (key != null) { + return new DateTime(key, DateTimeZone.UTC); + } + return null; } @Override @@ -68,13 +72,20 @@ public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements String keyAsString = super.getKeyAsString(); if (keyAsString != null) { return keyAsString; - } else { - return DocValueFormat.RAW.format((Long) super.getKey()); } + if (key != null) { + return Long.toString(key); + } + return null; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), key); } static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { - return parseXContent(parser, keyed, ParsedBucket::new, XContentParser::longValue); + return parseXContent(parser, keyed, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue()); } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java index 2b6730df2cc..eeec1cd81ba 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java @@ -21,14 +21,12 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import java.io.IOException; import java.util.List; -import java.util.stream.Collectors; -public class ParsedHistogram extends ParsedMultiBucketAggregation implements Histogram { +public class ParsedHistogram extends ParsedMultiBucketAggregation implements Histogram { @Override protected String getType() { @@ -37,7 +35,7 @@ public class ParsedHistogram extends ParsedMultiBucketAggregation implements His @Override public List getBuckets() { - return buckets.stream().map(bucket -> (Histogram.Bucket) bucket).collect(Collectors.toList()); + return buckets; } private static ObjectParser PARSER = @@ -54,20 +52,29 @@ public class ParsedHistogram extends ParsedMultiBucketAggregation implements His return aggregation; } - static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { + + private Double key; + + @Override + public Object getKey() { + return key; + } @Override public String getKeyAsString() { String keyAsString = super.getKeyAsString(); if (keyAsString != null) { return keyAsString; - } else { - return DocValueFormat.RAW.format((Double) getKey()); } + if (key != null) { + return Double.toString(key); + } + return null; } static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { - return parseXContent(parser, keyed, ParsedBucket::new, XContentParser::doubleValue); + return parseXContent(parser, keyed, ParsedBucket::new, (p, bucket) -> bucket.key = p.doubleValue()); } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java new file mode 100644 index 00000000000..3401cd021bd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedDoubleTerms.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedDoubleTerms extends ParsedTerms { + + @Override + protected String getType() { + return DoubleTerms.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedDoubleTerms.class.getSimpleName(), true, ParsedDoubleTerms::new); + static { + declareParsedTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedDoubleTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedDoubleTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedTerms.ParsedBucket { + + private Double key; + + @Override + public Object getKey() { + return key; + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + if (key != null) { + return Double.toString(key); + } + return null; + } + + public Number getKeyAsNumber() { + return key; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + builder.field(CommonFields.KEY.getPreferredName(), key); + if (super.getKeyAsString() != null) { + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); + } + return builder; + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.doubleValue()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java new file mode 100644 index 00000000000..6de25dfefe1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedLongTerms.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedLongTerms extends ParsedTerms { + + @Override + protected String getType() { + return LongTerms.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedLongTerms.class.getSimpleName(), true, ParsedLongTerms::new); + static { + declareParsedTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedLongTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedLongTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedTerms.ParsedBucket { + + private Long key; + + @Override + public Object getKey() { + return key; + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + if (key != null) { + return Long.toString(key); + } + return null; + } + + public Number getKeyAsNumber() { + return key; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + builder.field(CommonFields.KEY.getPreferredName(), key); + if (super.getKeyAsString() != null) { + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); + } + return builder; + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java new file mode 100644 index 00000000000..f258f7f847e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedStringTerms.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedStringTerms extends ParsedTerms { + + @Override + protected String getType() { + return StringTerms.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedStringTerms.class.getSimpleName(), true, ParsedStringTerms::new); + static { + declareParsedTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedStringTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedStringTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedTerms.ParsedBucket { + + private BytesRef key; + + @Override + public Object getKey() { + return getKeyAsString(); + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + if (key != null) { + return key.utf8ToString(); + } + return null; + } + + public Number getKeyAsNumber() { + if (key != null) { + return Double.parseDouble(key.utf8ToString()); + } + return null; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), getKey()); + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.utf8BytesOrNull()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java new file mode 100644 index 00000000000..6e24bd00383 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/ParsedTerms.java @@ -0,0 +1,151 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME; +import static org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.SUM_OF_OTHER_DOC_COUNTS; + +public abstract class ParsedTerms extends ParsedMultiBucketAggregation implements Terms { + + protected long docCountErrorUpperBound; + protected long sumOtherDocCount; + + @Override + public long getDocCountError() { + return docCountErrorUpperBound; + } + + @Override + public long getSumOfOtherDocCounts() { + return sumOtherDocCount; + } + + @Override + public List getBuckets() { + return buckets; + } + + @Override + public Terms.Bucket getBucketByKey(String term) { + for (Terms.Bucket bucket : getBuckets()) { + if (bucket.getKeyAsString().equals(term)) { + return bucket; + } + } + return null; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError()); + builder.field(SUM_OF_OTHER_DOC_COUNTS.getPreferredName(), getSumOfOtherDocCounts()); + builder.startArray(CommonFields.BUCKETS.getPreferredName()); + for (Terms.Bucket bucket : getBuckets()) { + bucket.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + static void declareParsedTermsFields(final ObjectParser objectParser, + final CheckedFunction bucketParser) { + declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply); + objectParser.declareLong((parsedTerms, value) -> parsedTerms.docCountErrorUpperBound = value , + DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME); + objectParser.declareLong((parsedTerms, value) -> parsedTerms.sumOtherDocCount = value, + SUM_OF_OTHER_DOC_COUNTS); + } + + public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Terms.Bucket { + + boolean showDocCountError = false; + protected long docCountError; + + @Override + public int compareTerm(Terms.Bucket other) { + throw new UnsupportedOperationException(); + } + + @Override + public long getDocCountError() { + return docCountError; + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + keyToXContent(builder); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); + if (showDocCountError) { + builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError()); + } + getAggregations().toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + + static B parseTermsBucketXContent(final XContentParser parser, final Supplier bucketSupplier, + final CheckedBiConsumer keyConsumer) + throws IOException { + + final B bucket = bucketSupplier.get(); + final List aggregations = new ArrayList<>(); + + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + keyConsumer.accept(parser, bucket); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } else if (DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName().equals(currentFieldName)) { + bucket.docCountError = parser.longValue(); + bucket.showDocCountError = true; + } + } else if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + bucket.setAggregations(new Aggregations(aggregations)); + return bucket; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java index a96fb2cd13c..48f3dccecef 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/ParsedPercentiles.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.ParsedAggregation; import java.io.IOException; @@ -60,7 +59,7 @@ public abstract class ParsedPercentiles extends ParsedAggregation implements Ite } Double value = getPercentile(percent); if (value != null) { - return DocValueFormat.RAW.format(value); + return Double.toString(value); } return null; } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java index 782e98e8885..a766dcbf5c8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.action.search.RestSearchAction; @@ -38,6 +39,12 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggre import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram; +import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; +import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; @@ -127,6 +134,9 @@ public abstract class InternalAggregationTestCase namedXContents.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); namedXContents.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); namedXContents.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); + namedXContents.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c)); + namedXContents.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c)); + namedXContents.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) @@ -259,12 +269,7 @@ public abstract class InternalAggregationTestCase assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); - String currentName = parser.currentName(); - int i = currentName.indexOf(InternalAggregation.TYPED_KEYS_DELIMITER); - String aggType = currentName.substring(0, i); - String aggName = currentName.substring(i + 1); - - parsedAggregation = parser.namedObject(Aggregation.class, aggType, aggName); + parsedAggregation = XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class); assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken()); assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); @@ -277,7 +282,7 @@ public abstract class InternalAggregationTestCase assertEquals(aggregation.getType(), ((ParsedAggregation) parsedAggregation).getType()); } - BytesReference parsedBytes = toXContent((ToXContent) parsedAggregation, xContentType, params, humanReadable); + BytesReference parsedBytes = toXContent(parsedAggregation, xContentType, params, humanReadable); assertToXContentEquivalent(originalBytes, parsedBytes, xContentType); return (P) parsedAggregation; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java index 757d5647a12..45531e27dde 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.ArrayList; @@ -33,17 +34,17 @@ import java.util.Set; public class DoubleTermsTests extends InternalTermsTestCase { @Override - protected InternalTerms createTestInstance( - String name, - List pipelineAggregators, - Map metaData) { + protected InternalTerms createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations, + boolean showTermDocCountError, + long docCountError) { Terms.Order order = Terms.Order.count(false); long minDocCount = 1; int requiredSize = 3; int shardSize = requiredSize + 2; - DocValueFormat format = DocValueFormat.RAW; - boolean showTermDocCountError = false; - long docCountError = -1; + DocValueFormat format = randomNumericDocValueFormat(); long otherDocCount = 0; List buckets = new ArrayList<>(); final int numBuckets = randomInt(shardSize); @@ -51,8 +52,7 @@ public class DoubleTermsTests extends InternalTermsTestCase { for (int i = 0; i < numBuckets; ++i) { double term = randomValueOtherThanMany(d -> terms.add(d) == false, random()::nextDouble); int docCount = randomIntBetween(1, 100); - buckets.add(new DoubleTerms.Bucket(term, docCount, InternalAggregations.EMPTY, - showTermDocCountError, docCountError, format)); + buckets.add(new DoubleTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format)); } return new DoubleTerms(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError); @@ -63,4 +63,9 @@ public class DoubleTermsTests extends InternalTermsTestCase { return DoubleTerms::new; } + @Override + protected Class implementationClass() { + return ParsedDoubleTerms.class; + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java index 03031633f74..cfd7a82da1a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTermsTestCase.java @@ -19,18 +19,44 @@ package org.elasticsearch.search.aggregations.bucket.terms; -import org.elasticsearch.search.aggregations.InternalAggregationTestCase; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.junit.Before; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Map.Entry; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -public abstract class InternalTermsTestCase extends InternalAggregationTestCase> { +public abstract class InternalTermsTestCase extends InternalMultiBucketAggregationTestCase> { + + private boolean showDocCount; + private long docCountError; + + @Before + public void init() { + showDocCount = randomBoolean(); + docCountError = showDocCount ? randomInt(1000) : -1; + } + + @Override + protected InternalTerms createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { + return createTestInstance(name, pipelineAggregators, metaData, aggregations, showDocCount, docCountError); + } + + protected abstract InternalTerms createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations, + boolean showTermDocCountError, + long docCountError); @Override protected InternalTerms createUnmappedInstance( diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java index ff95984bc32..cc97e4989a9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.ArrayList; @@ -33,17 +34,17 @@ import java.util.Set; public class LongTermsTests extends InternalTermsTestCase { @Override - protected InternalTerms createTestInstance( - String name, - List pipelineAggregators, - Map metaData) { + protected InternalTerms createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations, + boolean showTermDocCountError, + long docCountError) { Terms.Order order = Terms.Order.count(false); long minDocCount = 1; int requiredSize = 3; int shardSize = requiredSize + 2; - DocValueFormat format = DocValueFormat.RAW; - boolean showTermDocCountError = false; - long docCountError = -1; + DocValueFormat format = randomNumericDocValueFormat(); long otherDocCount = 0; List buckets = new ArrayList<>(); final int numBuckets = randomInt(shardSize); @@ -51,8 +52,7 @@ public class LongTermsTests extends InternalTermsTestCase { for (int i = 0; i < numBuckets; ++i) { long term = randomValueOtherThanMany(l -> terms.add(l) == false, random()::nextLong); int docCount = randomIntBetween(1, 100); - buckets.add(new LongTerms.Bucket(term, docCount, InternalAggregations.EMPTY, - showTermDocCountError, docCountError, format)); + buckets.add(new LongTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format)); } return new LongTerms(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError); @@ -63,4 +63,8 @@ public class LongTermsTests extends InternalTermsTestCase { return LongTerms::new; } + @Override + protected Class implementationClass() { + return ParsedLongTerms.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java index 64e814bd819..e909358be5e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.ArrayList; @@ -34,17 +35,17 @@ import java.util.Set; public class StringTermsTests extends InternalTermsTestCase { @Override - protected InternalTerms createTestInstance( - String name, - List pipelineAggregators, - Map metaData) { + protected InternalTerms createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations, + boolean showTermDocCountError, + long docCountError) { Terms.Order order = Terms.Order.count(false); long minDocCount = 1; int requiredSize = 3; int shardSize = requiredSize + 2; DocValueFormat format = DocValueFormat.RAW; - boolean showTermDocCountError = false; - long docCountError = -1; long otherDocCount = 0; List buckets = new ArrayList<>(); final int numBuckets = randomInt(shardSize); @@ -52,8 +53,7 @@ public class StringTermsTests extends InternalTermsTestCase { for (int i = 0; i < numBuckets; ++i) { BytesRef term = randomValueOtherThanMany(b -> terms.add(b) == false, () -> new BytesRef(randomAlphaOfLength(10))); int docCount = randomIntBetween(1, 100); - buckets.add(new StringTerms.Bucket(term, docCount, InternalAggregations.EMPTY, - showTermDocCountError, docCountError, format)); + buckets.add(new StringTerms.Bucket(term, docCount, aggregations, showTermDocCountError, docCountError, format)); } return new StringTerms(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format, shardSize, showTermDocCountError, otherDocCount, buckets, docCountError); @@ -64,4 +64,8 @@ public class StringTermsTests extends InternalTermsTestCase { return StringTerms::new; } + @Override + protected Class implementationClass() { + return ParsedStringTerms.class; + } } From 21784913873c6ff821e357b99cedb1eeba8cd599 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 10 May 2017 11:34:34 +0200 Subject: [PATCH 38/62] Fix checkstyle violation --- .../search/aggregations/ParsedMultiBucketAggregation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java index 02e29a1746f..52b5fad5379 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java @@ -78,7 +78,7 @@ public abstract class ParsedMultiBucketAggregation Date: Wed, 10 May 2017 11:37:20 +0200 Subject: [PATCH 39/62] [Test] Reference Terms aggregations in AggregationsTests --- .../search/aggregations/AggregationsTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index f986f614fa1..4ac6c62f30f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -29,6 +29,9 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests; +import org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsTests; +import org.elasticsearch.search.aggregations.bucket.terms.LongTermsTests; +import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests; import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests; import org.elasticsearch.search.aggregations.metrics.InternalMaxTests; import org.elasticsearch.search.aggregations.metrics.InternalStatsBucketTests; @@ -96,6 +99,9 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalGeoCentroidTests()); aggsTests.add(new InternalHistogramTests()); aggsTests.add(new InternalDateHistogramTests()); + aggsTests.add(new LongTermsTests()); + aggsTests.add(new DoubleTermsTests()); + aggsTests.add(new StringTermsTests()); return Collections.unmodifiableList(aggsTests); } From 3201e2271032e348d82c0725b381066e149a0e37 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 10 May 2017 14:05:43 +0200 Subject: [PATCH 40/62] Fix merging conflicts --- .../aggregations/InternalAggregation.java | 2 +- .../search/aggregations/ParsedAggregation.java | 2 +- .../bucket/histogram/ParsedDateHistogram.java | 2 +- .../bucket/histogram/ParsedHistogram.java | 2 +- .../bucket/sampler/InternalSampler.java | 2 +- .../significant/UnmappedSignificantTerms.java | 2 +- .../bucket/terms/ParsedDoubleTerms.java | 2 +- .../bucket/terms/ParsedLongTerms.java | 2 +- .../bucket/terms/ParsedStringTerms.java | 2 +- .../bucket/terms/UnmappedTerms.java | 2 +- .../aggregations/metrics/avg/ParsedAvg.java | 2 +- .../metrics/cardinality/ParsedCardinality.java | 2 +- .../metrics/geocentroid/ParsedGeoCentroid.java | 2 +- .../aggregations/metrics/max/ParsedMax.java | 2 +- .../aggregations/metrics/min/ParsedMin.java | 2 +- .../hdr/ParsedHDRPercentileRanks.java | 2 +- .../percentiles/hdr/ParsedHDRPercentiles.java | 2 +- .../tdigest/ParsedTDigestPercentileRanks.java | 2 +- .../tdigest/ParsedTDigestPercentiles.java | 2 +- .../metrics/stats/ParsedStats.java | 2 +- .../stats/extended/ParsedExtendedStats.java | 2 +- .../aggregations/metrics/sum/ParsedSum.java | 2 +- .../metrics/valuecount/ParsedValueCount.java | 2 +- .../pipeline/ParsedSimpleValue.java | 2 +- .../bucketmetrics/ParsedBucketMetricValue.java | 2 +- .../bucketmetrics/stats/ParsedStatsBucket.java | 2 +- .../extended/ParsedExtendedStatsBucket.java | 2 +- .../pipeline/derivative/ParsedDerivative.java | 2 +- .../search/aggregations/AggregationsTests.java | 1 + ...InternalMultiBucketAggregationTestCase.java | 1 + .../aggregations/ParsedAggregationTests.java | 4 ++-- .../metrics/avg/InternalAvgTests.java | 2 +- .../AbstractPercentilesTestCase.java | 2 +- .../InternalTDigestPercentilesRanksTests.java | 2 +- .../InternalBucketMetricValueTests.java | 2 +- .../test/InternalAggregationTestCase.java | 18 +++++++++++++++--- 36 files changed, 51 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java index 0618e5cb29d..c86d8afed5b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java @@ -174,7 +174,7 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, Na * the aggregation name in the response, so that it can later be used by REST clients * to determine the internal type of the aggregation. */ - protected String getType() { + public String getType() { return getWriteableName(); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index 6942b6aec5d..3ff91d8d36d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -63,7 +63,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { * to determine the internal type of the aggregation. */ //TODO it may make sense to move getType to the Aggregation interface given that we are duplicating it in both implementations - protected abstract String getType(); + public abstract String getType(); @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java index 9efc478aff1..ace0cb59907 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedDateHistogram.java @@ -32,7 +32,7 @@ import java.util.List; public class ParsedDateHistogram extends ParsedMultiBucketAggregation implements Histogram { @Override - protected String getType() { + public String getType() { return DateHistogramAggregationBuilder.NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java index eeec1cd81ba..6037c155886 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedHistogram.java @@ -29,7 +29,7 @@ import java.util.List; public class ParsedHistogram extends ParsedMultiBucketAggregation implements Histogram { @Override - protected String getType() { + public String getType() { return HistogramAggregationBuilder.NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java index 0c02f1ae935..3b5b42d59fd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java @@ -48,7 +48,7 @@ public class InternalSampler extends InternalSingleBucketAggregation implements } @Override - protected String getType() { + public String getType() { return "sampler"; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java index f9d8375af8c..e79b01abe4b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java @@ -76,7 +76,7 @@ public class UnmappedSignificantTerms extends InternalSignificantTerms private final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(getNamedXContents()); - static List getNamedXContents() { + public static List getNamedXContents() { Map> namedXContents = new HashMap<>(); namedXContents.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c)); namedXContents.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c)); @@ -184,7 +196,7 @@ public abstract class InternalAggregationTestCase protected abstract void assertReduced(T reduced, List inputs); @Override - protected final T createTestInstance() { + public final T createTestInstance() { return createTestInstance(randomAlphaOfLength(5)); } From 2e602d6641d009bcd541e9d33e41a599a9cc033b Mon Sep 17 00:00:00 2001 From: javanna Date: Wed, 10 May 2017 17:51:58 +0200 Subject: [PATCH 41/62] [TEST] reset subAggregationsSupplier in AggregationsTests to empty sub-aggs once the maxDepth is reached If we don't do this, although we don't set the subAggsSupplier again, we will reuse the one set in the previous run, hence we can end up in situations where we keep on generating sub aggregations till a StackOverflowError gets thrown. --- .../aggregations/AggregationsTests.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 99129529e2c..78df82c5503 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -150,21 +150,28 @@ public class AggregationsTests extends ESTestCase { } private static InternalAggregations createTestInstance() { - return createTestInstance(0, 5); + return createTestInstance(1, 0, 5); } - private static InternalAggregations createTestInstance(final int currentDepth, final int maxDepth) { - int numAggs = randomIntBetween(1, 5); + private static InternalAggregations createTestInstance(final int minNumAggs, final int currentDepth, final int maxDepth) { + int numAggs = randomIntBetween(minNumAggs, 4); List aggs = new ArrayList<>(numAggs); for (int i = 0; i < numAggs; i++) { InternalAggregationTestCase testCase = randomFrom(aggsTests); - if (testCase instanceof InternalMultiBucketAggregationTestCase && currentDepth < maxDepth) { + if (testCase instanceof InternalMultiBucketAggregationTestCase) { InternalMultiBucketAggregationTestCase multiBucketAggTestCase = (InternalMultiBucketAggregationTestCase) testCase; - multiBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(currentDepth + 1, maxDepth); - } - if (testCase instanceof InternalSingleBucketAggregationTestCase && currentDepth < maxDepth) { + if (currentDepth < maxDepth) { + multiBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(0, currentDepth + 1, maxDepth); + } else { + multiBucketAggTestCase.subAggregationsSupplier = () -> InternalAggregations.EMPTY; + } + } else if (testCase instanceof InternalSingleBucketAggregationTestCase) { InternalSingleBucketAggregationTestCase singleBucketAggTestCase = (InternalSingleBucketAggregationTestCase) testCase; - singleBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(currentDepth + 1, maxDepth); + if (currentDepth < maxDepth) { + singleBucketAggTestCase.subAggregationsSupplier = () -> createTestInstance(0, currentDepth + 1, maxDepth); + } else { + singleBucketAggTestCase.subAggregationsSupplier = () -> InternalAggregations.EMPTY; + } } aggs.add(testCase.createTestInstance()); } From c4fc8edc032b564b3f8379b475888ed3d38b6881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 11 May 2017 11:50:35 +0200 Subject: [PATCH 42/62] Add parsing for single bucket aggregations (#24564) This adds parsing to all implementations of SingleBucketAggregations. They are mostly similar, so they share the common base class `ParsedSingleBucketAggregation` and the shared base test `InternalSingleBucketAggregationTestCase`. --- .../aggregations/ParsedAggregation.java | 2 +- .../bucket/ParsedSingleBucketAggregation.java | 93 +++++++++++++++++++ .../bucket/children/ParsedChildren.java | 36 +++++++ .../bucket/filter/ParsedFilter.java | 36 +++++++ .../bucket/global/ParsedGlobal.java | 36 +++++++ .../bucket/missing/ParsedMissing.java | 36 +++++++ .../bucket/nested/ParsedNested.java | 36 +++++++ .../bucket/nested/ParsedReverseNested.java | 36 +++++++ .../bucket/sampler/InternalSampler.java | 2 +- .../bucket/sampler/ParsedSampler.java | 36 +++++++ .../aggregations/AggregationsTests.java | 14 +++ ...ternalSingleBucketAggregationTestCase.java | 51 +++++++++- .../children/InternalChildrenTests.java | 5 + .../bucket/filter/InternalFilterTests.java | 6 ++ .../bucket/global/InternalGlobalTests.java | 5 + .../bucket/missing/InternalMissingTests.java | 5 + .../bucket/nested/InternalNestedTests.java | 6 ++ .../nested/InternalReverseNestedTests.java | 6 ++ .../bucket/sampler/InternalSamplerTests.java | 6 ++ .../test/InternalAggregationTestCase.java | 23 ++++- 20 files changed, 471 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParsedChildren.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java index 3ff91d8d36d..97f27043124 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java @@ -41,7 +41,7 @@ public abstract class ParsedAggregation implements Aggregation, ToXContent { } private String name; - Map metadata; + protected Map metadata; @Override public final String getName() { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java new file mode 100644 index 00000000000..99d9bfa1955 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * A base class for all the single bucket aggregations. + */ +public abstract class ParsedSingleBucketAggregation extends ParsedAggregation implements SingleBucketAggregation { + + private long docCount; + protected Aggregations aggregations = new Aggregations(Collections.emptyList()); + + @Override + public long getDocCount() { + return docCount; + } + + protected void setDocCount(long docCount) { + this.docCount = docCount; + } + + @Override + public Aggregations getAggregations() { + return aggregations; + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + aggregations.toXContentInternal(builder, params); + return builder; + } + + protected static T parseXContent(final XContentParser parser, T aggregation, String name) + throws IOException { + aggregation.setName(name); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (token == XContentParser.Token.FIELD_NAME) { + token = parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); + + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + aggregation.setDocCount(parser.longValue()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (CommonFields.META.getPreferredName().equals(currentFieldName)) { + aggregation.metadata = parser.map(); + } else { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + } + aggregation.aggregations = new Aggregations(aggregations); + return aggregation; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParsedChildren.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParsedChildren.java new file mode 100644 index 00000000000..9ce6661923e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParsedChildren.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.children; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedChildren extends ParsedSingleBucketAggregation implements Children { + + @Override + public String getType() { + return ChildrenAggregationBuilder.NAME; + } + + public static ParsedChildren fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedChildren(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java new file mode 100644 index 00000000000..5f5cf104498 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/ParsedFilter.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.filter; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedFilter extends ParsedSingleBucketAggregation implements Filter { + + @Override + public String getType() { + return FilterAggregationBuilder.NAME; + } + + public static ParsedFilter fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedFilter(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java new file mode 100644 index 00000000000..062752805b1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/ParsedGlobal.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.global; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedGlobal extends ParsedSingleBucketAggregation implements Global { + + @Override + public String getType() { + return GlobalAggregationBuilder.NAME; + } + + public static ParsedGlobal fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedGlobal(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java new file mode 100644 index 00000000000..2897372df89 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/ParsedMissing.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.missing; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedMissing extends ParsedSingleBucketAggregation implements Missing { + + @Override + public String getType() { + return MissingAggregationBuilder.NAME; + } + + public static ParsedMissing fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedMissing(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java new file mode 100644 index 00000000000..f241675678c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedNested.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.nested; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedNested extends ParsedSingleBucketAggregation implements Nested { + + @Override + public String getType() { + return NestedAggregationBuilder.NAME; + } + + public static ParsedNested fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedNested(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java new file mode 100644 index 00000000000..dec15c3eded --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ParsedReverseNested.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.nested; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedReverseNested extends ParsedSingleBucketAggregation implements Nested { + + @Override + public String getType() { + return ReverseNestedAggregationBuilder.NAME; + } + + public static ParsedReverseNested fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedReverseNested(), name); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java index 3b5b42d59fd..5d7e19ccad5 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java @@ -49,7 +49,7 @@ public class InternalSampler extends InternalSingleBucketAggregation implements @Override public String getType() { - return "sampler"; + return NAME; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java new file mode 100644 index 00000000000..5e1c4d77b79 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.sampler; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; + +import java.io.IOException; + +public class ParsedSampler extends ParsedSingleBucketAggregation implements Sampler { + + @Override + public String getType() { + return InternalSampler.NAME; + } + + public static ParsedSampler fromXContent(XContentParser parser, final String name) throws IOException { + return parseXContent(parser, new ParsedSampler(), name); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 78df82c5503..a338c8ec9ac 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -27,8 +27,15 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.children.InternalChildrenTests; +import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; +import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests; +import org.elasticsearch.search.aggregations.bucket.missing.InternalMissingTests; +import org.elasticsearch.search.aggregations.bucket.nested.InternalNestedTests; +import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNestedTests; +import org.elasticsearch.search.aggregations.bucket.sampler.InternalSamplerTests; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.LongTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests; @@ -103,6 +110,13 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new LongTermsTests()); aggsTests.add(new DoubleTermsTests()); aggsTests.add(new StringTermsTests()); + aggsTests.add(new InternalMissingTests()); + aggsTests.add(new InternalNestedTests()); + aggsTests.add(new InternalReverseNestedTests()); + aggsTests.add(new InternalChildrenTests()); + aggsTests.add(new InternalGlobalTests()); + aggsTests.add(new InternalFilterTests()); + aggsTests.add(new InternalSamplerTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java index f84a364ab1d..cae34768ecd 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregationTestCase.java @@ -19,32 +19,45 @@ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.metrics.min.InternalMin; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.InternalAggregationTestCase; +import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; public abstract class InternalSingleBucketAggregationTestCase extends InternalAggregationTestCase { - private final boolean hasInternalMax = randomBoolean(); - private final boolean hasInternalMin = randomBoolean(); + private boolean hasInternalMax; + private boolean hasInternalMin; public Supplier subAggregationsSupplier; @Override public void setUp() throws Exception { super.setUp(); + hasInternalMax = randomBoolean(); + hasInternalMin = randomBoolean(); subAggregationsSupplier = () -> { List aggs = new ArrayList<>(); if (hasInternalMax) { @@ -89,4 +102,38 @@ public abstract class InternalSingleBucketAggregationTestCase expectedAggregations = new HashMap<>(); + int expectedNumberOfAggregations = 0; + for (Aggregation expectedAggregation : aggregations) { + // since we shuffle xContent, we cannot rely on the order of the original inner aggregations for comparison + assertTrue(expectedAggregation instanceof InternalAggregation); + expectedAggregations.put(expectedAggregation.getName(), expectedAggregation); + expectedNumberOfAggregations++; + } + int parsedNumberOfAggregations = 0; + for (Aggregation parsedAgg : parsed.getAggregations()) { + assertTrue(parsedAgg instanceof ParsedAggregation); + assertTrue(expectedAggregations.keySet().contains(parsedAgg.getName())); + Aggregation expectedInternalAggregation = expectedAggregations.get(parsedAgg.getName()); + final XContentType xContentType = randomFrom(XContentType.values()); + final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true")); + BytesReference expectedBytes = toXContent(expectedInternalAggregation, xContentType, params, false); + BytesReference actualBytes = toXContent(parsedAgg, xContentType, params, false); + assertToXContentEquivalent(expectedBytes, actualBytes, xContentType); + parsedNumberOfAggregations++; + } + assertEquals(expectedNumberOfAggregations, parsedNumberOfAggregations); + Class parsedClass = implementationClass(); + assertTrue(parsedClass != null && parsedClass.isInstance(parsedAggregation)); + } + + protected abstract Class implementationClass(); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildrenTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildrenTests.java index b248d5ed981..285837c6e47 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildrenTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildrenTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.children; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -44,4 +45,8 @@ public class InternalChildrenTests extends InternalSingleBucketAggregationTestCa return InternalChildren::new; } + @Override + protected Class implementationClass() { + return ParsedChildren.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java index 3e74b9c2187..8f888e13afe 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilterTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.filter; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -43,4 +44,9 @@ public class InternalFilterTests extends InternalSingleBucketAggregationTestCase protected Reader instanceReader() { return InternalFilter::new; } + + @Override + protected Class implementationClass() { + return ParsedFilter.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java index 9092c3e0280..392f88b4d4e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobalTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.global; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -44,4 +45,8 @@ public class InternalGlobalTests extends InternalSingleBucketAggregationTestCase return InternalGlobal::new; } + @Override + protected Class implementationClass() { + return ParsedGlobal.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java index f3e151721bf..75a28e87cef 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissingTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.missing; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -44,4 +45,8 @@ public class InternalMissingTests extends InternalSingleBucketAggregationTestCas return InternalMissing::new; } + @Override + protected Class implementationClass() { + return ParsedMissing.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java index 7b410723666..f6299ebf7bb 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNestedTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.nested; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -43,4 +44,9 @@ public class InternalNestedTests extends InternalSingleBucketAggregationTestCase protected Reader instanceReader() { return InternalNested::new; } + + @Override + protected Class implementationClass() { + return ParsedNested.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java index f918024733e..08940fcd3ae 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNestedTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.nested; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -43,4 +44,9 @@ public class InternalReverseNestedTests extends InternalSingleBucketAggregationT protected Reader instanceReader() { return InternalReverseNested::new; } + + @Override + protected Class implementationClass() { + return ParsedReverseNested.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java index 1c4fb6d2a65..06319080923 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.sampler; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.List; @@ -42,4 +43,9 @@ public class InternalSamplerTests extends InternalSingleBucketAggregationTestCas protected Writeable.Reader instanceReader() { return InternalSampler::new; } + + @Override + protected Class implementationClass() { + return ParsedSampler.class; + } } \ No newline at end of file diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 64b09a0245f..99cf71f65e4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -38,10 +38,24 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.children.ParsedChildren; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; +import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram; +import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; +import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested; +import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; +import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; @@ -139,6 +153,13 @@ public abstract class InternalAggregationTestCase namedXContents.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c)); namedXContents.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c)); namedXContents.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c)); + namedXContents.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c)); + namedXContents.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c)); + namedXContents.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c)); + namedXContents.put(ChildrenAggregationBuilder.NAME, (p, c) -> ParsedChildren.fromXContent(p, (String) c)); + namedXContents.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); + namedXContents.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); + namedXContents.put(InternalSampler.NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) @@ -248,7 +269,7 @@ public abstract class InternalAggregationTestCase } //norelease TODO make abstract - protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) { + protected void assertFromXContent(T aggregation, ParsedAggregation parsedAggregation) throws IOException { } @SuppressWarnings("unchecked") From 29a5694bb7ad9f5520be7299aec44f2d85269956 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 12 May 2017 15:44:39 +0200 Subject: [PATCH 43/62] Add parsing method to GeoHashGrid aggregation (#24589) --- .../bucket/geogrid/ParsedGeoHashGrid.java | 78 +++++++++++++++++++ .../aggregations/AggregationsTests.java | 2 + .../geogrid/InternalGeoHashGridTests.java | 25 ++++-- .../test/InternalAggregationTestCase.java | 4 + 4 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java new file mode 100644 index 00000000000..4551523e0fc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.List; + +public class ParsedGeoHashGrid extends ParsedMultiBucketAggregation implements GeoHashGrid { + + @Override + public String getType() { + return GeoGridAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedGeoHashGrid.class.getSimpleName(), true, ParsedGeoHashGrid::new); + static { + declareMultiBucketAggregationFields(PARSER, ParsedBucket::fromXContent, ParsedBucket::fromXContent); + } + + public static ParsedGeoHashGrid fromXContent(XContentParser parser, String name) throws IOException { + ParsedGeoHashGrid aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements GeoHashGrid.Bucket { + + private String geohashAsString; + + @Override + public GeoPoint getKey() { + return GeoPoint.fromGeohash(geohashAsString); + } + + @Override + public String getKeyAsString() { + return geohashAsString; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), geohashAsString); + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedBucket::new, (p, bucket) -> bucket.geohashAsString = p.textOrNull()); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index a338c8ec9ac..d741240e158 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregat import org.elasticsearch.search.aggregations.bucket.children.InternalChildrenTests; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests; import org.elasticsearch.search.aggregations.bucket.missing.InternalMissingTests; @@ -117,6 +118,7 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalGlobalTests()); aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); + aggsTests.add(new InternalGeoHashGridTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java index a27b47946f9..afa9defc78a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGridTests.java @@ -22,24 +22,30 @@ import org.apache.lucene.index.IndexWriter; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.test.InternalAggregationTestCase; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -public class InternalGeoHashGridTests extends InternalAggregationTestCase { +public class InternalGeoHashGridTests extends InternalMultiBucketAggregationTestCase { @Override - protected InternalGeoHashGrid createTestInstance(String name, List pipelineAggregators, - Map metaData) { - int size = randomIntBetween(1, 100); + protected InternalGeoHashGrid createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { + int size = randomIntBetween(1, 3); List buckets = new ArrayList<>(size); for (int i = 0; i < size; i++) { - long geoHashAsLong = GeoHashUtils.longEncode(randomInt(90), randomInt(90), 4); - buckets.add(new InternalGeoHashGrid.Bucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); + double latitude = randomDoubleBetween(-90.0, 90.0, false); + double longitude = randomDoubleBetween(-180.0, 180.0, false); + + long geoHashAsLong = GeoHashUtils.longEncode(longitude, latitude, 4); + buckets.add(new InternalGeoHashGrid.Bucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); } return new InternalGeoHashGrid(name, size, buckets, pipelineAggregators, metaData); } @@ -87,4 +93,9 @@ public class InternalGeoHashGridTests extends InternalAggregationTestCase implementationClass() { + return ParsedGeoHashGrid.class; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 99cf71f65e4..da7ac647c5c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.test; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -44,6 +45,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuil import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; @@ -160,6 +163,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); namedXContents.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); namedXContents.put(InternalSampler.NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); + namedXContents.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From b9d2ecc3eab93a2cf909e0c2a2540b668e3d207e Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 12 May 2017 16:52:47 +0200 Subject: [PATCH 44/62] Add parsing methods to Range aggregations (#24583) --- .../ParsedMultiBucketAggregation.java | 4 + .../bucket/range/ParsedRange.java | 193 ++++++++++++++++++ .../bucket/range/date/ParsedDateRange.java | 74 +++++++ .../range/geodistance/ParsedGeoDistance.java | 55 +++++ .../aggregations/AggregationsTests.java | 6 + ...nternalMultiBucketAggregationTestCase.java | 2 +- .../bucket/range/InternalRangeTestCase.java | 30 ++- .../bucket/range/InternalRangeTests.java | 26 ++- .../range/date/InternalDateRangeTests.java | 9 +- .../geodistance/InternalGeoDistanceTests.java | 10 +- .../test/InternalAggregationTestCase.java | 9 + 11 files changed, 407 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ParsedRange.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java index 52b5fad5379..df80ada8ddd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/ParsedMultiBucketAggregation.java @@ -107,6 +107,10 @@ public abstract class ParsedMultiBucketAggregation implements Range { + + @Override + public String getType() { + return RangeAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets; + } + + protected static void declareParsedRangeFields(final ObjectParser objectParser, + final CheckedFunction bucketParser, + final CheckedFunction keyedBucketParser) { + declareMultiBucketAggregationFields(objectParser, bucketParser::apply, keyedBucketParser::apply); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedRange.class.getSimpleName(), true, ParsedRange::new); + static { + declareParsedRangeFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedRange fromXContent(XContentParser parser, String name) throws IOException { + ParsedRange aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Range.Bucket { + + protected String key; + protected double from = Double.NEGATIVE_INFINITY; + protected String fromAsString; + protected double to = Double.POSITIVE_INFINITY; + protected String toAsString; + + @Override + public String getKey() { + return getKeyAsString(); + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + return key; + } + + @Override + public Object getFrom() { + return from; + } + + @Override + public String getFromAsString() { + if (fromAsString != null) { + return fromAsString; + } + return doubleAsString(from); + } + + @Override + public Object getTo() { + return to; + } + + @Override + public String getToAsString() { + if (toAsString != null) { + return toAsString; + } + return doubleAsString(to); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (isKeyed()) { + builder.startObject(key); + } else { + builder.startObject(); + builder.field(CommonFields.KEY.getPreferredName(), key); + } + if (Double.isInfinite(from) == false) { + builder.field(CommonFields.FROM.getPreferredName(), from); + if (fromAsString != null) { + builder.field(CommonFields.FROM_AS_STRING.getPreferredName(), fromAsString); + } + } + if (Double.isInfinite(to) == false) { + builder.field(CommonFields.TO.getPreferredName(), to); + if (toAsString != null) { + builder.field(CommonFields.TO_AS_STRING.getPreferredName(), toAsString); + } + } + builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); + getAggregations().toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + private static String doubleAsString(double d) { + return Double.isInfinite(d) ? null : Double.toString(d); + } + + protected static B parseRangeBucketXContent(final XContentParser parser, + final Supplier bucketSupplier, + final boolean keyed) throws IOException { + final B bucket = bucketSupplier.get(); + bucket.setKeyed(keyed); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (keyed) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + bucket.key = currentFieldName; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + } + + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + bucket.key = parser.text(); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } else if (CommonFields.FROM.getPreferredName().equals(currentFieldName)) { + bucket.from = parser.doubleValue(); + } else if (CommonFields.FROM_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.fromAsString = parser.text(); + } else if (CommonFields.TO.getPreferredName().equals(currentFieldName)) { + bucket.to = parser.doubleValue(); + } else if (CommonFields.TO_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.toAsString = parser.text(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + bucket.setAggregations(new Aggregations(aggregations)); + return bucket; + } + + static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException { + return parseRangeBucketXContent(parser, ParsedBucket::new, keyed); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java new file mode 100644 index 00000000000..b8f2f008cec --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/ParsedDateRange.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.range.date; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.range.ParsedRange; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.IOException; + +public class ParsedDateRange extends ParsedRange { + + @Override + public String getType() { + return DateRangeAggregationBuilder.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedDateRange.class.getSimpleName(), true, ParsedDateRange::new); + static { + declareParsedRangeFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedDateRange fromXContent(XContentParser parser, String name) throws IOException { + ParsedDateRange aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedRange.ParsedBucket { + + @Override + public Object getFrom() { + return doubleAsDateTime(from); + } + + @Override + public Object getTo() { + return doubleAsDateTime(to); + } + + private static DateTime doubleAsDateTime(Double d) { + if (d == null || Double.isInfinite(d)) { + return null; + } + return new DateTime(d.longValue(), DateTimeZone.UTC); + } + + static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException { + return parseRangeBucketXContent(parser, ParsedBucket::new, keyed); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java new file mode 100644 index 00000000000..a926499e924 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/ParsedGeoDistance.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.range.geodistance; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.range.ParsedRange; + +import java.io.IOException; + +public class ParsedGeoDistance extends ParsedRange { + + @Override + public String getType() { + return GeoDistanceAggregationBuilder.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedGeoDistance.class.getSimpleName(), true, ParsedGeoDistance::new); + static { + declareParsedRangeFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedGeoDistance fromXContent(XContentParser parser, String name) throws IOException { + ParsedGeoDistance aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedRange.ParsedBucket { + + static ParsedBucket fromXContent(final XContentParser parser, final boolean keyed) throws IOException { + return parseRangeBucketXContent(parser, ParsedBucket::new, keyed); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index d741240e158..4cfc87c171c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -37,6 +37,9 @@ import org.elasticsearch.search.aggregations.bucket.missing.InternalMissingTests import org.elasticsearch.search.aggregations.bucket.nested.InternalNestedTests; import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNestedTests; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSamplerTests; +import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTests; +import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRangeTests; +import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistanceTests; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.LongTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests; @@ -119,6 +122,9 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); aggsTests.add(new InternalGeoHashGridTests()); + aggsTests.add(new InternalRangeTests()); + aggsTests.add(new InternalDateRangeTests()); + aggsTests.add(new InternalGeoDistanceTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java index b16b7f49d7e..df977abbe81 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java @@ -117,7 +117,7 @@ public abstract class InternalMultiBucketAggregationTestCase extends InternalAggregationTestCase { +public abstract class InternalRangeTestCase extends InternalMultiBucketAggregationTestCase { private boolean keyed; @@ -40,13 +42,17 @@ public abstract class InternalRangeTestCase pipelineAggregators, Map metaData) { - return createTestInstance(name, pipelineAggregators, metaData, keyed); + protected T createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { + return createTestInstance(name, pipelineAggregators, metaData, aggregations, keyed); } protected abstract T createTestInstance(String name, List pipelineAggregators, Map metaData, + InternalAggregations aggregations, boolean keyed); @Override protected void assertReduced(T reduced, List inputs) { @@ -65,4 +71,20 @@ public abstract class InternalRangeTestCase { format = randomNumericDocValueFormat(); final int interval = randomFrom(1, 5, 10, 25, 50, 100); - final int numRanges = 1;//randomIntBetween(1, 10); + final int numRanges = randomIntBetween(1, 10); List> listOfRanges = new ArrayList<>(numRanges); for (int i = 0; i < numRanges; i++) { @@ -58,11 +59,23 @@ public class InternalRangeTests extends InternalRangeTestCase { listOfRanges.add(Tuple.tuple(0.0, max / 2)); listOfRanges.add(Tuple.tuple(max / 3, max / 3 * 2)); } + if (rarely()) { + listOfRanges.add(Tuple.tuple(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY)); + } + if (rarely()) { + listOfRanges.add(Tuple.tuple(Double.NEGATIVE_INFINITY, randomDouble())); + } + if (rarely()) { + listOfRanges.add(Tuple.tuple(randomDouble(), Double.POSITIVE_INFINITY)); + } ranges = Collections.unmodifiableList(listOfRanges); } @Override - protected InternalRange createTestInstance(String name, List pipelineAggregators, Map metaData, + protected InternalRange createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations, boolean keyed) { final List buckets = new ArrayList<>(); for (int i = 0; i < ranges.size(); ++i) { @@ -70,13 +83,18 @@ public class InternalRangeTests extends InternalRangeTestCase { int docCount = randomIntBetween(0, 1000); double from = range.v1(); double to = range.v2(); - buckets.add( new InternalRange.Bucket("range_" + i, from, to, docCount, InternalAggregations.EMPTY, keyed, format)); + buckets.add(new InternalRange.Bucket("range_" + i, from, to, docCount, aggregations, keyed, format)); } - return new InternalRange<>(name, buckets, format, keyed, pipelineAggregators, Collections.emptyMap()); + return new InternalRange<>(name, buckets, format, keyed, pipelineAggregators, metaData); } @Override protected Writeable.Reader instanceReader() { return InternalRange::new; } + + @Override + protected Class implementationClass() { + return ParsedRange.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java index bbfcdf7463a..318e5f6b5ad 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRangeTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.joda.time.DateTime; @@ -78,6 +79,7 @@ public class InternalDateRangeTests extends InternalRangeTestCase pipelineAggregators, Map metaData, + InternalAggregations aggregations, boolean keyed) { final List buckets = new ArrayList<>(); for (int i = 0; i < dateRanges.size(); ++i) { @@ -85,7 +87,7 @@ public class InternalDateRangeTests extends InternalRangeTestCase instanceReader() { return InternalDateRange::new; } + + @Override + protected Class implementationClass() { + return ParsedDateRange.class; + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java index 9dd2a7a67c7..25c0a9ae3e1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistanceTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket.range.geodistance; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTestCase; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.junit.Before; @@ -58,6 +59,7 @@ public class InternalGeoDistanceTests extends InternalRangeTestCase instanceReader() { return InternalGeoDistance::new; @@ -67,6 +69,7 @@ public class InternalGeoDistanceTests extends InternalRangeTestCase pipelineAggregators, Map metaData, + InternalAggregations aggregations, boolean keyed) { final List buckets = new ArrayList<>(); for (int i = 0; i < geoDistanceRanges.size(); ++i) { @@ -74,8 +77,13 @@ public class InternalGeoDistanceTests extends InternalRangeTestCase implementationClass() { + return ParsedGeoDistance.class; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index da7ac647c5c..66dd194e018 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -59,6 +59,12 @@ import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested; import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; +import org.elasticsearch.search.aggregations.bucket.range.ParsedRange; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.date.ParsedDateRange; +import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; @@ -164,6 +170,9 @@ public abstract class InternalAggregationTestCase namedXContents.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); namedXContents.put(InternalSampler.NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); namedXContents.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); + namedXContents.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); + namedXContents.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); + namedXContents.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); return namedXContents.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From 322a4c18dce4b1438990fe7f92d2d17b165188af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 12 May 2017 21:11:21 +0200 Subject: [PATCH 45/62] Fix msearch rest test using typed_keys --- .../resources/rest-api-spec/test/msearch/20_typed_keys.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml index 360405fd317..51a17b7c1e0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml @@ -107,8 +107,8 @@ setup: - {query: {match_all: {} }, size: 0, aggs: {test_sterms: {terms: {field: name}, aggs: {test_umsignificant_terms: {significant_terms: {field: surname} } } } } } - match: { responses.0.hits.total: 5 } - - match: { responses.0.aggregations.sampler#test_sampler.doc_count : 5 } - - match: { responses.0.aggregations.sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 } + - match: { responses.0.aggregations.mapped_sampler#test_sampler.doc_count : 5 } + - match: { responses.0.aggregations.mapped_sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 } - match: { responses.1.hits.total: 5 } - match: { responses.1.aggregations.sterms#test_umterms.doc_count_error_upper_bound : 0 } - match: { responses.2.hits.total: 5 } From bb59ee51b07e61a19d8b359cf365fb386c8c4b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 May 2017 11:49:47 +0200 Subject: [PATCH 46/62] Revert changing the InternalSampler type constant (#24667) --- .../aggregations/bucket/sampler/InternalSampler.java | 4 +++- .../aggregations/bucket/sampler/ParsedSampler.java | 2 +- .../rest-api-spec/test/msearch/20_typed_keys.yaml | 4 ++-- .../test/InternalAggregationTestCase.java | 11 +++++------ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java index 5d7e19ccad5..1a04133e82b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java @@ -29,6 +29,8 @@ import java.util.Map; public class InternalSampler extends InternalSingleBucketAggregation implements Sampler { public static final String NAME = "mapped_sampler"; + // InternalSampler and UnmappedSampler share the same parser name, so we use this when identifying the aggregation type + public static final String PARSER_NAME = "sampler"; InternalSampler(String name, long docCount, InternalAggregations subAggregations, List pipelineAggregators, Map metaData) { @@ -49,7 +51,7 @@ public class InternalSampler extends InternalSingleBucketAggregation implements @Override public String getType() { - return NAME; + return PARSER_NAME; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java index 5e1c4d77b79..3d5e946bead 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/ParsedSampler.java @@ -27,7 +27,7 @@ public class ParsedSampler extends ParsedSingleBucketAggregation implements Samp @Override public String getType() { - return InternalSampler.NAME; + return InternalSampler.PARSER_NAME; } public static ParsedSampler fromXContent(XContentParser parser, final String name) throws IOException { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml index 51a17b7c1e0..360405fd317 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yaml @@ -107,8 +107,8 @@ setup: - {query: {match_all: {} }, size: 0, aggs: {test_sterms: {terms: {field: name}, aggs: {test_umsignificant_terms: {significant_terms: {field: surname} } } } } } - match: { responses.0.hits.total: 5 } - - match: { responses.0.aggregations.mapped_sampler#test_sampler.doc_count : 5 } - - match: { responses.0.aggregations.mapped_sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 } + - match: { responses.0.aggregations.sampler#test_sampler.doc_count : 5 } + - match: { responses.0.aggregations.sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 } - match: { responses.1.hits.total: 5 } - match: { responses.1.aggregations.sterms#test_umterms.doc_count_error_upper_bound : 0 } - match: { responses.2.hits.total: 5 } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 66dd194e018..7646a84ee1d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -19,7 +19,6 @@ package org.elasticsearch.test; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -43,10 +42,10 @@ import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregation import org.elasticsearch.search.aggregations.bucket.children.ParsedChildren; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; -import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; @@ -57,14 +56,14 @@ import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuil import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested; import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; -import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; import org.elasticsearch.search.aggregations.bucket.range.ParsedRange; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.date.ParsedDateRange; import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance; +import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; +import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; @@ -168,7 +167,7 @@ public abstract class InternalAggregationTestCase namedXContents.put(ChildrenAggregationBuilder.NAME, (p, c) -> ParsedChildren.fromXContent(p, (String) c)); namedXContents.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); namedXContents.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); - namedXContents.put(InternalSampler.NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); + namedXContents.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); namedXContents.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); namedXContents.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); namedXContents.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); From 0b688a873354b6e2f9bf94bb458e03f99fe05fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 May 2017 15:06:01 +0200 Subject: [PATCH 47/62] Small improvement in InternalAggregationTestCase test setup after changes in master (#24675) --- .../search/aggregations/AggregationsTests.java | 8 +++----- .../join/aggregations/InternalChildrenTests.java | 11 +++++++---- .../test/InternalAggregationTestCase.java | 9 +++++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index a70c3c2d1b4..48fa42ad3ef 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -115,8 +115,6 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalMissingTests()); aggsTests.add(new InternalNestedTests()); aggsTests.add(new InternalReverseNestedTests()); - // TODO can we find a way to include the children aggregation in this test? - //aggsTests.add(new InternalChildrenTests()); aggsTests.add(new InternalGlobalTests()); aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); @@ -129,7 +127,7 @@ public class AggregationsTests extends ESTestCase { @Override protected NamedXContentRegistry xContentRegistry() { - return new NamedXContentRegistry(InternalAggregationTestCase.getNamedXContents()); + return new NamedXContentRegistry(InternalAggregationTestCase.getDefaultNamedXContents()); } @Before @@ -147,9 +145,9 @@ public class AggregationsTests extends ESTestCase { } public void testAllAggsAreBeingTested() { - assertEquals(InternalAggregationTestCase.getNamedXContents().size(), aggsTests.size()); + assertEquals(InternalAggregationTestCase.getDefaultNamedXContents().size(), aggsTests.size()); Set aggs = aggsTests.stream().map((testCase) -> testCase.createTestInstance().getType()).collect(Collectors.toSet()); - for (NamedXContentRegistry.Entry entry : InternalAggregationTestCase.getNamedXContents()) { + for (NamedXContentRegistry.Entry entry : InternalAggregationTestCase.getDefaultNamedXContents()) { assertTrue(aggs.contains(entry.name.getPreferredName())); } } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java index 089f0a19bd9..95cff14ef00 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java @@ -22,22 +22,25 @@ package org.elasticsearch.join.aggregations; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalSingleBucketAggregationTestCase; import org.elasticsearch.search.aggregations.bucket.ParsedSingleBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.junit.BeforeClass; +import java.util.ArrayList; import java.util.List; import java.util.Map; public class InternalChildrenTests extends InternalSingleBucketAggregationTestCase { - @BeforeClass - public static void init() { - namedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(ChildrenAggregationBuilder.NAME), + @Override + protected List getNamedXContents() { + List extendedNamedXContents = new ArrayList<>(super.getNamedXContents()); + extendedNamedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(ChildrenAggregationBuilder.NAME), (p, c) -> ParsedChildren.fromXContent(p, (String) c))); + return extendedNamedXContents ; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index d85f41fd20c..bd084bd5c08 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -130,7 +130,8 @@ public abstract class InternalAggregationTestCase new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables()); private final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(getNamedXContents()); - protected static final List namedXContents; + + private static final List namedXContents; static { Map> map = new HashMap<>(); map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c)); @@ -175,7 +176,11 @@ public abstract class InternalAggregationTestCase .collect(Collectors.toList()); } - public static List getNamedXContents() { + public static List getDefaultNamedXContents() { + return namedXContents; + } + + protected List getNamedXContents() { return namedXContents; } From 60505c9100e6a06a09b37cc99706957cea5abd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 May 2017 15:26:35 +0200 Subject: [PATCH 48/62] Add parsing for InternalFilters aggregation (#24648) This adds parsing to the InternalFilters aggregation. --- .../bucket/filters/ParsedFilters.java | 141 ++++++++++++++++++ .../aggregations/AggregationsTests.java | 2 + .../bucket/filters/InternalFiltersTests.java | 30 ++-- .../test/InternalAggregationTestCase.java | 3 + 4 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java new file mode 100644 index 00000000000..a77577a3ccc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/ParsedFilters.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.filters; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +public class ParsedFilters extends ParsedMultiBucketAggregation implements Filters { + + private Map bucketMap; + + @Override + public String getType() { + return FiltersAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets; + } + + @Override + public ParsedBucket getBucketByKey(String key) { + if (bucketMap == null) { + bucketMap = new HashMap<>(buckets.size()); + for (ParsedBucket bucket : buckets) { + bucketMap.put(bucket.getKey(), bucket); + } + } + return bucketMap.get(key); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedFilters.class.getSimpleName(), true, ParsedFilters::new); + static { + declareMultiBucketAggregationFields(PARSER, + parser -> ParsedBucket.fromXContent(parser, false), + parser -> ParsedBucket.fromXContent(parser, true)); + } + + public static ParsedFilters fromXContent(XContentParser parser, String name) throws IOException { + ParsedFilters aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + // in case this is not a keyed aggregation, we need to add numeric keys to the buckets + if (aggregation.keyed == false) { + int i = 0; + for (ParsedBucket bucket : aggregation.buckets) { + assert bucket.key == null; + bucket.key = String.valueOf(i); + i++; + } + } + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Filters.Bucket { + + private String key; + + @Override + public String getKey() { + return key; + } + + @Override + public String getKeyAsString() { + return key; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (isKeyed()) { + builder.startObject(key); + } else { + builder.startObject(); + } + builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); + getAggregations().toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + + static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { + final ParsedBucket bucket = new ParsedBucket(); + bucket.setKeyed(keyed); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (keyed) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + bucket.key = currentFieldName; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + } + + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + bucket.setAggregations(new Aggregations(aggregations)); + return bucket; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 48fa42ad3ef..871cf2756f2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; +import org.elasticsearch.search.aggregations.bucket.filters.InternalFiltersTests; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -122,6 +123,7 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); + aggsTests.add(new InternalFiltersTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java index b21f201e79b..03b5cb13d9c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFiltersTests.java @@ -21,8 +21,9 @@ package org.elasticsearch.search.aggregations.bucket.filters; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; import java.util.ArrayList; @@ -30,30 +31,36 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -public class InternalFiltersTests extends InternalAggregationTestCase { +public class InternalFiltersTests extends InternalMultiBucketAggregationTestCase { private boolean keyed; - private final List keys = new ArrayList<>(); + private List keys; @Override @Before public void setUp() throws Exception { super.setUp(); keyed = randomBoolean(); - int numKeys = randomIntBetween(1,10); - for (int i = 0; i < numKeys; i++) { - keys.add(randomAlphaOfLength(5)); + keys = new ArrayList<>(); + int numBuckets = randomIntBetween(1, 5); + for (int i = 0; i < numBuckets; i++) { + if (keyed) { + keys.add(randomAlphaOfLength(5)); + } else { + // this is what the FiltersAggregationBuilder ctor does when not providing KeyedFilter + keys.add(String.valueOf(i)); + } } - } @Override - protected InternalFilters createTestInstance(String name, List pipelineAggregators, Map metaData) { + protected InternalFilters createTestInstance(String name, List pipelineAggregators, Map metaData, + InternalAggregations aggregations) { final List buckets = new ArrayList<>(); for (int i = 0; i < keys.size(); ++i) { String key = keys.get(i); int docCount = randomIntBetween(0, 1000); - buckets.add( new InternalFilters.InternalBucket(key, docCount, InternalAggregations.EMPTY, keyed)); + buckets.add(new InternalFilters.InternalBucket(key, docCount, aggregations, keyed)); } return new InternalFilters(name, buckets, keyed, pipelineAggregators, metaData); } @@ -80,4 +87,9 @@ public class InternalFiltersTests extends InternalAggregationTestCase implementationClass() { + return ParsedFilters.class; + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index bd084bd5c08..7bd02f82c5e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -40,6 +40,8 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; +import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.filters.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; @@ -170,6 +172,7 @@ public abstract class InternalAggregationTestCase map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); + map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c)); namedXContents = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From ef7c2e62c3818e77997c84908f4308307ef184c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 16 May 2017 14:35:49 +0200 Subject: [PATCH 49/62] Add parsing for InternalAdjacencyMatrix aggregation (#24700) --- .../adjacency/ParsedAdjacencyMatrix.java | 88 +++++++++++++++++++ .../aggregations/AggregationsTests.java | 2 + .../InternalAdjacencyMatrixTests.java | 14 ++- .../test/InternalAggregationTestCase.java | 3 + 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java new file mode 100644 index 00000000000..1fb356d45c2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/ParsedAdjacencyMatrix.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.adjacency; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ParsedAdjacencyMatrix extends ParsedMultiBucketAggregation implements AdjacencyMatrix { + + private Map bucketMap; + + @Override + public String getType() { + return AdjacencyMatrixAggregationBuilder.NAME; + } + + @Override + public List getBuckets() { + return buckets; + } + + @Override + public ParsedBucket getBucketByKey(String key) { + if (bucketMap == null) { + bucketMap = new HashMap<>(buckets.size()); + for (ParsedBucket bucket : buckets) { + bucketMap.put(bucket.getKey(), bucket); + } + } + return bucketMap.get(key); + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedAdjacencyMatrix.class.getSimpleName(), true, ParsedAdjacencyMatrix::new); + static { + declareMultiBucketAggregationFields(PARSER, + parser -> ParsedBucket.fromXContent(parser), + parser -> ParsedBucket.fromXContent(parser)); + } + + public static ParsedAdjacencyMatrix fromXContent(XContentParser parser, String name) throws IOException { + ParsedAdjacencyMatrix aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements AdjacencyMatrix.Bucket { + + private String key; + + @Override + public String getKey() { + return key; + } + + @Override + public String getKeyAsString() { + return key; + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedBucket::new, (p, bucket) -> bucket.key = p.text()); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 871cf2756f2..fd4b5ae11c2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.search.aggregations.bucket.adjacency.InternalAdjacencyMatrixTests; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; import org.elasticsearch.search.aggregations.bucket.filters.InternalFiltersTests; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests; @@ -124,6 +125,7 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); aggsTests.add(new InternalFiltersTests()); + aggsTests.add(new InternalAdjacencyMatrixTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java index 53ae1db4544..58c3cc8a22c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java @@ -21,8 +21,9 @@ package org.elasticsearch.search.aggregations.bucket.adjacency; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; import java.util.ArrayList; @@ -30,7 +31,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -public class InternalAdjacencyMatrixTests extends InternalAggregationTestCase { +public class InternalAdjacencyMatrixTests extends InternalMultiBucketAggregationTestCase { private List keys; @@ -58,12 +59,12 @@ public class InternalAdjacencyMatrixTests extends InternalAggregationTestCase pipelineAggregators, - Map metaData) { + Map metaData, InternalAggregations aggregations) { final List buckets = new ArrayList<>(); for (int i = 0; i < keys.size(); ++i) { String key = keys.get(i); int docCount = randomIntBetween(0, 1000); - buckets.add(new InternalAdjacencyMatrix.InternalBucket(key, docCount, InternalAggregations.EMPTY)); + buckets.add(new InternalAdjacencyMatrix.InternalBucket(key, docCount, aggregations)); } return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators, metaData); } @@ -89,4 +90,9 @@ public class InternalAdjacencyMatrixTests extends InternalAggregationTestCase instanceReader() { return InternalAdjacencyMatrix::new; } + + @Override + protected Class implementationClass() { + return ParsedAdjacencyMatrix.class; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 7bd02f82c5e..99b0046a887 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -38,6 +38,8 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder; @@ -173,6 +175,7 @@ public abstract class InternalAggregationTestCase map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c)); + map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c)); namedXContents = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From d5fc520741943c83cfac58b4f3289f65af233c6d Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 16 May 2017 14:54:42 +0200 Subject: [PATCH 50/62] Add parsing to Significant Terms aggregations (#24682) Related to #23331 --- .../significant/InternalSignificantTerms.java | 4 +- .../ParsedSignificantLongTerms.java | 87 +++++++++ .../ParsedSignificantStringTerms.java | 84 +++++++++ .../significant/ParsedSignificantTerms.java | 166 ++++++++++++++++++ .../aggregations/AggregationsTests.java | 4 + ...nternalMultiBucketAggregationTestCase.java | 29 +-- .../InternalSignificantTermsTestCase.java | 40 ++++- .../SignificantLongTermsTests.java | 21 ++- .../SignificantStringTermsTests.java | 18 +- .../test/InternalAggregationTestCase.java | 6 + 10 files changed, 432 insertions(+), 27 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java index acb655b19b4..ac60829c4ca 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java @@ -44,8 +44,8 @@ import java.util.Objects; public abstract class InternalSignificantTerms, B extends InternalSignificantTerms.Bucket> extends InternalMultiBucketAggregation implements SignificantTerms, ToXContent { - private static final String SCORE = "score"; - private static final String BG_COUNT = "bg_count"; + public static final String SCORE = "score"; + public static final String BG_COUNT = "bg_count"; @SuppressWarnings("PMD.ConstructorCallsOverridableMethod") public abstract static class Bucket> extends InternalMultiBucketAggregation.InternalBucket diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java new file mode 100644 index 00000000000..79e05f4be9a --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantLongTerms.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedSignificantLongTerms extends ParsedSignificantTerms { + + @Override + public String getType() { + return SignificantLongTerms.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedSignificantLongTerms.class.getSimpleName(), true, ParsedSignificantLongTerms::new); + static { + declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedSignificantLongTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedSignificantLongTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket { + + private Long key; + + @Override + public Object getKey() { + return key; + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + return Long.toString(key); + } + + public Number getKeyAsNumber() { + return key; + } + + @Override + public int compareTerm(SignificantTerms.Bucket other) { + return key.compareTo(((ParsedBucket) other).key); + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + builder.field(CommonFields.KEY.getPreferredName(), key); + if (super.getKeyAsString() != null) { + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); + } + return builder; + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> bucket.key = p.longValue()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java new file mode 100644 index 00000000000..28d889b15c2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantStringTerms.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedSignificantStringTerms extends ParsedSignificantTerms { + + @Override + public String getType() { + return SignificantStringTerms.NAME; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedSignificantStringTerms.class.getSimpleName(), true, ParsedSignificantStringTerms::new); + static { + declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedSignificantStringTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedSignificantStringTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket { + + private BytesRef key; + + @Override + public Object getKey() { + return getKeyAsString(); + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + return key.utf8ToString(); + } + + public Number getKeyAsNumber() { + return Double.parseDouble(key.utf8ToString()); + } + + @Override + public int compareTerm(SignificantTerms.Bucket other) { + return key.compareTo(((ParsedBucket) other).key); + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), getKey()); + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> bucket.key = p.utf8BytesOrNull()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java new file mode 100644 index 00000000000..56be0aa6071 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/ParsedSignificantTerms.java @@ -0,0 +1,166 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public abstract class ParsedSignificantTerms extends ParsedMultiBucketAggregation + implements SignificantTerms { + + private Map bucketMap; + protected long subsetSize; + + protected long getSubsetSize() { + return subsetSize; + } + + @Override + public List getBuckets() { + return buckets; + } + + @Override + public SignificantTerms.Bucket getBucketByKey(String term) { + if (bucketMap == null) { + bucketMap = buckets.stream().collect(Collectors.toMap(SignificantTerms.Bucket::getKeyAsString, Function.identity())); + } + return bucketMap.get(term); + } + + @Override + public Iterator iterator() { + return buckets.stream().map(bucket -> (SignificantTerms.Bucket) bucket).collect(Collectors.toList()).iterator(); + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(CommonFields.DOC_COUNT.getPreferredName(), subsetSize); + builder.startArray(CommonFields.BUCKETS.getPreferredName()); + for (SignificantTerms.Bucket bucket : buckets) { + bucket.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + static void declareParsedSignificantTermsFields(final ObjectParser objectParser, + final CheckedFunction bucketParser) { + declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply); + objectParser.declareLong((parsedTerms, value) -> parsedTerms.subsetSize = value , CommonFields.DOC_COUNT); + } + + public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements SignificantTerms.Bucket { + + protected long subsetDf; + protected long supersetDf; + protected double score; + + @Override + public long getDocCount() { + return getSubsetDf(); + } + + @Override + public long getSubsetDf() { + return subsetDf; + } + + @Override + public long getSupersetDf() { + return supersetDf; + } + + @Override + public double getSignificanceScore() { + return score; + } + + @Override + public long getSupersetSize() { + throw new UnsupportedOperationException(); + } + + @Override + public long getSubsetSize() { + throw new UnsupportedOperationException(); + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + keyToXContent(builder); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); + builder.field(InternalSignificantTerms.SCORE, getSignificanceScore()); + builder.field(InternalSignificantTerms.BG_COUNT, getSupersetDf()); + getAggregations().toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException; + + static B parseSignificantTermsBucketXContent(final XContentParser parser, final B bucket, + final CheckedBiConsumer keyConsumer) throws IOException { + + final List aggregations = new ArrayList<>(); + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + keyConsumer.accept(parser, bucket); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + long value = parser.longValue(); + bucket.subsetDf = value; + bucket.setDocCount(value); + } else if (InternalSignificantTerms.SCORE.equals(currentFieldName)) { + bucket.score = parser.longValue(); + } else if (InternalSignificantTerms.BG_COUNT.equals(currentFieldName)) { + bucket.supersetDf = parser.longValue(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + aggregations.add(XContentParserUtils.parseTypedKeysObject(parser, Aggregation.TYPED_KEYS_DELIMITER, Aggregation.class)); + } + } + bucket.setAggregations(new Aggregations(aggregations)); + return bucket; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index fd4b5ae11c2..85deb604a66 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -40,6 +40,8 @@ import org.elasticsearch.search.aggregations.bucket.range.InternalRangeTests; import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRangeTests; import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistanceTests; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSamplerTests; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTermsTests; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.LongTermsTests; import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests; @@ -126,6 +128,8 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalGeoDistanceTests()); aggsTests.add(new InternalFiltersTests()); aggsTests.add(new InternalAdjacencyMatrixTests()); + aggsTests.add(new SignificantLongTermsTests()); + aggsTests.add(new SignificantStringTermsTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java index df977abbe81..38e8624eeb9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregationTestCase.java @@ -67,28 +67,22 @@ public abstract class InternalMultiBucketAggregationTestCase parsedClass = implementationClass(); - assertTrue(parsedClass != null && parsedClass.isInstance(actual)); - - assertTrue(expected instanceof InternalAggregation && actual instanceof ParsedAggregation); - assertEquals(expected.getName(), actual.getName()); - assertEquals(expected.getMetaData(), actual.getMetaData()); - assertEquals(((InternalAggregation) expected).getType(), ((ParsedAggregation) actual).getType()); + assertMultiBucketsAggregation(expectedMultiBucketsAggregation, actualMultiBucketsAggregation, checkOrder); List expectedBuckets = expectedMultiBucketsAggregation.getBuckets(); List actualBuckets = actualMultiBucketsAggregation.getBuckets(); @@ -117,6 +111,17 @@ public abstract class InternalMultiBucketAggregationTestCase parsedClass = implementationClass(); + assertNotNull("Parsed aggregation class must not be null", parsedClass); + assertTrue(parsedClass.isInstance(actual)); + + assertTrue(expected instanceof InternalAggregation); + assertEquals(expected.getName(), actual.getName()); + assertEquals(expected.getMetaData(), actual.getMetaData()); + assertEquals(((InternalAggregation) expected).getType(), ((ParsedAggregation) actual).getType()); + } + protected void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) { assertTrue(expected instanceof InternalMultiBucketAggregation.InternalBucket); assertTrue(actual instanceof ParsedMultiBucketAggregation.ParsedBucket); @@ -136,13 +141,13 @@ public abstract class InternalMultiBucketAggregationTestCase> { +public abstract class InternalSignificantTermsTestCase extends InternalMultiBucketAggregationTestCase> { @Override protected InternalSignificantTerms createUnmappedInstance(String name, @@ -61,6 +62,41 @@ public abstract class InternalSignificantTermsTestCase extends InternalAggregati } } + @Override + protected void assertMultiBucketsAggregation(MultiBucketsAggregation expected, MultiBucketsAggregation actual, boolean checkOrder) { + super.assertMultiBucketsAggregation(expected, actual, checkOrder); + + assertTrue(expected instanceof InternalSignificantTerms); + assertTrue(actual instanceof ParsedSignificantTerms); + + InternalSignificantTerms expectedSigTerms = (InternalSignificantTerms) expected; + ParsedSignificantTerms actualSigTerms = (ParsedSignificantTerms) actual; + assertEquals(expectedSigTerms.getSubsetSize(), actualSigTerms.getSubsetSize()); + + for (SignificantTerms.Bucket bucket : (SignificantTerms) expected) { + String key = bucket.getKeyAsString(); + assertBucket(expectedSigTerms.getBucketByKey(key), actualSigTerms.getBucketByKey(key), checkOrder); + } + } + + @Override + protected void assertBucket(MultiBucketsAggregation.Bucket expected, MultiBucketsAggregation.Bucket actual, boolean checkOrder) { + super.assertBucket(expected, actual, checkOrder); + + assertTrue(expected instanceof InternalSignificantTerms.Bucket); + assertTrue(actual instanceof ParsedSignificantTerms.ParsedBucket); + + SignificantTerms.Bucket expectedSigTerm = (SignificantTerms.Bucket) expected; + SignificantTerms.Bucket actualSigTerm = (SignificantTerms.Bucket) actual; + + assertEquals(expectedSigTerm.getSignificanceScore(), actualSigTerm.getSignificanceScore(), 0.0); + assertEquals(expectedSigTerm.getSubsetDf(), actualSigTerm.getSubsetDf()); + assertEquals(expectedSigTerm.getSupersetDf(), actualSigTerm.getSupersetDf()); + + expectThrows(UnsupportedOperationException.class, actualSigTerm::getSubsetSize); + expectThrows(UnsupportedOperationException.class, actualSigTerm::getSupersetSize); + } + private static Map toCounts(Stream buckets, Function fn) { return buckets.collect(Collectors.toMap(SignificantTerms.Bucket::getKey, fn, Long::sum)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java index 7e80cf61608..5b0e715c8a6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsTests.java @@ -21,6 +21,8 @@ package org.elasticsearch.search.aggregations.bucket.significant; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore; @@ -35,22 +37,24 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.elasticsearch.search.aggregations.InternalAggregations.EMPTY; - public class SignificantLongTermsTests extends InternalSignificantTermsTestCase { private SignificanceHeuristic significanceHeuristic; + private DocValueFormat format; + @Override @Before - public void setUpSignificanceHeuristic() { + public void setUp() throws Exception { + super.setUp(); significanceHeuristic = randomSignificanceHeuristic(); + format = randomNumericDocValueFormat(); } @Override protected InternalSignificantTerms createTestInstance(String name, List pipelineAggregators, - Map metaData) { - DocValueFormat format = DocValueFormat.RAW; + Map metaData, + InternalAggregations aggregations) { int requiredSize = randomIntBetween(1, 5); int shardSize = requiredSize + 2; final int numBuckets = randomInt(shardSize); @@ -70,7 +74,7 @@ public class SignificantLongTermsTests extends InternalSignificantTermsTestCase globalSubsetSize += subsetDf; globalSupersetSize += supersetSize; - buckets.add(new SignificantLongTerms.Bucket(subsetDf, subsetDf, supersetDf, supersetSize, term, EMPTY, format)); + buckets.add(new SignificantLongTerms.Bucket(subsetDf, subsetDf, supersetDf, supersetSize, term, aggregations, format)); } return new SignificantLongTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, globalSubsetSize, globalSupersetSize, significanceHeuristic, buckets); @@ -81,6 +85,11 @@ public class SignificantLongTermsTests extends InternalSignificantTermsTestCase return SignificantLongTerms::new; } + @Override + protected Class implementationClass() { + return ParsedSignificantLongTerms.class; + } + private static SignificanceHeuristic randomSignificanceHeuristic() { return randomFrom( new JLHScore(), diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java index 82cd21cdf38..07fe4c1ae82 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsTests.java @@ -22,6 +22,8 @@ package org.elasticsearch.search.aggregations.bucket.significant; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore; @@ -36,21 +38,22 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.elasticsearch.search.aggregations.InternalAggregations.EMPTY; - public class SignificantStringTermsTests extends InternalSignificantTermsTestCase { private SignificanceHeuristic significanceHeuristic; + @Override @Before - public void setUpSignificanceHeuristic() { + public void setUp() throws Exception { + super.setUp(); significanceHeuristic = randomSignificanceHeuristic(); } @Override protected InternalSignificantTerms createTestInstance(String name, List pipelineAggregators, - Map metaData) { + Map metaData, + InternalAggregations aggregations) { DocValueFormat format = DocValueFormat.RAW; int requiredSize = randomIntBetween(1, 5); int shardSize = requiredSize + 2; @@ -71,7 +74,7 @@ public class SignificantStringTermsTests extends InternalSignificantTermsTestCas globalSubsetSize += subsetDf; globalSupersetSize += supersetSize; - buckets.add(new SignificantStringTerms.Bucket(term, subsetDf, subsetDf, supersetDf, supersetSize, EMPTY, format)); + buckets.add(new SignificantStringTerms.Bucket(term, subsetDf, subsetDf, supersetDf, supersetSize, aggregations, format)); } return new SignificantStringTerms(name, requiredSize, 1L, pipelineAggregators, metaData, format, globalSubsetSize, globalSupersetSize, significanceHeuristic, buckets); @@ -82,6 +85,11 @@ public class SignificantStringTermsTests extends InternalSignificantTermsTestCas return SignificantStringTerms::new; } + @Override + protected Class implementationClass() { + return ParsedSignificantStringTerms.class; + } + private static SignificanceHeuristic randomSignificanceHeuristic() { return randomFrom( new JLHScore(), diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 99b0046a887..c256275b994 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -66,6 +66,10 @@ import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanc import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; +import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms; +import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; @@ -176,6 +180,8 @@ public abstract class InternalAggregationTestCase map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c)); map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c)); + map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c)); + map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c)); namedXContents = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From 9fc9db26fdc19ba840f0895af41e390a020a00c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 17 May 2017 18:55:57 +0200 Subject: [PATCH 51/62] Add parsing for InternalScriptedMetric aggregation (#24738) --- .../common/xcontent/ObjectParser.java | 3 +- .../scripted/InternalScriptedMetric.java | 2 +- .../scripted/ParsedScriptedMetric.java | 92 ++++++++++++++++ .../aggregations/AggregationsTests.java | 2 + .../scripted/InternalScriptedMetricTests.java | 101 +++++++++++++++++- .../test/InternalAggregationTestCase.java | 3 + 6 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index 5f9f7b7efa6..ed1d85b5a76 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -412,7 +412,8 @@ public final class ObjectParser extends AbstractObjectParser tokens; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java index 73975b25778..ea0fa4ce196 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java @@ -124,7 +124,7 @@ public class InternalScriptedMetric extends InternalAggregation implements Scrip @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - return builder.field("value", aggregation()); + return builder.field(CommonFields.VALUE.getPreferredName(), aggregation()); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java new file mode 100644 index 00000000000..949ff49cc77 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ParsedScriptedMetric.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.scripted; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class ParsedScriptedMetric extends ParsedAggregation implements ScriptedMetric { + private List aggregation; + + @Override + public String getType() { + return ScriptedMetricAggregationBuilder.NAME; + } + + @Override + public Object aggregation() { + assert aggregation.size() == 1; // see InternalScriptedMetric#aggregations() for why we can assume this + return aggregation.get(0); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + return builder.field(CommonFields.VALUE.getPreferredName(), aggregation()); + } + + private static final ObjectParser PARSER = new ObjectParser<>(ParsedScriptedMetric.class.getSimpleName(), true, + ParsedScriptedMetric::new); + + static { + declareAggregationFields(PARSER); + PARSER.declareField((agg, value) -> agg.aggregation = Collections.singletonList(value), + ParsedScriptedMetric::parseValue, CommonFields.VALUE, ValueType.VALUE_OBJECT_ARRAY); + } + + private static Object parseValue(XContentParser parser) throws IOException { + Token token = parser.currentToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token.isValue()) { + if (token == XContentParser.Token.VALUE_STRING) { + //binary values will be parsed back and returned as base64 strings when reading from json and yaml + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + //binary values will be parsed back and returned as BytesArray when reading from cbor and smile + value = new BytesArray(parser.binaryValue()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.map(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.list(); + } + return value; + } + + public static ParsedScriptedMetric fromXContent(XContentParser parser, final String name) { + ParsedScriptedMetric aggregation = PARSER.apply(parser, null); + aggregation.setName(name); + return aggregation; + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 85deb604a66..d47f9357f82 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDR import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentilesTests; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesRanksTests; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentilesTests; +import org.elasticsearch.search.aggregations.metrics.scripted.InternalScriptedMetricTests; import org.elasticsearch.search.aggregations.metrics.sum.InternalSumTests; import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCountTests; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests; @@ -130,6 +131,7 @@ public class AggregationsTests extends ESTestCase { aggsTests.add(new InternalAdjacencyMatrixTests()); aggsTests.add(new SignificantLongTermsTests()); aggsTests.add(new SignificantStringTermsTests()); + aggsTests.add(new InternalScriptedMetricTests()); return Collections.unmodifiableList(aggsTests); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index 75975d5a39f..f1fb42a4903 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics.scripted; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -30,20 +31,46 @@ import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.InternalAggregationTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class InternalScriptedMetricTests extends InternalAggregationTestCase { private static final String REDUCE_SCRIPT_NAME = "reduceScript"; - // randomized only once so that any random test instance has the same value - private boolean hasReduceScript = randomBoolean(); + private boolean hasReduceScript; + private Supplier[] valueTypes; + private final Supplier[] leafValueSuppliers = new Supplier[] { () -> randomInt(), () -> randomLong(), () -> randomDouble(), + () -> randomFloat(), () -> randomBoolean(), () -> randomAlphaOfLength(5), () -> new GeoPoint(randomDouble(), randomDouble()), + () -> null }; + private final Supplier[] nestedValueSuppliers = new Supplier[] { () -> new HashMap(), + () -> new ArrayList<>() }; + + @Override + public void setUp() throws Exception { + super.setUp(); + hasReduceScript = randomBoolean(); + // we want the same value types (also for nested lists, maps) for all random aggregations + int levels = randomIntBetween(1, 3); + valueTypes = new Supplier[levels]; + for (int i = 0; i < levels; i++) { + if (i < levels - 1) { + valueTypes[i] = randomFrom(nestedValueSuppliers); + } else { + // the last one needs to be a leaf value, not map or list + valueTypes[i] = randomFrom(leafValueSuppliers); + } + } + } @Override protected InternalScriptedMetric createTestInstance(String name, List pipelineAggregators, @@ -56,7 +83,27 @@ public class InternalScriptedMetricTests extends InternalAggregationTestCase[] valueTypes, int level) { + Object value = valueTypes[level].get(); + if (value instanceof Map) { + int elements = randomIntBetween(1, 5); + Map map = (Map) value; + for (int i = 0; i < elements; i++) { + map.put(randomAlphaOfLength(5), randomValue(valueTypes, level + 1)); + } + } else if (value instanceof List) { + int elements = randomIntBetween(1,5); + List list = (List) value; + for (int i = 0; i < elements; i++) { + list.add(randomValue(valueTypes, level + 1)); + } + } + return value; } /** @@ -105,4 +152,52 @@ public class InternalScriptedMetricTests extends InternalAggregationTestCase pointMap = (Map) actual; + assertEquals(point.getLat(), pointMap.get("lat")); + assertEquals(point.getLon(), pointMap.get("lon")); + } else if (expected instanceof Map) { + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + assertEquals(expectedMap.size(), actualMap.size()); + for (String key : expectedMap.keySet()) { + assertValues(expectedMap.get(key), actualMap.get(key)); + } + } else if (expected instanceof List) { + List expectedList = (List) expected; + List actualList = (List) actual; + assertEquals(expectedList.size(), actualList.size()); + Iterator actualIterator = actualList.iterator(); + for (Object element : expectedList) { + assertValues(element, actualIterator.next()); + } + } else { + assertEquals(expected, actual); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index c256275b994..da3b96b559d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -96,6 +96,8 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.Interna import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.scripted.ParsedScriptedMetric; +import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats; import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder; @@ -182,6 +184,7 @@ public abstract class InternalAggregationTestCase map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c)); map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c)); map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c)); + map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c)); namedXContents = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) From da669f0554e3dbab0f70ad6b36c42daa3ae91b20 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 17 May 2017 20:28:33 +0200 Subject: [PATCH 52/62] Add fromXContent method to SearchResponse (#24720) SearchResponse#fromXContent allows to parse a search response, including search hits, aggregations, suggestions and profile results. Only the aggs that we can parse today are supported (which means all of them but a couple that are left to support). SearchResponseTests reuses the existing test infra to randomize aggregations, suggestions and profile response. Relates to #23331 --- .../action/search/SearchResponse.java | 107 ++++++++++- .../rest/action/RestActions.java | 17 +- .../org/elasticsearch/search/SearchHits.java | 8 +- .../elasticsearch/search/suggest/Suggest.java | 2 +- .../action/search/SearchResponseTests.java | 180 ++++++++++++++++++ .../search/ShardSearchFailureTests.java | 3 +- .../aggregations/AggregationsTests.java | 2 +- .../SearchProfileShardResultsTests.java | 2 +- .../search/suggest/SuggestTests.java | 17 +- .../search/suggest/SuggestionTests.java | 4 +- 10 files changed, 318 insertions(+), 24 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 3e83996c7c6..6e37a9333da 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -21,30 +21,45 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.profile.ProfileShardResult; +import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.suggest.Suggest; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken; + /** * A response of a search request. */ public class SearchResponse extends ActionResponse implements StatusToXContentObject { + private static final ParseField SCROLL_ID = new ParseField("_scroll_id"); + private static final ParseField TOOK = new ParseField("took"); + private static final ParseField TIMED_OUT = new ParseField("timed_out"); + private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early"); + private static final ParseField NUM_REDUCE_PHASES = new ParseField("num_reduce_phases"); + private SearchResponseSections internalResponse; private String scrollId; @@ -175,7 +190,8 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb * * @return The profile results or an empty map */ - @Nullable public Map getProfileResults() { + @Nullable + public Map getProfileResults() { return internalResponse.profile(); } @@ -189,15 +205,15 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { if (scrollId != null) { - builder.field("_scroll_id", scrollId); + builder.field(SCROLL_ID.getPreferredName(), scrollId); } - builder.field("took", tookInMillis); - builder.field("timed_out", isTimedOut()); + builder.field(TOOK.getPreferredName(), tookInMillis); + builder.field(TIMED_OUT.getPreferredName(), isTimedOut()); if (isTerminatedEarly() != null) { - builder.field("terminated_early", isTerminatedEarly()); + builder.field(TERMINATED_EARLY.getPreferredName(), isTerminatedEarly()); } if (getNumReducePhases() != 1) { - builder.field("num_reduce_phases", getNumReducePhases()); + builder.field(NUM_REDUCE_PHASES.getPreferredName(), getNumReducePhases()); } RestActions.buildBroadcastShardsHeader(builder, params, getTotalShards(), getSuccessfulShards(), getFailedShards(), getShardFailures()); @@ -205,6 +221,85 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb return builder; } + public static SearchResponse fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParser.Token token; + String currentFieldName = null; + SearchHits hits = null; + Aggregations aggs = null; + Suggest suggest = null; + SearchProfileShardResults profile = null; + boolean timedOut = false; + Boolean terminatedEarly = null; + int numReducePhases = 1; + long tookInMillis = -1; + int successfulShards = -1; + int totalShards = -1; + String scrollId = null; + List failures = new ArrayList<>(); + while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (SCROLL_ID.match(currentFieldName)) { + scrollId = parser.text(); + } else if (TOOK.match(currentFieldName)) { + tookInMillis = parser.longValue(); + } else if (TIMED_OUT.match(currentFieldName)) { + timedOut = parser.booleanValue(); + } else if (TERMINATED_EARLY.match(currentFieldName)) { + terminatedEarly = parser.booleanValue(); + } else if (NUM_REDUCE_PHASES.match(currentFieldName)) { + numReducePhases = parser.intValue(); + } else { + throwUnknownField(currentFieldName, parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (SearchHits.Fields.HITS.equals(currentFieldName)) { + hits = SearchHits.fromXContent(parser); + } else if (Aggregations.AGGREGATIONS_FIELD.equals(currentFieldName)) { + aggs = Aggregations.fromXContent(parser); + } else if (Suggest.NAME.equals(currentFieldName)) { + suggest = Suggest.fromXContent(parser); + } else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) { + profile = SearchProfileShardResults.fromXContent(parser); + } else if (RestActions._SHARDS_FIELD.match(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (RestActions.FAILED_FIELD.match(currentFieldName)) { + parser.intValue(); // we don't need it but need to consume it + } else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName)) { + successfulShards = parser.intValue(); + } else if (RestActions.TOTAL_FIELD.match(currentFieldName)) { + totalShards = parser.intValue(); + } else { + throwUnknownField(currentFieldName, parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (RestActions.FAILURES_FIELD.match(currentFieldName)) { + while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + failures.add(ShardSearchFailure.fromXContent(parser)); + } + } else { + throwUnknownField(currentFieldName, parser.getTokenLocation()); + } + } else { + throwUnknownToken(token, parser.getTokenLocation()); + } + } + } else { + throwUnknownField(currentFieldName, parser.getTokenLocation()); + } + } + } + SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly, + profile, numReducePhases); + return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, tookInMillis, + failures.toArray(new ShardSearchFailure[failures.size()])); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java index 74836b1dc2d..15199c5a926 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java +++ b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContent.Params; @@ -46,6 +47,12 @@ import java.util.List; public class RestActions { + public static final ParseField _SHARDS_FIELD = new ParseField("_shards"); + public static final ParseField TOTAL_FIELD = new ParseField("total"); + public static final ParseField SUCCESSFUL_FIELD = new ParseField("successful"); + public static final ParseField FAILED_FIELD = new ParseField("failed"); + public static final ParseField FAILURES_FIELD = new ParseField("failures"); + public static long parseVersion(RestRequest request) { if (request.hasParam("version")) { return request.paramAsLong("version", Versions.MATCH_ANY); @@ -71,12 +78,12 @@ public class RestActions { public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params, int total, int successful, int failed, ShardOperationFailedException[] shardFailures) throws IOException { - builder.startObject("_shards"); - builder.field("total", total); - builder.field("successful", successful); - builder.field("failed", failed); + builder.startObject(_SHARDS_FIELD.getPreferredName()); + builder.field(TOTAL_FIELD.getPreferredName(), total); + builder.field(SUCCESSFUL_FIELD.getPreferredName(), successful); + builder.field(FAILED_FIELD.getPreferredName(), failed); if (shardFailures != null && shardFailures.length > 0) { - builder.startArray("failures"); + builder.startArray(FAILURES_FIELD.getPreferredName()); final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default for (ShardOperationFailedException shardFailure : group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures) { builder.startObject(); diff --git a/core/src/main/java/org/elasticsearch/search/SearchHits.java b/core/src/main/java/org/elasticsearch/search/SearchHits.java index ada09d2e38b..0b49ba8ec12 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchHits.java +++ b/core/src/main/java/org/elasticsearch/search/SearchHits.java @@ -105,10 +105,10 @@ public final class SearchHits implements Streamable, ToXContent, Iterable>>, Streamable, ToXContent { - static final String NAME = "suggest"; + public static final String NAME = "suggest"; public static final Comparator