honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HLRC support for string_stats (#52163) (#52297) 

This adds a builder and parsed results for the `string_stats`
aggregation directly to the high level rest client. Without this the
HLRC can't access the `string_stats` API without the elastic licensed
`analytics` module.

While I'm in there this adds a few of our usual unit tests and
modernizes the parsing.
This commit is contained in:
Nik Everett 2020-02-12 19:25:05 -05:00 committed by GitHub
parent 12e378b3ac
commit 2dac36de4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 727 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
View File

@ -54,6 +54,8 @@ import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.analytics.ParsedStringStats;
import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.core.GetSourceRequest;
@ -1926,6 +1928,7 @@ public class RestHighLevelClient implements Closeable {
map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c));
List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
.map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
.collect(Collectors.toList());

172
client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedStringStats.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.analytics;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.aggregations.ParsedAggregation;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static java.util.Collections.unmodifiableMap;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
/**
* Results from the {@code string_stats} aggregation.
*/
public class ParsedStringStats extends ParsedAggregation {
private static final ParseField COUNT_FIELD = new ParseField("count");
private static final ParseField MIN_LENGTH_FIELD = new ParseField("min_length");
private static final ParseField MAX_LENGTH_FIELD = new ParseField("max_length");
private static final ParseField AVG_LENGTH_FIELD = new ParseField("avg_length");
private static final ParseField ENTROPY_FIELD = new ParseField("entropy");
private static final ParseField DISTRIBUTION_FIELD = new ParseField("distribution");
private final long count;
private final int minLength;
private final int maxLength;
private final double avgLength;
private final double entropy;
private final boolean showDistribution;
private final Map<String, Double> distribution;
private ParsedStringStats(String name, long count, int minLength, int maxLength, double avgLength, double entropy,
boolean showDistribution, Map<String, Double> distribution) {
setName(name);
this.count = count;
this.minLength = minLength;
this.maxLength = maxLength;
this.avgLength = avgLength;
this.entropy = entropy;
this.showDistribution = showDistribution;
this.distribution = distribution;
}
/**
* The number of non-empty fields counted.
*/
public long getCount() {
return count;
}
/**
* The length of the shortest term.
*/
public int getMinLength() {
return minLength;
}
/**
* The length of the longest term.
*/
public int getMaxLength() {
return maxLength;
}
/**
* The average length computed over all terms.
*/
public double getAvgLength() {
return avgLength;
}
/**
* The <a href="https://en.wikipedia.org/wiki/Entropy_(information_theory)">Shannon Entropy</a>
* value computed over all terms collected by the aggregation.
* Shannon entropy quantifies the amount of information contained in
* the field. It is a very useful metric for measuring a wide range of
* properties of a data set, such as diversity, similarity,
* randomness etc.
*/
public double getEntropy() {
return entropy;
}
/**
* The probability distribution for all characters. {@code null} unless
* explicitly requested with {@link StringStatsAggregationBuilder#showDistribution(boolean)}.
*/
public Map<String, Double> getDistribution() {
return distribution;
}
@Override
public String getType() {
return StringStatsAggregationBuilder.NAME;
}
private static final Object NULL_DISTRIBUTION_MARKER = new Object();
public static final ConstructingObjectParser<ParsedStringStats, String> PARSER = new ConstructingObjectParser<>(
StringStatsAggregationBuilder.NAME, true, (args, name) -> {
long count = (long) args[0];
boolean disributionWasExplicitNull = args[5] == NULL_DISTRIBUTION_MARKER;
if (count == 0) {
return new ParsedStringStats(name, count, 0, 0, 0, 0, disributionWasExplicitNull, null);
}
int minLength = (int) args[1];
int maxLength = (int) args[2];
double averageLength = (double) args[3];
double entropy = (double) args[4];
if (disributionWasExplicitNull) {
return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
disributionWasExplicitNull, null);
} else {
@SuppressWarnings("unchecked")
Map<String, Double> distribution = (Map<String, Double>) args[5];
return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
distribution != null, distribution);
}
});
static {
PARSER.declareLong(constructorArg(), COUNT_FIELD);
PARSER.declareIntOrNull(constructorArg(), 0, MIN_LENGTH_FIELD);
PARSER.declareIntOrNull(constructorArg(), 0, MAX_LENGTH_FIELD);
PARSER.declareDoubleOrNull(constructorArg(), 0, AVG_LENGTH_FIELD);
PARSER.declareDoubleOrNull(constructorArg(), 0, ENTROPY_FIELD);
PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> unmodifiableMap(p.map(HashMap::new, XContentParser::doubleValue)),
NULL_DISTRIBUTION_MARKER, DISTRIBUTION_FIELD);
ParsedAggregation.declareAggregationFields(PARSER);
}
@Override
protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
builder.field(COUNT_FIELD.getPreferredName(), count);
if (count == 0) {
builder.nullField(MIN_LENGTH_FIELD.getPreferredName());
builder.nullField(MAX_LENGTH_FIELD.getPreferredName());
builder.nullField(AVG_LENGTH_FIELD.getPreferredName());
builder.field(ENTROPY_FIELD.getPreferredName(), 0.0);
} else {
builder.field(MIN_LENGTH_FIELD.getPreferredName(), minLength);
builder.field(MAX_LENGTH_FIELD.getPreferredName(), maxLength);
builder.field(AVG_LENGTH_FIELD.getPreferredName(), avgLength);
builder.field(ENTROPY_FIELD.getPreferredName(), entropy);
}
if (showDistribution) {
builder.field(DISTRIBUTION_FIELD.getPreferredName(), distribution);
}
return builder;
}
}

116
client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.analytics;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
/**
* Builds the {@code string_stats} aggregation request.
* <p>
* NOTE: This extends {@linkplain AbstractAggregationBuilder} for compatibility
* with {@link SearchSourceBuilder#aggregation(AggregationBuilder)} but it
* doesn't support any "server" side things like
* {@linkplain Writeable#writeTo(StreamOutput)},
* {@linkplain AggregationBuilder#rewrite(QueryRewriteContext)}, or
* {@linkplain AbstractAggregationBuilder#build(QueryShardContext, AggregatorFactory)}.
*/
public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, StringStatsAggregationBuilder> {
public static final String NAME = "string_stats";
private static final ParseField SHOW_DISTRIBUTION_FIELD = new ParseField("show_distribution");
private boolean showDistribution = false;
public StringStatsAggregationBuilder(String name) {
super(name, CoreValuesSourceType.BYTES, ValueType.STRING);
}
/**
* Compute the distribution of each character. Disabled by default.
* @return this for chaining
*/
public StringStatsAggregationBuilder showDistribution(boolean showDistribution) {
this.showDistribution = showDistribution;
return this;
}
@Override
public String getType() {
return NAME;
}
@Override
public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
return builder.field(StringStatsAggregationBuilder.SHOW_DISTRIBUTION_FIELD.getPreferredName(), showDistribution);
}
@Override
protected void innerWriteTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
protected ValuesSourceAggregatorFactory<Bytes> innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig<Bytes> config,
AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
throw new UnsupportedOperationException();
}
@Override
protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metaData) {
throw new UnsupportedOperationException();
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), showDistribution);
}
@Override
public boolean equals(Object obj) {
if (obj == null || getClass() != obj.getClass()) {
return false;
}
if (false == super.equals(obj)) {
return false;
}
StringStatsAggregationBuilder other = (StringStatsAggregationBuilder) obj;
return showDistribution == other.showDistribution;
}
}

2
client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.client;
import com.fasterxml.jackson.core.JsonParseException;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
@ -675,6 +676,7 @@ public class RestHighLevelClientTests extends ESTestCase {
List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getDefaultNamedXContents();
int expectedInternalAggregations = InternalAggregationTestCase.getDefaultNamedXContents().size();
int expectedSuggestions = 3;
assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("string_stats")));
assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size());
Map<Class<?>, Integer> categories = new HashMap<>();
for (NamedXContentRegistry.Entry namedXContent : namedXContents) {

58
client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.analytics;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasEntry;
public class AnalyticsAggsIT extends ESRestHighLevelClientTestCase {
public void testBasic() throws IOException {
BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
bulk.add(new IndexRequest().source(XContentType.JSON, "message", "trying out elasticsearch"));
bulk.add(new IndexRequest().source(XContentType.JSON, "message", "more words"));
highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
SearchRequest search = new SearchRequest("test");
search.source().aggregation(new StringStatsAggregationBuilder("test").field("message.keyword").showDistribution(true));
SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
ParsedStringStats stats = response.getAggregations().get("test");
assertThat(stats.getCount(), equalTo(2L));
assertThat(stats.getMinLength(), equalTo(10));
assertThat(stats.getMaxLength(), equalTo(24));
assertThat(stats.getAvgLength(), equalTo(17.0));
assertThat(stats.getEntropy(), closeTo(4, .1));
assertThat(stats.getDistribution(), aMapWithSize(18));
assertThat(stats.getDistribution(), hasEntry(equalTo("o"), closeTo(.09, .005)));
assertThat(stats.getDistribution(), hasEntry(equalTo("r"), closeTo(.12, .005)));
assertThat(stats.getDistribution(), hasEntry(equalTo("t"), closeTo(.09, .005)));
}
}

1
docs/java-rest/high-level/aggs-builders.asciidoc
View File

@ -26,6 +26,7 @@ This page lists all the available aggregations with their corresponding `Aggrega
| {ref}/search-aggregations-metrics-sum-aggregation.html[Sum] | {agg-ref}/metrics/sum/SumAggregationBuilder.html[SumAggregationBuilder] | {agg-ref}/AggregationBuilders.html#sum-java.lang.String-[AggregationBuilders.sum()]
| {ref}/search-aggregations-metrics-top-hits-aggregation.html[Top hits] | {agg-ref}/metrics/tophits/TopHitsAggregationBuilder.html[TopHitsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#topHits-java.lang.String-[AggregationBuilders.topHits()]
| {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count] | {agg-ref}/metrics/valuecount/ValueCountAggregationBuilder.html[ValueCountAggregationBuilder] | {agg-ref}/AggregationBuilders.html#count-java.lang.String-[AggregationBuilders.count()]
| {ref}/search-aggregations-metrics-string-stats-aggregation.html[String Stats] | {javadoc-client}/analytics/StringStatsAggregationBuilder.html[StringStatsAggregationBuilder] | None
|======
==== Bucket Aggregations

28
libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
View File

@ -148,6 +148,15 @@ public abstract class AbstractObjectParser<Value, Context>
declareField(consumer, (p, c) -> objectParser.parse(p, c), field, ValueType.OBJECT);
}
/**
* Declare an object field that parses explicit {@code null}s in the json to a default value.
*/
public <T> void declareObjectOrNull(BiConsumer<Value, T> consumer, ContextParser<Context, T> objectParser, T nullValue,
ParseField field) {
declareField(consumer, (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? nullValue : objectParser.parse(p, c),
field, ValueType.OBJECT_OR_NULL);
}
public void declareFloat(BiConsumer<Value, Float> consumer, ParseField field) {
// Using a method reference here angers some compilers
declareField(consumer, p -> p.floatValue(), field, ValueType.FLOAT);
@ -158,16 +167,33 @@ public abstract class AbstractObjectParser<Value, Context>
declareField(consumer, p -> p.doubleValue(), field, ValueType.DOUBLE);
}
/**
* Declare a double field that parses explicit {@code null}s in the json to a default value.
*/
public void declareDoubleOrNull(BiConsumer<Value, Double> consumer, double nullValue, ParseField field) {
declareField(consumer, p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? nullValue : p.doubleValue(),
field, ValueType.DOUBLE_OR_NULL);
}
public void declareLong(BiConsumer<Value, Long> consumer, ParseField field) {
// Using a method reference here angers some compilers
declareField(consumer, p -> p.longValue(), field, ValueType.LONG);
}
public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
// Using a method reference here angers some compilers
// Using a method reference here angers some compilers
declareField(consumer, p -> p.intValue(), field, ValueType.INT);
}
/**
* Declare a double field that parses explicit {@code null}s in the json to a default value.
*/
public void declareIntOrNull(BiConsumer<Value, Integer> consumer, int nullValue, ParseField field) {
declareField(consumer, p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? nullValue : p.intValue(),
field, ValueType.INT_OR_NULL);
}
public void declareString(BiConsumer<Value, String> consumer, ParseField field) {
declareField(consumer, XContentParser::text, field, ValueType.STRING);
}

48
libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
View File

@ -43,6 +43,7 @@ import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
public class ObjectParserTests extends ESTestCase {
@ -275,6 +276,24 @@ public class ObjectParserTests extends ESTestCase {
assertNotNull(s.object);
}
public void testObjectOrNullWhenNull() throws IOException {
StaticTestStruct nullMarker = new StaticTestStruct();
XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : null}");
ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
objectParser.declareObjectOrNull(StaticTestStruct::setObject, objectParser, nullMarker, new ParseField("object"));
StaticTestStruct s = objectParser.parse(parser, null);
assertThat(s.object, equalTo(nullMarker));
}
public void testObjectOrNullWhenNonNull() throws IOException {
StaticTestStruct nullMarker = new StaticTestStruct();
XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : {}}");
ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
objectParser.declareObjectOrNull(StaticTestStruct::setObject, objectParser, nullMarker, new ParseField("object"));
StaticTestStruct s = objectParser.parse(parser, null);
assertThat(s.object, not(nullValue()));
}
public void testEmptyObjectInArray() throws IOException {
XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object_array\" : [{}]}");
ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
@ -321,15 +340,32 @@ public class ObjectParserTests extends ESTestCase {
}
public void testAllVariants() throws IOException {
double expectedNullableDouble;
int expectedNullableInt;
XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
builder.startObject();
builder.field("int_field", randomBoolean() ? "1" : 1);
if (randomBoolean()) {
builder.nullField("nullable_int_field");
expectedNullableInt = -1;
} else {
expectedNullableInt = randomInt();
builder.field("nullable_int_field", expectedNullableInt);
}
if (randomBoolean()) {
builder.array("int_array_field", randomBoolean() ? "1" : 1);
} else {
builder.field("int_array_field", randomBoolean() ? "1" : 1);
}
builder.field("double_field", randomBoolean() ? "2.1" : 2.1d);
if (randomBoolean()) {
builder.nullField("nullable_double_field");
expectedNullableDouble = Double.NaN;
} else {
expectedNullableDouble = randomDouble();
builder.field("nullable_double_field", expectedNullableDouble);
}
if (randomBoolean()) {
builder.array("double_array_field", randomBoolean() ? "2.1" : 2.1d);
} else {
@ -364,9 +400,11 @@ public class ObjectParserTests extends ESTestCase {
XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder));
class TestStruct {
int int_field;
int nullableIntField;
long long_field;
float float_field;
double double_field;
double nullableDoubleField;
String string_field;
List<Integer> int_array_field;
List<Long> long_array_field;
@ -378,6 +416,9 @@ public class ObjectParserTests extends ESTestCase {
public void setInt_field(int int_field) {
this.int_field = int_field;
}
public void setNullableIntField(int nullableIntField) {
this.nullableIntField = nullableIntField;
}
public void setLong_field(long long_field) {
this.long_field = long_field;
}
@ -387,6 +428,9 @@ public class ObjectParserTests extends ESTestCase {
public void setDouble_field(double double_field) {
this.double_field = double_field;
}
public void setNullableDoubleField(double nullableDoubleField) {
this.nullableDoubleField = nullableDoubleField;
}
public void setString_field(String string_field) {
this.string_field = string_field;
}
@ -416,10 +460,12 @@ public class ObjectParserTests extends ESTestCase {
}
ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
objectParser.declareInt(TestStruct::setInt_field, new ParseField("int_field"));
objectParser.declareIntOrNull(TestStruct::setNullableIntField, -1, new ParseField("nullable_int_field"));
objectParser.declareIntArray(TestStruct::setInt_array_field, new ParseField("int_array_field"));
objectParser.declareLong(TestStruct::setLong_field, new ParseField("long_field"));
objectParser.declareLongArray(TestStruct::setLong_array_field, new ParseField("long_array_field"));
objectParser.declareDouble(TestStruct::setDouble_field, new ParseField("double_field"));
objectParser.declareDoubleOrNull(TestStruct::setNullableDoubleField, Double.NaN, new ParseField("nullable_double_field"));
objectParser.declareDoubleArray(TestStruct::setDouble_array_field, new ParseField("double_array_field"));
objectParser.declareFloat(TestStruct::setFloat_field, new ParseField("float_field"));
objectParser.declareFloatArray(TestStruct::setFloat_array_field, new ParseField("float_array_field"));
@ -431,6 +477,7 @@ public class ObjectParserTests extends ESTestCase {
TestStruct parse = objectParser.parse(parser, new TestStruct(), null);
assertArrayEquals(parse.double_array_field.toArray(), Collections.singletonList(2.1d).toArray());
assertEquals(parse.double_field, 2.1d, 0.0d);
assertThat(parse.nullableDoubleField, equalTo(expectedNullableDouble));
assertArrayEquals(parse.long_array_field.toArray(), Collections.singletonList(4L).toArray());
assertEquals(parse.long_field, 4L);
@ -440,6 +487,7 @@ public class ObjectParserTests extends ESTestCase {
assertArrayEquals(parse.int_array_field.toArray(), Collections.singletonList(1).toArray());
assertEquals(parse.int_field, 1);
assertThat(parse.nullableIntField, equalTo(expectedNullableInt));
assertArrayEquals(parse.float_array_field.toArray(), Collections.singletonList(3.1f).toArray());
assertEquals(parse.float_field, 3.1f, 0.0f);

4
server/src/main/java/org/elasticsearch/search/aggregations/ParsedAggregation.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.search.aggregations;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.AbstractObjectParser;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -36,7 +36,7 @@ import java.util.Map;
*/
public abstract class ParsedAggregation implements Aggregation, ToXContentFragment {
protected static void declareAggregationFields(ObjectParser<? extends ParsedAggregation, Void> objectParser) {
protected static void declareAggregationFields(AbstractObjectParser<? extends ParsedAggregation, ?> objectParser) {
objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata),
(parser, context) -> parser.map(), InternalAggregation.CommonFields.META);
}

30
test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java
View File

@ -56,7 +56,9 @@ public class NotEqualMessageBuilder {
actual = new TreeMap<>(actual);
expected = new TreeMap<>(expected);
for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
boolean hadKey = actual.containsKey(expectedEntry.getKey());
Object actualValue = actual.remove(expectedEntry.getKey());
compare(expectedEntry.getKey(), hadKey, actualValue, expectedEntry.getValue());
}
for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
@ -69,7 +71,7 @@ public class NotEqualMessageBuilder {
public void compareLists(List<Object> actual, List<Object> expected) {
int i = 0;
while (i < actual.size() && i < expected.size()) {
compare(Integer.toString(i), actual.get(i), expected.get(i));
compare(Integer.toString(i), true, actual.get(i), expected.get(i));
i++;
}
if (actual.size() == expected.size()) {
@ -87,12 +89,16 @@ public class NotEqualMessageBuilder {
* Compare two values.
* @param field the name of the field being compared.
*/
public void compare(String field, @Nullable Object actual, Object expected) {
public void compare(String field, boolean hadKey, @Nullable Object actual, Object expected) {
if (expected instanceof Map) {
if (actual == null) {
if (false == hadKey) {
field(field, "expected map but not found");
return;
}
if (actual == null) {
field(field, "expected map but was [null]");
return;
}
if (false == actual instanceof Map) {
field(field, "expected map but found [" + actual + "]");
return;
@ -112,10 +118,14 @@ public class NotEqualMessageBuilder {
return;
}
if (expected instanceof List) {
if (actual == null) {
if (false == hadKey) {
field(field, "expected list but not found");
return;
}
if (actual == null) {
field(field, "expected list but was [null]");
return;
}
if (false == actual instanceof List) {
field(field, "expected list but found [" + actual + "]");
return;
@ -134,10 +144,18 @@ public class NotEqualMessageBuilder {
indent -= 1;
return;
}
if (actual == null) {
if (false == hadKey) {
field(field, "expected [" + expected + "] but not found");
return;
}
if (actual == null) {
if (expected == null) {
field(field, "same [" + expected + "]");
return;
}
field(field, "expected [" + expected + "] but was [null]");
return;
}
if (Objects.equals(expected, actual)) {
if (expected instanceof String) {
String expectedString = (String) expected;

2
test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
View File

@ -89,7 +89,7 @@ public class MatchAssertion extends Assertion {
if (expectedValue.equals(actualValue) == false) {
NotEqualMessageBuilder message = new NotEqualMessageBuilder();
message.compare(getField(), actualValue, expectedValue);
message.compare(getField(), true, actualValue, expectedValue);
throw new AssertionError(getField() + " didn't match expected value:\n" + message);
}
}

13
test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/MatchAssertionTests.java
View File

@ -21,6 +21,10 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.ESTestCase;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.containsString;
public class MatchAssertionTests extends ESTestCase {
public void testNull() {
@ -39,4 +43,13 @@ public class MatchAssertionTests extends ESTestCase {
expectThrows(AssertionError.class, () -> matchAssertion.doAssert(null, "/exp/"));
}
}
public void testNullInMap() {
XContentLocation xContentLocation = new XContentLocation(0, 0);
MatchAssertion matchAssertion = new MatchAssertion(xContentLocation, "field", singletonMap("a", null));
matchAssertion.doAssert(singletonMap("a", null), matchAssertion.getExpectedValue());
AssertionError e = expectThrows(AssertionError.class, () ->
matchAssertion.doAssert(emptyMap(), matchAssertion.getExpectedValue()));
assertThat(e.getMessage(), containsString("expected [null] but not found"));
}
}

2
x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java
View File

@ -67,7 +67,7 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
new AggregationSpec(
StringStatsAggregationBuilder.NAME,
StringStatsAggregationBuilder::new,
StringStatsAggregationBuilder::parse).addResultReader(InternalStringStats::new),
StringStatsAggregationBuilder.PARSER).addResultReader(InternalStringStats::new),
new AggregationSpec(
BoxplotAggregationBuilder.NAME,
BoxplotAggregationBuilder::new,

13
x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStats.java
View File

@ -107,6 +107,10 @@ public class InternalStringStats extends InternalAggregation {
return count;
}
long getTotalLength () {
return totalLength;
}
public int getMinLength() {
return minLength;
}
@ -153,6 +157,14 @@ public class InternalStringStats extends InternalAggregation {
return Math.log(d) / Math.log(2.0);
}
Map<String, Long> getCharOccurrences() {
return charOccurrences;
}
boolean getShowDistribution() {
return showDistribution;
}
public String getCountAsString() {
return format.format(getCount()).toString();
}
@ -282,6 +294,7 @@ public class InternalStringStats extends InternalAggregation {
minLength == other.minLength &&
maxLength == other.maxLength &&
totalLength == other.totalLength &&
Objects.equals(charOccurrences, other.charOccurrences) &&
showDistribution == other.showDistribution;
}
}

13
x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java
View File

@ -10,7 +10,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories;
@ -27,23 +26,17 @@ import java.util.Map;
import java.util.Objects;
public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, StringStatsAggregationBuilder> {
public static final String NAME = "string_stats";
private boolean showDistribution = false;
private static final ObjectParser<StringStatsAggregationBuilder, Void> PARSER;
private static final ParseField SHOW_DISTRIBUTION_FIELD = new ParseField("show_distribution");
public static final ObjectParser<StringStatsAggregationBuilder, String> PARSER =
ObjectParser.fromBuilder(NAME, StringStatsAggregationBuilder::new);
static {
PARSER = new ObjectParser<>(StringStatsAggregationBuilder.NAME);
ValuesSourceParserHelper.declareBytesFields(PARSER, true, true);
PARSER.declareBoolean(StringStatsAggregationBuilder::showDistribution, StringStatsAggregationBuilder.SHOW_DISTRIBUTION_FIELD);
}
public static StringStatsAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
return PARSER.parse(parser, new StringStatsAggregationBuilder(aggregationName), null);
}
private boolean showDistribution = false;
public StringStatsAggregationBuilder(String name) {
super(name, CoreValuesSourceType.BYTES, ValueType.STRING);

146
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStatsTests.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.analytics.stringstats;
import org.elasticsearch.client.analytics.ParsedStringStats;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.ParsedAggregation;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.InternalAggregationTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import static java.util.Collections.emptyMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
public class InternalStringStatsTests extends InternalAggregationTestCase<InternalStringStats> {
@Override
protected List<NamedXContentRegistry.Entry> getNamedXContents() {
List<NamedXContentRegistry.Entry> result = new ArrayList<>(super.getNamedXContents());
result.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(StringStatsAggregationBuilder.NAME),
(p, c) -> ParsedStringStats.PARSER.parse(p, (String) c)));
return result;
}
protected InternalStringStats createTestInstance(
String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
if (randomBoolean()) {
return new InternalStringStats(name, 0, 0, 0, 0, emptyMap(), randomBoolean(), DocValueFormat.RAW,
pipelineAggregators, metaData);
}
return new InternalStringStats(name, randomLongBetween(1, Long.MAX_VALUE),
randomNonNegativeLong(), between(0, Integer.MAX_VALUE), between(0, Integer.MAX_VALUE), randomCharOccurrences(),
randomBoolean(), DocValueFormat.RAW,
pipelineAggregators, metaData);
};
@Override
protected InternalStringStats mutateInstance(InternalStringStats instance) throws IOException {
String name = instance.getName();
long count = instance.getCount();
long totalLength = instance.getTotalLength();
int minLength = instance.getMinLength();
int maxLength = instance.getMaxLength();
Map<String, Long> charOccurrences = instance.getCharOccurrences();
boolean showDistribution = instance.getShowDistribution();
switch (between(0, 6)) {
case 0:
name = name + "a";
break;
case 1:
count = randomValueOtherThan(count, () -> randomLongBetween(1, Long.MAX_VALUE));
break;
case 2:
totalLength = randomValueOtherThan(totalLength, ESTestCase::randomNonNegativeLong);
break;
case 3:
minLength = randomValueOtherThan(minLength, () -> between(0, Integer.MAX_VALUE));
break;
case 4:
maxLength = randomValueOtherThan(maxLength, () -> between(0, Integer.MAX_VALUE));
break;
case 5:
charOccurrences = randomValueOtherThan(charOccurrences, this::randomCharOccurrences);
break;
case 6:
showDistribution = !showDistribution;
break;
}
return new InternalStringStats(name, count, totalLength, minLength, maxLength, charOccurrences, showDistribution,
DocValueFormat.RAW, instance.pipelineAggregators(), instance.getMetaData());
}
@Override
protected Reader<InternalStringStats> instanceReader() {
return InternalStringStats::new;
}
@Override
protected void assertFromXContent(InternalStringStats aggregation, ParsedAggregation parsedAggregation) throws IOException {
ParsedStringStats parsed = (ParsedStringStats) parsedAggregation;
assertThat(parsed.getName(), equalTo(aggregation.getName()));
if (aggregation.getCount() == 0) {
assertThat(parsed.getCount(), equalTo(0L));
assertThat(parsed.getMinLength(), equalTo(0));
assertThat(parsed.getMaxLength(), equalTo(0));
assertThat(parsed.getAvgLength(), equalTo(0d));
assertThat(parsed.getEntropy(), equalTo(0d));
assertThat(parsed.getDistribution(), nullValue());
return;
}
assertThat(parsed.getCount(), equalTo(aggregation.getCount()));
assertThat(parsed.getMinLength(), equalTo(aggregation.getMinLength()));
assertThat(parsed.getMaxLength(), equalTo(aggregation.getMaxLength()));
assertThat(parsed.getAvgLength(), equalTo(aggregation.getAvgLength()));
assertThat(parsed.getEntropy(), equalTo(aggregation.getEntropy()));
if (aggregation.getShowDistribution()) {
assertThat(parsed.getDistribution(), equalTo(aggregation.getDistribution()));
} else {
assertThat(parsed.getDistribution(), nullValue());
}
}
@Override
protected Predicate<String> excludePathsFromXContentInsertion() {
return path -> path.endsWith(".distribution");
}
@Override
protected void assertReduced(InternalStringStats reduced, List<InternalStringStats> inputs) {
assertThat(reduced.getCount(), equalTo(inputs.stream().mapToLong(InternalStringStats::getCount).sum()));
assertThat(reduced.getMinLength(), equalTo(inputs.stream().mapToInt(InternalStringStats::getMinLength).min().getAsInt()));
assertThat(reduced.getMaxLength(), equalTo(inputs.stream().mapToInt(InternalStringStats::getMaxLength).max().getAsInt()));
assertThat(reduced.getTotalLength(), equalTo(inputs.stream().mapToLong(InternalStringStats::getTotalLength).sum()));
Map<String, Long> reducedChars = new HashMap<>();
for (InternalStringStats stats : inputs) {
for (Map.Entry<String, Long> e : stats.getCharOccurrences().entrySet()) {
reducedChars.merge(e.getKey(), e.getValue(), (lhs, rhs) -> lhs + rhs);
}
}
assertThat(reduced.getCharOccurrences(), equalTo(reducedChars));
}
private Map<String, Long> randomCharOccurrences() {
Map<String, Long> charOccurrences = new HashMap<String, Long>();
int occurrencesSize = between(0, 1000);
while (charOccurrences.size() < occurrencesSize) {
charOccurrences.put(randomAlphaOfLength(5), randomNonNegativeLong());
}
return charOccurrences;
}
}

97
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilderTests.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.analytics.stringstats;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
import org.elasticsearch.test.AbstractSerializingTestCase;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.Arrays;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
public class StringStatsAggregationBuilderTests extends AbstractSerializingTestCase<StringStatsAggregationBuilder> {
@Override
protected NamedXContentRegistry xContentRegistry() {
return new NamedXContentRegistry(Arrays.asList(
new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, new ParseField(StringStatsAggregationBuilder.NAME),
(p, c) -> StringStatsAggregationBuilder.PARSER.parse(p, (String) c))));
}
@Override
protected StringStatsAggregationBuilder doParseInstance(XContentParser parser) throws IOException {
assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
String name = parser.currentName();
assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("string_stats"));
StringStatsAggregationBuilder parsed = StringStatsAggregationBuilder.PARSER.apply(parser, name);
assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
return parsed;
}
@Override
protected Reader<StringStatsAggregationBuilder> instanceReader() {
return StringStatsAggregationBuilder::new;
}
@Override
protected StringStatsAggregationBuilder createTestInstance() {
StringStatsAggregationBuilder builder = new StringStatsAggregationBuilder(randomAlphaOfLength(5));
builder.showDistribution(randomBoolean());
return builder;
}
@Override
protected StringStatsAggregationBuilder mutateInstance(StringStatsAggregationBuilder instance) throws IOException {
if (randomBoolean()) {
StringStatsAggregationBuilder mutant = new StringStatsAggregationBuilder(instance.getName());
mutant.showDistribution(!instance.showDistribution());
return mutant;
}
StringStatsAggregationBuilder mutant = new StringStatsAggregationBuilder(randomAlphaOfLength(4));
mutant.showDistribution(instance.showDistribution());
return mutant;
}
public void testClientBuilder() throws IOException {
AbstractXContentTestCase.xContentTester(
this::createParser, this::createTestInstance, this::toXContentThroughClientBuilder,
p -> {
p.nextToken();
AggregatorFactories.Builder b = AggregatorFactories.parseAggregators(p);
assertThat(b.getAggregatorFactories(), hasSize(1));
assertThat(b.getPipelineAggregatorFactories(), empty());
return (StringStatsAggregationBuilder) b.getAggregatorFactories().iterator().next();
} ).test();
}
private void toXContentThroughClientBuilder(StringStatsAggregationBuilder serverBuilder, XContentBuilder builder) throws IOException {
builder.startObject();
createClientBuilder(serverBuilder).toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
}
private org.elasticsearch.client.analytics.StringStatsAggregationBuilder createClientBuilder(
StringStatsAggregationBuilder serverBuilder) {
org.elasticsearch.client.analytics.StringStatsAggregationBuilder builder =
new org.elasticsearch.client.analytics.StringStatsAggregationBuilder(serverBuilder.getName());
return builder.showDistribution(serverBuilder.showDistribution());
}
}