diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java index ea2d4cca90d..ae72e27e79b 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; -public abstract class FieldStats implements Streamable, ToXContent { +public abstract class FieldStats> implements Streamable, ToXContent { private byte type; private long maxDoc; @@ -120,6 +120,12 @@ public abstract class FieldStats implements Streamable, ToXContent { */ public abstract String getMaxValue(); + /** + * @param value The string to be parsed + * @return The concrete object represented by the string argument + */ + protected abstract T valueOf(String value); + /** * Merges the provided stats into this stats instance. */ @@ -142,6 +148,34 @@ public abstract class FieldStats implements Streamable, ToXContent { } } + /** + * @return true if this instance matches with the provided index constraint, otherwise false is returned + */ + public boolean match(IndexConstraint constraint) { + int cmp; + T value = valueOf(constraint.getValue()); + if (constraint.getProperty() == IndexConstraint.Property.MIN) { + cmp = minValue.compareTo(value); + } else if (constraint.getProperty() == IndexConstraint.Property.MAX) { + cmp = maxValue.compareTo(value); + } else { + throw new IllegalArgumentException("Unsupported property [" + constraint.getProperty() + "]"); + } + + switch (constraint.getComparison()) { + case GT: + return cmp > 0; + case GTE: + return cmp >= 0; + case LT: + return cmp < 0; + case LTE: + return cmp <= 0; + default: + throw new IllegalArgumentException("Unsupported comparison [" + constraint.getComparison() + "]"); + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -210,6 +244,11 @@ public abstract class FieldStats implements Streamable, ToXContent { this.maxValue = Math.max(other.maxValue, maxValue); } + @Override + protected java.lang.Long valueOf(String value) { + return java.lang.Long.valueOf(value); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -255,6 +294,11 @@ public abstract class FieldStats implements Streamable, ToXContent { this.maxValue = Math.max(other.maxValue, maxValue); } + @Override + protected java.lang.Float valueOf(String value) { + return java.lang.Float.valueOf(value); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -300,6 +344,11 @@ public abstract class FieldStats implements Streamable, ToXContent { this.maxValue = Math.max(other.maxValue, maxValue); } + @Override + protected java.lang.Double valueOf(String value) { + return java.lang.Double.valueOf(value); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -349,6 +398,11 @@ public abstract class FieldStats implements Streamable, ToXContent { } } + @Override + protected BytesRef valueOf(String value) { + return new BytesRef(value); + } + @Override protected void toInnerXContent(XContentBuilder builder) throws IOException { builder.field(Fields.MIN_VALUE, getMinValue()); @@ -393,6 +447,11 @@ public abstract class FieldStats implements Streamable, ToXContent { return dateFormatter.printer().print(maxValue); } + @Override + protected java.lang.Long valueOf(String value) { + return dateFormatter.parser().parseMillis(value); + } + @Override protected void toInnerXContent(XContentBuilder builder) throws IOException { builder.field(Fields.MIN_VALUE, getMinValue()); diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java index e157865ecdf..aa107518110 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java @@ -22,10 +22,17 @@ package org.elasticsearch.action.fieldstats; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.broadcast.BroadcastRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** */ @@ -33,17 +40,113 @@ public class FieldStatsRequest extends BroadcastRequest { public final static String DEFAULT_LEVEL = "cluster"; - private String[] fields; + private String[] fields = Strings.EMPTY_ARRAY; private String level = DEFAULT_LEVEL; + private IndexConstraint[] indexConstraints = new IndexConstraint[0]; - public String[] fields() { + public String[] getFields() { return fields; } - public void fields(String[] fields) { + public void setFields(String[] fields) { + if (fields == null) { + throw new NullPointerException("specified fields can't be null"); + } this.fields = fields; } + public IndexConstraint[] getIndexConstraints() { + return indexConstraints; + } + + public void setIndexConstraints(IndexConstraint[] indexConstraints) { + if (indexConstraints == null) { + throw new NullPointerException("specified index_contraints can't be null"); + } + this.indexConstraints = indexConstraints; + } + + public void source(BytesReference content) throws IOException { + List indexConstraints = new ArrayList<>(); + List fields = new ArrayList<>(); + try (XContentParser parser = XContentHelper.createParser(content)) { + String fieldName = null; + Token token = parser.nextToken(); + assert token == Token.START_OBJECT; + for (token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) { + switch (token) { + case FIELD_NAME: + fieldName = parser.currentName(); + break; + case START_OBJECT: + if ("index_constraints".equals(fieldName)) { + parseIndexContraints(indexConstraints, parser); + } else { + throw new IllegalArgumentException("unknown field [" + fieldName + "]"); + } + break; + case START_ARRAY: + if ("fields".equals(fieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token.isValue()) { + fields.add(parser.text()); + } else { + throw new IllegalArgumentException("unexpected token [" + token + "]"); + } + } + } else { + throw new IllegalArgumentException("unknown field [" + fieldName + "]"); + } + break; + default: + throw new IllegalArgumentException("unexpected token [" + token + "]"); + } + } + } + this.fields = fields.toArray(new String[fields.size()]); + this.indexConstraints = indexConstraints.toArray(new IndexConstraint[indexConstraints.size()]); + } + + private void parseIndexContraints(List indexConstraints, XContentParser parser) throws IOException { + Token token = parser.currentToken(); + assert token == Token.START_OBJECT; + String field = null; + String currentName = null; + for (token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) { + if (token == Token.FIELD_NAME) { + field = currentName = parser.currentName(); + } else if (token == Token.START_OBJECT) { + for (Token fieldToken = parser.nextToken(); fieldToken != Token.END_OBJECT; fieldToken = parser.nextToken()) { + if (fieldToken == Token.FIELD_NAME) { + currentName = parser.currentName(); + } else if (fieldToken == Token.START_OBJECT) { + IndexConstraint.Property property = IndexConstraint.Property.parse(currentName); + Token propertyToken = parser.nextToken(); + if (propertyToken != Token.FIELD_NAME) { + throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); + } + IndexConstraint.Comparison comparison = IndexConstraint.Comparison.parse(parser.currentName()); + propertyToken = parser.nextToken(); + if (propertyToken.isValue() == false) { + throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); + } + String value = parser.text(); + indexConstraints.add(new IndexConstraint(field, property, comparison, value)); + + propertyToken = parser.nextToken(); + if (propertyToken != Token.END_OBJECT) { + throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); + } + } else { + throw new IllegalArgumentException("unexpected token [" + fieldToken + "]"); + } + } + } else { + throw new IllegalArgumentException("unexpected token [" + token + "]"); + } + } + } + public String level() { return level; } @@ -58,6 +161,9 @@ public class FieldStatsRequest extends BroadcastRequest { if ("cluster".equals(level) == false && "indices".equals(level) == false) { validationException = ValidateActions.addValidationError("invalid level option [" + level + "]", validationException); } + if (fields == null || fields.length == 0) { + validationException = ValidateActions.addValidationError("no fields specified", validationException); + } return validationException; } @@ -65,6 +171,11 @@ public class FieldStatsRequest extends BroadcastRequest { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); fields = in.readStringArray(); + int size = in.readVInt(); + indexConstraints = new IndexConstraint[size]; + for (int i = 0; i < size; i++) { + indexConstraints[i] = new IndexConstraint(in); + } level = in.readString(); } @@ -72,6 +183,14 @@ public class FieldStatsRequest extends BroadcastRequest { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArrayNullable(fields); + out.writeVInt(indexConstraints.length); + for (IndexConstraint indexConstraint : indexConstraints) { + out.writeString(indexConstraint.getField()); + out.writeByte(indexConstraint.getProperty().getId()); + out.writeByte(indexConstraint.getComparison().getId()); + out.writeString(indexConstraint.getValue()); + } out.writeString(level); } + } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java index 862f24291fa..c5270a7e0bc 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java @@ -31,7 +31,12 @@ public class FieldStatsRequestBuilder extends BroadcastOperationRequestBuilder fields = new HashSet<>(); + fields.addAll(Arrays.asList(request.getFields())); + for (IndexConstraint indexConstraint : request.getIndexConstraints()) { + fields.add(indexConstraint.getField()); + } + this.fields = fields.toArray(new String[fields.size()]); } public String[] getFields() { diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java b/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java new file mode 100644 index 00000000000..2493e34204d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.fieldstats; + +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; +import java.util.Locale; + +public class IndexConstraint { + + private final String field; + private final Property property; + private final Comparison comparison; + private final String value; + + IndexConstraint(StreamInput input) throws IOException { + this.field = input.readString(); + this.property = Property.read(input.readByte()); + this.comparison = Comparison.read(input.readByte()); + this.value = input.readString(); + } + + public IndexConstraint(String field, Property property, Comparison comparison, String value) { + this.field = field; + this.property = property; + this.comparison = comparison; + this.value = value; + } + + public String getField() { + return field; + } + + public Comparison getComparison() { + return comparison; + } + + public Property getProperty() { + return property; + } + + public String getValue() { + return value; + } + + public enum Property { + + MIN((byte) 0), + MAX((byte) 1); + + private final byte id; + + Property(byte id) { + this.id = id; + } + + public byte getId() { + return id; + } + + public static Property read(byte id) { + switch (id) { + case 0: + return MIN; + case 1: + return MAX; + default: + throw new IllegalArgumentException("Unknown property [" + id + "]"); + } + } + + public static Property parse(String value) { + value = value.toLowerCase(Locale.ROOT); + switch (value) { + case "min_value": + return MIN; + case "max_value": + return MAX; + default: + throw new IllegalArgumentException("Unknown property [" + value + "]"); + } + } + + } + + public enum Comparison { + + LT((byte) 0), + LTE((byte) 1), + GT((byte) 2), + GTE((byte) 3); + + private final byte id; + + Comparison(byte id) { + this.id = id; + } + + public byte getId() { + return id; + } + + public static Comparison read(byte id) { + switch (id) { + case 0: + return LT; + case 1: + return LTE; + case 2: + return GT; + case 3: + return GTE; + default: + throw new IllegalArgumentException("Unknown comparison [" + id + "]"); + } + } + + public static Comparison parse(String value) { + value = value.toLowerCase(Locale.ROOT); + switch (value) { + case "lt": + return LT; + case "lte": + return LTE; + case "gt": + return GT; + case "gte": + return GTE; + default: + throw new IllegalArgumentException("Unknown comparison [" + value + "]"); + } + } + + } + +} diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java index 0b64f62dec3..f0308d7fb7a 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java @@ -47,10 +47,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReferenceArray; public class TransportFieldStatsTransportAction extends TransportBroadcastAction { @@ -86,7 +83,7 @@ public class TransportFieldStatsTransportAction extends TransportBroadcastAction } else if ("indices".equals(request.level())) { indexName = shardResponse.getIndex(); } else { - // should already have been catched by the FieldStatsRequest#validate(...) + // should already have been caught by the FieldStatsRequest#validate(...) throw new IllegalArgumentException("Illegal level option [" + request.level() + "]"); } @@ -104,7 +101,6 @@ public class TransportFieldStatsTransportAction extends TransportBroadcastAction "trying to merge the field stats of field [" + entry.getKey() + "] from index [" + shardResponse.getIndex() + "] but the field type is incompatible, try to set the 'level' option to 'indices'" ); } - existing.append(entry.getValue()); } else { indexMergedFieldStats.put(entry.getKey(), entry.getValue()); @@ -112,6 +108,28 @@ public class TransportFieldStatsTransportAction extends TransportBroadcastAction } } } + + if (request.getIndexConstraints().length != 0) { + Set fieldStatFields = new HashSet<>(Arrays.asList(request.getFields())); + for (IndexConstraint indexConstraint : request.getIndexConstraints()) { + Iterator>> iterator = indicesMergedFieldStats.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + FieldStats indexConstraintFieldStats = entry.getValue().get(indexConstraint.getField()); + if (indexConstraintFieldStats.match(indexConstraint)) { + // If the field stats didn't occur in the list of fields in the original request we need to remove the + // field stats, because it was never requested and was only needed to validate the index constraint + if (fieldStatFields.contains(indexConstraint.getField()) == false) { + entry.getValue().remove(indexConstraint.getField()); + } + } else { + // The index constraint didn't match, so we remove all the field stats of the index we're checking + iterator.remove(); + } + } + } + } + return new FieldStatsResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures, indicesMergedFieldStats); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java index 6850b8b91ec..647728b24ab 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; import java.util.Map; @@ -51,12 +52,20 @@ public class RestFieldStatsAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception { + if (RestActions.hasBodyContent(request) && request.hasParam("fields")) { + throw new IllegalArgumentException("can't specify a request body and [fields] request parameter, either specify a request body or the [fields] request parameter"); + } + final FieldStatsRequest fieldStatsRequest = new FieldStatsRequest(); fieldStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); fieldStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, fieldStatsRequest.indicesOptions())); - fieldStatsRequest.fields(Strings.splitStringByCommaToArray(request.param("fields"))); fieldStatsRequest.level(request.param("level", FieldStatsRequest.DEFAULT_LEVEL)); + if (RestActions.hasBodyContent(request)) { + fieldStatsRequest.source(RestActions.getRestContent(request)); + } else { + fieldStatsRequest.setFields(Strings.splitStringByCommaToArray(request.param("fields"))); + } client.fieldStats(fieldStatsRequest, new RestBuilderListener(channel) { @Override diff --git a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTest.java b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTest.java new file mode 100644 index 00000000000..e8521c7d502 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.fieldstats; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.test.ElasticsearchTestCase; + +import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.*; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN; +import static org.hamcrest.Matchers.equalTo; + +public class FieldStatsRequestTest extends ElasticsearchTestCase { + + public void testFieldsParsing() throws Exception { + byte[] data = Streams.copyToBytesFromClasspath("/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json"); + FieldStatsRequest request = new FieldStatsRequest(); + request.source(new BytesArray(data)); + + assertThat(request.getFields().length, equalTo(5)); + assertThat(request.getFields()[0], equalTo("field1")); + assertThat(request.getFields()[1], equalTo("field2")); + assertThat(request.getFields()[2], equalTo("field3")); + assertThat(request.getFields()[3], equalTo("field4")); + assertThat(request.getFields()[4], equalTo("field5")); + + assertThat(request.getIndexConstraints().length, equalTo(6)); + assertThat(request.getIndexConstraints()[0].getField(), equalTo("field2")); + assertThat(request.getIndexConstraints()[0].getValue(), equalTo("9")); + assertThat(request.getIndexConstraints()[0].getProperty(), equalTo(MAX)); + assertThat(request.getIndexConstraints()[0].getComparison(), equalTo(GTE)); + assertThat(request.getIndexConstraints()[1].getField(), equalTo("field3")); + assertThat(request.getIndexConstraints()[1].getValue(), equalTo("5")); + assertThat(request.getIndexConstraints()[1].getProperty(), equalTo(MIN)); + assertThat(request.getIndexConstraints()[1].getComparison(), equalTo(GT)); + assertThat(request.getIndexConstraints()[2].getField(), equalTo("field4")); + assertThat(request.getIndexConstraints()[2].getValue(), equalTo("a")); + assertThat(request.getIndexConstraints()[2].getProperty(), equalTo(MIN)); + assertThat(request.getIndexConstraints()[2].getComparison(), equalTo(GTE)); + assertThat(request.getIndexConstraints()[3].getField(), equalTo("field4")); + assertThat(request.getIndexConstraints()[3].getValue(), equalTo("g")); + assertThat(request.getIndexConstraints()[3].getProperty(), equalTo(MAX)); + assertThat(request.getIndexConstraints()[3].getComparison(), equalTo(LTE)); + assertThat(request.getIndexConstraints()[4].getField(), equalTo("field5")); + assertThat(request.getIndexConstraints()[4].getValue(), equalTo("2")); + assertThat(request.getIndexConstraints()[4].getProperty(), equalTo(MAX)); + assertThat(request.getIndexConstraints()[4].getComparison(), equalTo(GT)); + assertThat(request.getIndexConstraints()[5].getField(), equalTo("field5")); + assertThat(request.getIndexConstraints()[5].getValue(), equalTo("9")); + assertThat(request.getIndexConstraints()[5].getProperty(), equalTo(MAX)); + assertThat(request.getIndexConstraints()[5].getComparison(), equalTo(LT)); + } + +} diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationTests.java index 97231bec442..d17c202f960 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationTests.java @@ -22,18 +22,19 @@ package org.elasticsearch.fieldstats; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.fieldstats.FieldStatsResponse; +import org.elasticsearch.action.fieldstats.IndexConstraint; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.*; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; /** */ @@ -205,7 +206,118 @@ public class FieldStatsIntegrationTests extends ElasticsearchIntegrationTest { assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo("b")); } - private void indexRange(String index, long from, long to) throws ExecutionException, InterruptedException { + public void testFieldStatsFiltering() throws Exception { + assertAcked(prepareCreate("test1").addMapping( + "test", "value", "type=long" + )); + assertAcked(prepareCreate("test2").addMapping( + "test", "value", "type=long" + )); + assertAcked(prepareCreate("test3").addMapping( + "test", "value", "type=long" + )); + ensureGreen("test1", "test2", "test3"); + + indexRange("test1", -10, 100); + indexRange("test2", 101, 200); + indexRange("test3", 201, 300); + + FieldStatsResponse response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "200"), new IndexConstraint("value", MAX , LTE, "300")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MAX, LTE, "200")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(Long.toString(-10))); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(Long.toString(100))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "100")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "-20"), new IndexConstraint("value", MAX, LT, "-10")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "-100"), new IndexConstraint("value", MAX, LTE, "-20")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "100"), new IndexConstraint("value", MAX, LTE, "200")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "150"), new IndexConstraint("value", MAX, LTE, "300")) + .setLevel("indices") + .get(); + assertAllSuccessful(response); + assertThat(response.getAllFieldStats(), nullValue()); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + } + + public void testIncompatibleFilter() throws Exception { + assertAcked(prepareCreate("test1").addMapping( + "test", "value", "type=long" + )); + indexRange("test1", -10, 100); + try { + client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MAX, LTE, "abc")) + .setLevel("indices") + .get(); + fail("exception should have been thrown, because value abc is incompatible"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("For input string: \"abc\"")); + } + } + + private void indexRange(String index, long from, long to) throws Exception { List requests = new ArrayList<>(); for (long value = from; value <= to; value++) { requests.add(client().prepareIndex(index, "test").setSource("value", value)); diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java index 7b6a51982ca..68d136d1cb1 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.fieldstats; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.fieldstats.FieldStatsResponse; +import org.elasticsearch.action.fieldstats.IndexConstraint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ElasticsearchSingleNodeTest; @@ -28,6 +29,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.*; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX; +import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN; import static org.hamcrest.Matchers.*; /** @@ -191,4 +195,168 @@ public class FieldStatsTests extends ElasticsearchSingleNodeTest { assertThat(result.getIndicesMergedFieldStats().get("_all").get("field1").getMaxValue(), equalTo("b")); } + public void testNumberFiltering() { + createIndex("test1", Settings.EMPTY, "type", "value", "type=long"); + client().prepareIndex("test1", "test").setSource("value", 1).get(); + createIndex("test2", Settings.EMPTY, "type", "value", "type=long"); + client().prepareIndex("test2", "test").setSource("value", 3).get(); + client().admin().indices().prepareRefresh().get(); + + FieldStatsResponse response = client().prepareFieldStats() + .setFields("value") + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "-1"), new IndexConstraint("value", MAX, LTE, "0")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "0"), new IndexConstraint("value", MAX, LT, "1")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "0"), new IndexConstraint("value", MAX, LTE, "1")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "1"), new IndexConstraint("value", MAX, LTE, "2")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "1"), new IndexConstraint("value", MAX, LTE, "2")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "2"), new IndexConstraint("value", MAX, LTE, "3")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "3"), new IndexConstraint("value", MAX, LTE, "4")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "3"), new IndexConstraint("value", MAX, LTE, "4")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "1"), new IndexConstraint("value", MAX, LTE, "3")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "1"), new IndexConstraint("value", MAX, LT, "3")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + } + + public void testDateFiltering() { + createIndex("test1", Settings.EMPTY, "type", "value", "type=date"); + client().prepareIndex("test1", "test").setSource("value", "2014-01-01T00:00:00.000Z").get(); + createIndex("test2", Settings.EMPTY, "type", "value", "type=date"); + client().prepareIndex("test2", "test").setSource("value", "2014-01-02T00:00:00.000Z").get(); + client().admin().indices().prepareRefresh().get(); + + FieldStatsResponse response = client().prepareFieldStats() + .setFields("value") + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "2013-12-30T00:00:00.000Z"), new IndexConstraint("value", MAX, LTE, "2013-12-31T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "2013-12-31T00:00:00.000Z"), new IndexConstraint("value", MAX, LTE, "2014-01-01T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "2014-01-01T00:00:00.000Z"), new IndexConstraint("value", MAX, LTE, "2014-01-02T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, "2014-01-02T00:00:00.000Z"), new IndexConstraint("value", MAX, LTE, "2014-01-03T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "2014-01-01T23:00:00.000Z"), new IndexConstraint("value", MAX, LTE, "2014-01-02T01:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "2014-01-01T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MAX, LTE, "2014-01-02T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + } + } \ No newline at end of file diff --git a/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json b/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json new file mode 100644 index 00000000000..525a5692122 --- /dev/null +++ b/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json @@ -0,0 +1,33 @@ +{ + "fields" : [ + "field1", "field2", "field3", "field4", "field5" + ], + "index_constraints": { + "field2": { + "max_value" : { + "gte": 9 + } + }, + "field3": { + "min_value" : { + "gt": 5 + } + }, + "field4": { + "min_value" : { + "gte": "a" + }, + "max_value" : { + "lte": "g" + } + }, + "field5": { + "max_value" : { + "gt": 2 + }, + "max_value" : { + "lt": 9 + } + } + } +} \ No newline at end of file diff --git a/docs/reference/search/field-stats.asciidoc b/docs/reference/search/field-stats.asciidoc index 097f93962ba..5e7b08c7a79 100644 --- a/docs/reference/search/field-stats.asciidoc +++ b/docs/reference/search/field-stats.asciidoc @@ -33,6 +33,17 @@ Supported request options: `level`:: Defines if field stats should be returned on a per index level or on a cluster wide level. Valid values are `indices` and `cluster` (default). +Alternatively the `fields` option can also be defined in the request body: + +[source,js] +-------------------------------------------------- +curl -XPOST 'http://localhost:9200/_field_stats?level=indices' -d '{ + "fields" : ["rating"] +}' +-------------------------------------------------- + +This is equivalent to the previous request. + [float] === Field statistics @@ -205,4 +216,46 @@ GET /_field_stats?fields=rating,answer_count,creation_date,display_name&level=in <1> The `stack` key means it contains all field stats for the `stack` index. +[float] +=== Field stats index constraints + +Field stats index constraints allows to omit all field stats for indices that don't match with the constraint. An index +constraint can exclude indices' field stats based on the `min_value` and `max_value` statistic. This option is only +useful if the `level` option is set to `indices`. + +For example index constraints can be useful to find out the min and max value of a particular property of your data in +a time based scenario. The following request only returns field stats for the `answer_count` property for indices +holding questions created in the year 2014: + +[source,js] +-------------------------------------------------- +curl -XPOST 'http://localhost:9200/_field_stats?level=indices' -d '{ + "fields" : ["answer_count"] <1> + "index_constraints" : { <2> + "creation_date" : { <3> + "min_value" : { <4> + "gte" : "2014-01-01T00:00:00.000Z", + }, + "max_value" : { + "lt" : "2015-01-01T00:00:00.000Z" + } + } + } +}' +-------------------------------------------------- + +<1> The fields to compute and return field stats for. +<2> The set index constraints. Note that index constrains can be defined for fields that aren't defined in the `fields` option. +<3> Index constraints for the field `creation_date`. +<4> An index constraint on the `min_value` property of a field statistic. + +For a field, index constraints can be defined on the `min_value` statistic, `max_value` statistic or both. +Each index constraint support the following comparisons: + +[horizontal] +`gte`:: Greater-than or equal to +`gt`:: Greater-than +`lte`:: Less-than or equal to +`lt`:: Less-than + ================================================== diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json index e3c5e6d45df..c7e95dd683e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_stats.json @@ -41,6 +41,9 @@ } } }, - "body": null + "body": { + "description": "Field json objects containing the name and optionally a range to filter out indices result, that have results outside the defined bounds", + "required": false + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml index 61a575603da..a7d5f05f781 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml @@ -50,3 +50,44 @@ - match: { indices.test_1.fields.number.doc_count: 1 } - match: { indices.test_1.fields.number.min_value: 123 } - match: { indices.test_1.fields.number.max_value: 123 } + +--- +"Field stats with filtering": + - do: + index: + index: test_1 + type: test + id: id_1 + body: { foo: "bar", number: 123 } + + - do: + indices.refresh: {} + + - do: + field_stats: + level: indices + index: test_1 + body: { fields: ["foo"], index_constraints: { number: { min_value : { gte: 100 } } }} + + - match: { indices.test_1.fields.foo.max_doc: 1 } + - match: { indices.test_1.fields.foo.doc_count: 1 } + - match: { indices.test_1.fields.foo.min_value: "bar" } + - match: { indices.test_1.fields.foo.max_value: "bar" } + - is_false: indices.test_1.fields.number + + - do: + field_stats: + level: indices + index: test_1 + body: { fields: ["foo"], index_constraints : { number: { min_value : { gte: 200} } }} + + - is_false: indices.test_1 + +--- +"Field stats both source and fields": + - do: + catch: request + field_stats: + index: test_1 + fields : ["foo"] + body: { fields: ["foo"]}