From 1d6b2f074ef681e2a26989ba98618f465ada3c0f Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 11 May 2020 17:02:40 -0400 Subject: [PATCH] [Transform] adds geotile_grid support in group_by (#56514) (#56549) This adds support for grouping by geo points. This uses the agg [geotile_grid](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-geotilegrid-aggregation.html). I am opting to store the tile results of group_by as a `geo_shape` so that users can query the results. Additionally, the shapes could be visualized and filtered in the kibana maps app. relates to https://github.com/elastic/elasticsearch/issues/56121 --- .../transforms/pivot/GeoTileGroupSource.java | 126 ++++++++++++ .../transforms/pivot/GroupConfig.java | 3 + .../transforms/pivot/SingleGroupSource.java | 3 +- .../pivot/GeoTileGroupSourceTests.java | 67 ++++++ .../transforms/pivot/GroupConfigTests.java | 5 +- .../pivot/hlrc/GeoTileGroupSourceTests.java | 72 +++++++ .../bucket/geogrid/GeoTileUtils.java | 8 + .../transforms/pivot/GeoTileGroupSource.java | 193 ++++++++++++++++++ .../transforms/pivot/GroupConfig.java | 5 + .../transforms/pivot/SingleGroupSource.java | 24 ++- .../pivot/GeoTileGroupSourceTests.java | 49 +++++ .../transforms/pivot/GroupConfigTests.java | 5 +- .../integration/TransformPivotRestIT.java | 68 ++++++ .../pivot/AggregationResultUtils.java | 4 +- .../transforms/pivot/SchemaUtil.java | 48 ++++- 15 files changed, 664 insertions(+), 16 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSourceTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/hlrc/GeoTileGroupSourceTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSource.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java new file mode 100644 index 00000000000..8c696bc9684 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transform.transforms.pivot; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/* + * A geotile_grid aggregation source for group_by + */ +public class GeoTileGroupSource extends SingleGroupSource implements ToXContentObject { + private static final String NAME = "transform_geo_tile_group"; + + private static final ParseField PRECISION = new ParseField("precision"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, (args) -> { + String field = (String) args[0]; + Integer precision = (Integer) args[1]; + GeoBoundingBox boundingBox = (GeoBoundingBox) args[2]; + + return new GeoTileGroupSource(field, precision, boundingBox); + }); + + static { + PARSER.declareString(optionalConstructorArg(), FIELD); + PARSER.declareInt(optionalConstructorArg(), PRECISION); + PARSER.declareField( + optionalConstructorArg(), + (p, context) -> GeoBoundingBox.parseBoundingBox(p), + GeoBoundingBox.BOUNDS_FIELD, + ObjectParser.ValueType.OBJECT + ); + } + private final Integer precision; + private final GeoBoundingBox geoBoundingBox; + + public GeoTileGroupSource(final String field, final Integer precision, final GeoBoundingBox boundingBox) { + super(field, null); + if (precision != null) { + GeoTileUtils.checkPrecisionRange(precision); + } + this.precision = precision; + this.geoBoundingBox = boundingBox; + } + + @Override + public Type getType() { + return Type.GEOTILE_GRID; + } + + public Integer getPrecision() { + return precision; + } + + public GeoBoundingBox getGeoBoundingBox() { + return geoBoundingBox; + } + + public static GeoTileGroupSource fromXContent(final XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + super.innerXContent(builder, params); + if (precision != null) { + builder.field(PRECISION.getPreferredName(), precision); + } + if (geoBoundingBox != null) { + geoBoundingBox.toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final GeoTileGroupSource that = (GeoTileGroupSource) other; + + return Objects.equals(this.field, that.field) + && Objects.equals(this.precision, that.precision) + && Objects.equals(this.geoBoundingBox, that.geoBoundingBox); + } + + @Override + public int hashCode() { + return Objects.hash(field, precision, geoBoundingBox); + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfig.java index 22ee9682998..62cf1f3a15c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfig.java @@ -97,6 +97,9 @@ public class GroupConfig implements ToXContentObject { case "date_histogram": groupSource = DateHistogramGroupSource.fromXContent(parser); break; + case "geotile_grid": + groupSource = GeoTileGroupSource.fromXContent(parser); + break; default: // not a valid group source. Consume up to the dest field end object consumeUntilEndObject(parser, 2); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/SingleGroupSource.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/SingleGroupSource.java index cb117ab5161..ef5edcf3281 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/SingleGroupSource.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/SingleGroupSource.java @@ -36,7 +36,8 @@ public abstract class SingleGroupSource implements ToXContentObject { public enum Type { TERMS, HISTOGRAM, - DATE_HISTOGRAM; + DATE_HISTOGRAM, + GEOTILE_GRID; public String value() { return name().toLowerCase(Locale.ROOT); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSourceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSourceTests.java new file mode 100644 index 00000000000..97776f56611 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSourceTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transform.transforms.pivot; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.function.Predicate; + +public class GeoTileGroupSourceTests extends AbstractXContentTestCase { + + public static GeoTileGroupSource randomGeoTileGroupSource() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoTileGroupSource( + randomBoolean() ? null : randomAlphaOfLength(10), + randomBoolean() ? null : randomIntBetween(1, GeoTileUtils.MAX_ZOOM), + randomBoolean() ? null : new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ) + ); + } + + @Override + protected GeoTileGroupSource createTestInstance() { + return randomGeoTileGroupSource(); + } + + @Override + protected GeoTileGroupSource doParseInstance(XContentParser parser) throws IOException { + return GeoTileGroupSource.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // allow unknown fields in the root of the object only + return field -> !field.isEmpty(); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfigTests.java index 1749c390994..50a60d573d1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/GroupConfigTests.java @@ -55,8 +55,11 @@ public class GroupConfigTests extends AbstractXContentTestCase { groupBy = HistogramGroupSourceTests.randomHistogramGroupSource(); break; case DATE_HISTOGRAM: - default: groupBy = DateHistogramGroupSourceTests.randomDateHistogramGroupSource(); + break; + case GEOTILE_GRID: + default: + groupBy = GeoTileGroupSourceTests.randomGeoTileGroupSource(); } groups.put(targetFieldName, groupBy); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/hlrc/GeoTileGroupSourceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/hlrc/GeoTileGroupSourceTests.java new file mode 100644 index 00000000000..fd56d981031 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/pivot/hlrc/GeoTileGroupSourceTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transform.transforms.pivot.hlrc; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.xpack.core.transform.transforms.pivot.GeoTileGroupSource; + + +import static org.hamcrest.Matchers.equalTo; + +public class GeoTileGroupSourceTests extends AbstractResponseTestCase< + GeoTileGroupSource, + org.elasticsearch.client.transform.transforms.pivot.GeoTileGroupSource> { + + public static GeoTileGroupSource randomGeoTileGroupSource() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoTileGroupSource( + randomBoolean() ? null : randomAlphaOfLength(10), + randomBoolean() ? null : randomIntBetween(1, GeoTileUtils.MAX_ZOOM), + randomBoolean() ? null : new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ) + ); + } + + @Override + protected GeoTileGroupSource createServerTestInstance(XContentType xContentType) { + return randomGeoTileGroupSource(); + } + + @Override + protected org.elasticsearch.client.transform.transforms.pivot.GeoTileGroupSource doParseToClientInstance(XContentParser parser) { + return org.elasticsearch.client.transform.transforms.pivot.GeoTileGroupSource.fromXContent(parser); + } + + @Override + protected void assertInstances( + GeoTileGroupSource serverTestInstance, + org.elasticsearch.client.transform.transforms.pivot.GeoTileGroupSource clientInstance + ) { + assertThat(serverTestInstance.getField(), equalTo(clientInstance.getField())); + assertNull(clientInstance.getScript()); + assertThat(serverTestInstance.getPrecision(), equalTo(clientInstance.getPrecision())); + assertThat(serverTestInstance.getGeoBoundingBox(), equalTo(clientInstance.getGeoBoundingBox())); + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 41e6d6ba08e..909e8b58418 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -251,6 +251,14 @@ public final class GeoTileUtils { return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); } + /** + * Decode a string bucket key in "zoom/x/y" format to a bounding box of the tile corners + */ + public static Rectangle toBoundingBox(String hash) { + int[] hashAsInts = parseHash(hash); + return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); + } + public static Rectangle toBoundingBox(int xTile, int yTile, int precision) { final double tiles = validateZXY(precision, xTile, yTile); final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSource.java new file mode 100644 index 00000000000..3f4b25252af --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSource.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.transform.transforms.pivot; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/* + * A geotile_grid aggregation source for group_by + */ +public class GeoTileGroupSource extends SingleGroupSource { + private static final String NAME = "transform_geo_tile_group"; + + private static final ParseField PRECISION = new ParseField("precision"); + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + + private static ConstructingObjectParser createParser(boolean lenient) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, (args) -> { + String field = (String) args[0]; + Integer precision = (Integer) args[1]; + GeoBoundingBox boundingBox = (GeoBoundingBox) args[2]; + + return new GeoTileGroupSource(field, precision, boundingBox); + }); + parser.declareString(optionalConstructorArg(), FIELD); + parser.declareInt(optionalConstructorArg(), PRECISION); + parser.declareField( + optionalConstructorArg(), + (p, context) -> GeoBoundingBox.parseBoundingBox(p), + GeoBoundingBox.BOUNDS_FIELD, + ObjectParser.ValueType.OBJECT + ); + return parser; + } + private final Integer precision; + private final GeoBoundingBox geoBoundingBox; + + public GeoTileGroupSource(final String field, final Integer precision, final GeoBoundingBox boundingBox) { + super(field, null); + if (precision != null) { + GeoTileUtils.checkPrecisionRange(precision); + } + this.precision = precision; + this.geoBoundingBox = boundingBox; + } + + public GeoTileGroupSource(StreamInput in) throws IOException { + super(in); + precision = in.readOptionalVInt(); + geoBoundingBox = in.readOptionalWriteable(GeoBoundingBox::new); + } + + @Override + public Type getType() { + return Type.GEOTILE_GRID; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalVInt(precision); + out.writeOptionalWriteable(geoBoundingBox); + } + + public Integer getPrecision() { + return precision; + } + + public GeoBoundingBox getGeoBoundingBox() { + return geoBoundingBox; + } + + public static GeoTileGroupSource fromXContent(final XContentParser parser, boolean lenient) { + return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); + } + + @Override + public QueryBuilder getIncrementalBucketUpdateFilterQuery( + Set changedBuckets, + String synchronizationField, + long synchronizationTimestamp + ) { + if (changedBuckets != null && changedBuckets.isEmpty() == false) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + changedBuckets.stream() + .map(GeoTileUtils::toBoundingBox) + .map(this::toGeoQuery) + .forEach(boolQueryBuilder::should); + return boolQueryBuilder; + } + return null; + } + + @Override + public boolean supportsIncrementalBucketUpdate() { + return true; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + super.innerXContent(builder, params); + if (precision != null) { + builder.field(PRECISION.getPreferredName(), precision); + } + if (geoBoundingBox != null) { + geoBoundingBox.toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final GeoTileGroupSource that = (GeoTileGroupSource) other; + + return Objects.equals(this.field, that.field) + && Objects.equals(this.precision, that.precision) + && Objects.equals(this.geoBoundingBox, that.geoBoundingBox); + } + + @Override + public int hashCode() { + return Objects.hash(field, precision, geoBoundingBox); + } + + @Override + public String getMappingType() { + return GeoShapeFieldMapper.CONTENT_TYPE; + } + + @Override + public Object transformBucketKey(Object key) { + assert key instanceof String; + Rectangle rectangle = GeoTileUtils.toBoundingBox(key.toString()); + final Map geoShape = new HashMap<>(); + geoShape.put(ShapeParser.FIELD_TYPE.getPreferredName(), "BBOX"); + geoShape.put( + ShapeParser.FIELD_COORDINATES.getPreferredName(), + Arrays.asList( + new Double[] { rectangle.getMinLon(), rectangle.getMaxLat() }, + new Double[] { rectangle.getMaxLon(), rectangle.getMinLat() } + ) + ); + return geoShape; + } + + private GeoBoundingBoxQueryBuilder toGeoQuery(Rectangle rectangle) { + return QueryBuilders.geoBoundingBoxQuery(field) + .setCorners( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfig.java index e97315edc49..9e1350b877e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfig.java @@ -60,6 +60,8 @@ public class GroupConfig implements Writeable, ToXContentObject { return new HistogramGroupSource(stream); case DATE_HISTOGRAM: return new DateHistogramGroupSource(stream); + case GEOTILE_GRID: + return new GeoTileGroupSource(stream); default: throw new IOException("Unknown group type"); } @@ -177,6 +179,9 @@ public class GroupConfig implements Writeable, ToXContentObject { case DATE_HISTOGRAM: groupSource = DateHistogramGroupSource.fromXContent(parser, lenient); break; + case GEOTILE_GRID: + groupSource = GeoTileGroupSource.fromXContent(parser, lenient); + break; default: throw new ParsingException(parser.getTokenLocation(), "invalid grouping type: " + groupType); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java index 569bf08d820..6f5fd1b8ecc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.transform.transforms.pivot; import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -32,7 +33,8 @@ public abstract class SingleGroupSource implements Writeable, ToXContentObject { public enum Type { TERMS(0), HISTOGRAM(1), - DATE_HISTOGRAM(2); + DATE_HISTOGRAM(2), + GEOTILE_GRID(3); private final byte id; @@ -52,6 +54,8 @@ public abstract class SingleGroupSource implements Writeable, ToXContentObject { return HISTOGRAM; case 2: return DATE_HISTOGRAM; + case 3: + return GEOTILE_GRID; default: throw new IllegalArgumentException("unknown type"); } @@ -154,4 +158,22 @@ public abstract class SingleGroupSource implements Writeable, ToXContentObject { public String toString() { return Strings.toString(this, true, true); } + + /** + * @return The preferred mapping type if it exists. Is nullable. + */ + @Nullable + public String getMappingType() { + return null; + } + + /** + * This will transform a composite aggregation bucket key into the desired format for indexing. + * + * @param key The bucket key for this group source + * @return the transformed bucket key for indexing + */ + public Object transformBucketKey(Object key) { + return key; + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java new file mode 100644 index 00000000000..ff7a5838cd0 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.transform.transforms.pivot; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class GeoTileGroupSourceTests extends AbstractSerializingTestCase { + + public static GeoTileGroupSource randomGeoTileGroupSource() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoTileGroupSource( + randomBoolean() ? null : randomAlphaOfLength(10), + randomBoolean() ? null : randomIntBetween(1, GeoTileUtils.MAX_ZOOM), + randomBoolean() ? null : new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ) + ); + } + + @Override + protected GeoTileGroupSource doParseInstance(XContentParser parser) throws IOException { + return GeoTileGroupSource.fromXContent(parser, false); + } + + @Override + protected GeoTileGroupSource createTestInstance() { + return randomGeoTileGroupSource(); + } + + @Override + protected Reader instanceReader() { + return GeoTileGroupSource::new; + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java index 6f999878bc7..c1cba8f6021 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java @@ -50,8 +50,11 @@ public class GroupConfigTests extends AbstractSerializingTestCase { groupBy = HistogramGroupSourceTests.randomHistogramGroupSource(); break; case DATE_HISTOGRAM: - default: groupBy = DateHistogramGroupSourceTests.randomDateHistogramGroupSource(); + break; + case GEOTILE_GRID: + default: + groupBy = GeoTileGroupSourceTests.randomGeoTileGroupSource(); } source.put(targetFieldName, Collections.singletonMap(type.value(), getSource(groupBy))); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 6f59a0e2775..992d0ad5fd4 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -25,10 +25,14 @@ import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class TransformPivotRestIT extends TransformRestTestCase { @@ -930,6 +934,70 @@ public class TransformPivotRestIT extends TransformRestTestCase { assertEquals((4 + 15), Double.valueOf(latlon[1]), 0.000001); } + @SuppressWarnings("unchecked") + public void testPivotWithGeotileGroupBy() throws Exception { + String transformId = "geotile_grid_group_by"; + String transformIndex = "geotile_grid_pivot_reviews"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex); + + final Request createTransformRequest = createRequestWithAuth( + "PUT", + getTransformEndpoint() + transformId, + BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS + ); + + String config = "{" + + " \"source\": {\"index\":\"" + + REVIEWS_INDEX_NAME + + "\"}," + + " \"dest\": {\"index\":\"" + + transformIndex + + "\"},"; + + config += " \"pivot\": {" + + " \"group_by\": {" + + " \"tile\": {" + + " \"geotile_grid\": {" + + " \"field\": \"location\"," + + " \"precision\": 12" + + " } } }," + + " \"aggregations\": {" + + " \"avg_rating\": {" + + " \"avg\": {" + + " \"field\": \"stars\"" + + " } }," + + " \"boundary\": {" + + " \"geo_bounds\": {\"field\": \"location\"}" + + " } } }" + + "}"; + + createTransformRequest.setJsonEntity(config); + Map createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); + assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); + + startAndWaitForTransform(transformId, transformIndex, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS); + assertTrue(indexExists(transformIndex)); + + // we expect 27 documents as there are that many tiles at this zoom + Map indexStats = getAsMap(transformIndex + "/_stats"); + assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", indexStats)); + + // Verify that the format is sane for the geo grid + Map searchResult = getAsMap(transformIndex + "/_search?size=1"); + Number actual = (Number) ((List) XContentMapValues.extractValue("hits.hits._source.avg_rating", searchResult)).get(0); + assertThat(actual, is(not(nullValue()))); + Map actualObj = (Map) ((List) XContentMapValues.extractValue( + "hits.hits._source.tile", + searchResult + )).get(0); + assertThat(actualObj.get("type"), equalTo("BBOX")); + List> coordinates = (List>) actualObj.get("coordinates"); + assertThat(coordinates, is(not(nullValue()))); + assertThat(coordinates, hasSize(2)); + assertThat(coordinates.get(0), hasSize(2)); + assertThat(coordinates.get(1), hasSize(2)); + } + public void testPivotWithWeightedAvgAgg() throws Exception { String transformId = "weighted_avg_agg_transform"; String transformIndex = "weighted_avg_pivot_reviews"; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index b6cf82c4930..1475cd0babd 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -81,10 +81,10 @@ public final class AggregationResultUtils { // - update documents IDGenerator idGen = new IDGenerator(); - groups.getGroups().keySet().forEach(destinationFieldName -> { + groups.getGroups().forEach((destinationFieldName, singleGroupSource) -> { Object value = bucket.getKey().get(destinationFieldName); idGen.add(destinationFieldName, value); - updateDocument(document, destinationFieldName, value); + updateDocument(document, destinationFieldName, singleGroupSource.transformBucketKey(value)); }); List aggNames = aggregationBuilders.stream().map(AggregationBuilder::getName).collect(Collectors.toList()); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java index bd726b7e00f..44d15a201dc 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.transform.transforms.pivot; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; @@ -15,6 +16,7 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; @@ -68,10 +70,20 @@ public final class SchemaUtil { Map aggregationTypes = new HashMap<>(); // collects the fieldnames and target fieldnames used for grouping Map fieldNamesForGrouping = new HashMap<>(); + // collects the target mapping types used for grouping + Map fieldTypesForGrouping = new HashMap<>(); config.getGroupConfig() .getGroups() - .forEach((destinationFieldName, group) -> { fieldNamesForGrouping.put(destinationFieldName, group.getField()); }); + .forEach((destinationFieldName, group) -> { + // We will always need the field name for the grouping to create the mapping + fieldNamesForGrouping.put(destinationFieldName, group.getField()); + // Sometimes the group config will supply a desired mapping as well + if (group.getMappingType() != null) { + fieldTypesForGrouping.put(destinationFieldName, group.getMappingType()); + } + }); + for (AggregationBuilder agg : config.getAggregationConfig().getAggregatorFactories()) { Tuple, Map> inputAndOutputTypes = Aggregations.getAggregationInputAndOutputTypes(agg); @@ -95,7 +107,13 @@ public final class SchemaUtil { allFieldNames.values().toArray(new String[0]), ActionListener.wrap( sourceMappings -> listener.onResponse( - resolveMappings(aggregationSourceFieldNames, aggregationTypes, fieldNamesForGrouping, sourceMappings) + resolveMappings( + aggregationSourceFieldNames, + aggregationTypes, + fieldNamesForGrouping, + fieldTypesForGrouping, + sourceMappings + ) ), listener::onFailure ) @@ -131,6 +149,7 @@ public final class SchemaUtil { Map aggregationSourceFieldNames, Map aggregationTypes, Map fieldNamesForGrouping, + Map fieldTypesForGrouping, Map sourceMappings ) { Map targetMapping = new HashMap<>(); @@ -140,25 +159,34 @@ public final class SchemaUtil { String sourceMapping = sourceFieldName == null ? null : sourceMappings.get(sourceFieldName); String destinationMapping = Aggregations.resolveTargetMapping(aggregationName, sourceMapping); - logger.debug("Deduced mapping for: [{}], agg type [{}] to [{}]", targetFieldName, aggregationName, destinationMapping); + logger.debug(() -> new ParameterizedMessage( + "Deduced mapping for: [{}], agg type [{}] to [{}]", + targetFieldName, + aggregationName, + destinationMapping + )); if (Aggregations.isDynamicMapping(destinationMapping)) { - logger.debug("Dynamic target mapping set for field [{}] and aggregation [{}]", targetFieldName, aggregationName); + logger.debug(() -> new ParameterizedMessage( + "Dynamic target mapping set for field [{}] and aggregation [{}]", + targetFieldName, + aggregationName + )); } else if (destinationMapping != null) { targetMapping.put(targetFieldName, destinationMapping); } else { - logger.warn("Failed to deduce mapping for [" + targetFieldName + "], fall back to dynamic mapping."); + logger.warn("Failed to deduce mapping for [{}], fall back to dynamic mapping.", targetFieldName); } }); fieldNamesForGrouping.forEach((targetFieldName, sourceFieldName) -> { - String destinationMapping = sourceMappings.get(sourceFieldName); - logger.debug("Deduced mapping for: [{}] to [{}]", targetFieldName, destinationMapping); + String destinationMapping = fieldTypesForGrouping.computeIfAbsent(targetFieldName, (s) -> sourceMappings.get(sourceFieldName)); + logger.debug(() -> new ParameterizedMessage("Deduced mapping for: [{}] to [{}]", targetFieldName, destinationMapping)); if (destinationMapping != null) { targetMapping.put(targetFieldName, destinationMapping); } else { - logger.warn("Failed to deduce mapping for [" + targetFieldName + "], fall back to keyword."); - targetMapping.put(targetFieldName, "keyword"); + logger.warn("Failed to deduce mapping for [{}], fall back to keyword.", targetFieldName); + targetMapping.put(targetFieldName, KeywordFieldMapper.CONTENT_TYPE); } }); @@ -196,7 +224,7 @@ public final class SchemaUtil { // TODO: overwrites types, requires resolve if // types are mixed capabilitiesMap.forEach((name, capability) -> { - logger.trace("Extracted type for [{}] : [{}]", fieldName, capability.getType()); + logger.trace(() -> new ParameterizedMessage("Extracted type for [{}] : [{}]", fieldName, capability.getType())); extractedTypes.put(fieldName, capability.getType()); }); }