From 1ad00d1cebdfc8556cb5beb4a3151caf59f3371c Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 9 Jul 2020 11:41:41 +0200 Subject: [PATCH] Add Support in geo_match enrichment policy for any type of geometry (#59276) geo_match enrichment works currently only with points. This change adds the ability to use any type of geometry. --- .../common/geo/GeometryParser.java | 65 +++++++++++++++++++ .../common/geo/GeometryParserTests.java | 65 +++++++++++++++++++ .../xpack/enrich/EnrichProcessorFactory.java | 6 +- .../xpack/enrich/GeoMatchProcessor.java | 44 ++++--------- .../xpack/enrich/GeoMatchProcessorTests.java | 47 ++++++++++++-- 5 files changed, 188 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java index 84972baf6b7..d39e7752a2d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java @@ -20,16 +20,25 @@ package org.elasticsearch.common.geo; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.MapXContentParser; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.StandardValidator; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownText; import java.io.IOException; import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; /** * An utility class with a geometry parser methods supporting different shape representation formats @@ -38,11 +47,13 @@ public final class GeometryParser { private final GeoJson geoJsonParser; private final WellKnownText wellKnownTextParser; + private final boolean ignoreZValue; public GeometryParser(boolean rightOrientation, boolean coerce, boolean ignoreZValue) { GeometryValidator validator = new StandardValidator(ignoreZValue); geoJsonParser = new GeoJson(rightOrientation, coerce, validator); wellKnownTextParser = new WellKnownText(coerce, validator); + this.ignoreZValue = ignoreZValue; } /** @@ -109,4 +120,58 @@ public final class GeometryParser { } throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates"); } + + /** + * Parses the value as a {@link Geometry}. The following types of values are supported: + *

+ * Object: has to contain either lat and lon or geohash fields + *

+ * String: expected to be in "latitude, longitude" format, a geohash or WKT + *

+ * Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true + *

+ * Json structure: valid geojson definition + */ + public Geometry parseGeometry(Object value) throws ElasticsearchParseException { + if (value instanceof List) { + List values = (List) value; + if (values.size() == 2 && values.get(0) instanceof Number) { + GeoPoint point = GeoUtils.parseGeoPoint(values, ignoreZValue); + return new Point(point.lon(), point.lat()); + } else { + List geometries = new ArrayList<>(values.size()); + for (Object object : values) { + geometries.add(parseGeometry(object)); + } + return new GeometryCollection<>(geometries); + } + } + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("null_value", value), null)) { + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + if (isPoint(value)) { + GeoPoint point = GeoUtils.parseGeoPoint(parser, new GeoPoint(), ignoreZValue); + return new Point(point.lon(), point.lat()); + } else { + return parse(parser); + } + + } catch (IOException | ParseException ex) { + throw new ElasticsearchParseException("error parsing geometry ", ex); + } + } + + private boolean isPoint(Object value) { + // can we do this better? + if (value instanceof Map) { + Map map = (Map) value; + return map.containsKey("lat") && map.containsKey("lon"); + } else if (value instanceof String) { + String string = (String) value; + return Character.isDigit(string.charAt(0)) || string.indexOf('(') == -1; + } + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java index 4c53d40fb75..1503c6b5b81 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java @@ -26,12 +26,18 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + /** * Tests for {@link GeometryParser} */ @@ -173,4 +179,63 @@ public class GeometryParserTests extends ESTestCase { assertEquals("shape must be an object consisting of type and coordinates", ex.getMessage()); } } + + public void testBasics() { + GeometryParser parser = new GeometryParser(true, randomBoolean(), randomBoolean()); + // point + Point expectedPoint = new Point(-122.084110, 37.386637); + testBasics(parser, mapOf("lat", 37.386637, "lon", -122.084110), expectedPoint); + testBasics(parser, "37.386637, -122.084110", expectedPoint); + testBasics(parser, "POINT (-122.084110 37.386637)", expectedPoint); + testBasics(parser, Arrays.asList(-122.084110, 37.386637), expectedPoint); + testBasics(parser, mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), expectedPoint); + // line + Line expectedLine = new Line(new double[] { 0, 1 }, new double[] { 0, 1 }); + testBasics(parser, "LINESTRING(0 0, 1 1)", expectedLine); + testBasics(parser, + mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))), + expectedLine + ); + // polygon + Polygon expectedPolygon = new Polygon(new LinearRing(new double[] { 0, 1, 1, 0, 0 }, new double[] { 0, 0, 1, 1, 0 })); + testBasics(parser, "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon); + testBasics(parser, + mapOf( + "type", + "Polygon", + "coordinates", + Arrays.asList( + Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 0), Arrays.asList(1, 1), Arrays.asList(0, 1), Arrays.asList(0, 0)) + ) + ), + expectedPolygon + ); + // geometry collection + testBasics(parser, + Arrays.asList( + Arrays.asList(-122.084110, 37.386637), + "37.386637, -122.084110", + "POINT (-122.084110 37.386637)", + mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), + mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))), + "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" + ), + new GeometryCollection<>( + Arrays.asList(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon) + ) + ); + expectThrows(ElasticsearchParseException.class, () -> testBasics(parser, "not a geometry", null)); + } + + private void testBasics(GeometryParser parser, Object value, Geometry expected) { + Geometry geometry = parser.parseGeometry(value); + assertEquals(expected, geometry); + } + + private static Map mapOf(K key1, V value1, K key2, V value2) { + Map map = new HashMap<>(); + map.put(key1, value1); + map.put(key2, value2); + return map; + } } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index b89e1485b3b..ec53e67d0f1 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.Processor; @@ -83,6 +84,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer points = new ArrayList<>(); - if (fieldValue instanceof List) { - List values = (List) fieldValue; - if (values.size() == 2 && values.get(0) instanceof Number) { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(values, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } else { - for (Object value : values) { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(value, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } - } - } else { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(fieldValue, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } - final Geometry queryGeometry; - if (points.isEmpty()) { - throw new IllegalArgumentException("no geopoints found"); - } else if (points.size() == 1) { - queryGeometry = points.get(0); - } else { - queryGeometry = new MultiPoint(points); - } + final Geometry queryGeometry = parser.parseGeometry(fieldValue); GeoShapeQueryBuilder shapeQuery = new GeoShapeQueryBuilder(matchField, queryGeometry); shapeQuery.relation(shapeRelation); return shapeQuery; diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java index f762b032413..37fffce0b0f 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java @@ -14,12 +14,16 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.routing.Preference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -51,16 +55,48 @@ import static org.hamcrest.Matchers.nullValue; public class GeoMatchProcessorTests extends ESTestCase { public void testBasics() { + // point Point expectedPoint = new Point(-122.084110, 37.386637); testBasicsForFieldValue(mapOf("lat", 37.386637, "lon", -122.084110), expectedPoint); testBasicsForFieldValue("37.386637, -122.084110", expectedPoint); testBasicsForFieldValue("POINT (-122.084110 37.386637)", expectedPoint); testBasicsForFieldValue(Arrays.asList(-122.084110, 37.386637), expectedPoint); + testBasicsForFieldValue(mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), expectedPoint); + // line + Line expectedLine = new Line(new double[] { 0, 1 }, new double[] { 0, 1 }); + testBasicsForFieldValue("LINESTRING(0 0, 1 1)", expectedLine); testBasicsForFieldValue( - Arrays.asList(Arrays.asList(-122.084110, 37.386637), "37.386637, -122.084110", "POINT (-122.084110 37.386637)"), - new MultiPoint(Arrays.asList(expectedPoint, expectedPoint, expectedPoint)) + mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))), + expectedLine + ); + // polygon + Polygon expectedPolygon = new Polygon(new LinearRing(new double[] { 0, 1, 1, 0, 0 }, new double[] { 0, 0, 1, 1, 0 })); + testBasicsForFieldValue("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon); + testBasicsForFieldValue( + mapOf( + "type", + "Polygon", + "coordinates", + Arrays.asList( + Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 0), Arrays.asList(1, 1), Arrays.asList(0, 1), Arrays.asList(0, 0)) + ) + ), + expectedPolygon + ); + // geometry collection + testBasicsForFieldValue( + Arrays.asList( + Arrays.asList(-122.084110, 37.386637), + "37.386637, -122.084110", + "POINT (-122.084110 37.386637)", + mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), + mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))), + "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" + ), + new GeometryCollection<>( + Arrays.asList(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon) + ) ); - testBasicsForFieldValue("not a point", null); } @@ -78,7 +114,8 @@ public class GeoMatchProcessorTests extends ESTestCase { false, "shape", maxMatches, - ShapeRelation.INTERSECTS + ShapeRelation.INTERSECTS, + ShapeBuilder.Orientation.CCW ); IngestDocument ingestDocument = new IngestDocument( "_index",