honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Support in geo_match enrichment policy for any type of geometry (#59276) 

geo_match enrichment works currently only with points. This change adds the ability to
use any type of geometry.
This commit is contained in:
Ignacio Vera 2020-07-09 11:41:41 +02:00 committed by GitHub
parent c0e0bca84c
commit 1ad00d1ceb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 188 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java
View File

@ -20,16 +20,25 @@
package org.elasticsearch.common.geo;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.MapXContentParser;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.GeometryCollection;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.utils.StandardValidator;
import org.elasticsearch.geometry.utils.GeometryValidator;
import org.elasticsearch.geometry.utils.WellKnownText;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* An utility class with a geometry parser methods supporting different shape representation formats
@ -38,11 +47,13 @@ public final class GeometryParser {
private final GeoJson geoJsonParser;
private final WellKnownText wellKnownTextParser;
private final boolean ignoreZValue;
public GeometryParser(boolean rightOrientation, boolean coerce, boolean ignoreZValue) {
GeometryValidator validator = new StandardValidator(ignoreZValue);
geoJsonParser = new GeoJson(rightOrientation, coerce, validator);
wellKnownTextParser = new WellKnownText(coerce, validator);
this.ignoreZValue = ignoreZValue;
}
/**
@ -109,4 +120,58 @@ public final class GeometryParser {
}
throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates");
}
/**
* Parses the value as a {@link Geometry}. The following types of values are supported:
* <p>
* Object: has to contain either lat and lon or geohash fields
* <p>
* String: expected to be in "latitude, longitude" format, a geohash or WKT
* <p>
* Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true
* <p>
* Json structure: valid geojson definition
*/
public Geometry parseGeometry(Object value) throws ElasticsearchParseException {
if (value instanceof List) {
List<?> values = (List<?>) value;
if (values.size() == 2 && values.get(0) instanceof Number) {
GeoPoint point = GeoUtils.parseGeoPoint(values, ignoreZValue);
return new Point(point.lon(), point.lat());
} else {
List<Geometry> geometries = new ArrayList<>(values.size());
for (Object object : values) {
geometries.add(parseGeometry(object));
}
return new GeometryCollection<>(geometries);
}
}
try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
Collections.singletonMap("null_value", value), null)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value
if (isPoint(value)) {
GeoPoint point = GeoUtils.parseGeoPoint(parser, new GeoPoint(), ignoreZValue);
return new Point(point.lon(), point.lat());
} else {
return parse(parser);
}
} catch (IOException | ParseException ex) {
throw new ElasticsearchParseException("error parsing geometry ", ex);
}
}
private boolean isPoint(Object value) {
// can we do this better?
if (value instanceof Map) {
Map<?, ?> map = (Map<?, ?>) value;
return map.containsKey("lat") && map.containsKey("lon");
} else if (value instanceof String) {
String string = (String) value;
return Character.isDigit(string.charAt(0)) || string.indexOf('(') == -1;
}
return false;
}
}

65
server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java
View File

@ -26,12 +26,18 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParseException;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.GeometryCollection;
import org.elasticsearch.geometry.Line;
import org.elasticsearch.geometry.LinearRing;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.Polygon;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* Tests for {@link GeometryParser}
*/
@ -173,4 +179,63 @@ public class GeometryParserTests extends ESTestCase {
assertEquals("shape must be an object consisting of type and coordinates", ex.getMessage());
}
}
public void testBasics() {
GeometryParser parser = new GeometryParser(true, randomBoolean(), randomBoolean());
// point
Point expectedPoint = new Point(-122.084110, 37.386637);
testBasics(parser, mapOf("lat", 37.386637, "lon", -122.084110), expectedPoint);
testBasics(parser, "37.386637, -122.084110", expectedPoint);
testBasics(parser, "POINT (-122.084110 37.386637)", expectedPoint);
testBasics(parser, Arrays.asList(-122.084110, 37.386637), expectedPoint);
testBasics(parser, mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), expectedPoint);
// line
Line expectedLine = new Line(new double[] { 0, 1 }, new double[] { 0, 1 });
testBasics(parser, "LINESTRING(0 0, 1 1)", expectedLine);
testBasics(parser,
mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))),
expectedLine
);
// polygon
Polygon expectedPolygon = new Polygon(new LinearRing(new double[] { 0, 1, 1, 0, 0 }, new double[] { 0, 0, 1, 1, 0 }));
testBasics(parser, "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon);
testBasics(parser,
mapOf(
"type",
"Polygon",
"coordinates",
Arrays.asList(
Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 0), Arrays.asList(1, 1), Arrays.asList(0, 1), Arrays.asList(0, 0))
)
),
expectedPolygon
);
// geometry collection
testBasics(parser,
Arrays.asList(
Arrays.asList(-122.084110, 37.386637),
"37.386637, -122.084110",
"POINT (-122.084110 37.386637)",
mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)),
mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))),
"POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"
),
new GeometryCollection<>(
Arrays.asList(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon)
)
);
expectThrows(ElasticsearchParseException.class, () -> testBasics(parser, "not a geometry", null));
}
private void testBasics(GeometryParser parser, Object value, Geometry expected) {
Geometry geometry = parser.parseGeometry(value);
assertEquals(expected, geometry);
}
private static <K, V> Map<K, V> mapOf(K key1, V value1, K key2, V value2) {
Map<K, V> map = new HashMap<>();
map.put(key1, value1);
map.put(key2, value2);
return map;
}
}

6
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.Processor;
@ -83,6 +84,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
case EnrichPolicy.GEO_MATCH_TYPE:
String relationStr = ConfigurationUtils.readStringProperty(TYPE, tag, config, "shape_relation", "intersects");
ShapeRelation shapeRelation = ShapeRelation.getRelationByName(relationStr);
String orientationStr = ConfigurationUtils.readStringProperty(TYPE, tag, config, "orientation", "CCW");
ShapeBuilder.Orientation orientation = ShapeBuilder.Orientation.fromString(orientationStr);
return new GeoMatchProcessor(
tag,
description,
@ -94,7 +97,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
ignoreMissing,
matchField,
maxMatches,
shapeRelation
shapeRelation,
orientation
);
default:
throw new IllegalArgumentException("unsupported policy type [" + policyType + "]");

44
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java
View File

@ -8,23 +8,20 @@ package org.elasticsearch.xpack.enrich;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.MultiPoint;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.index.query.GeoShapeQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.script.TemplateScript;
import java.util.ArrayList;
import java.util.List;
import java.util.function.BiConsumer;
public final class GeoMatchProcessor extends AbstractEnrichProcessor {
private ShapeRelation shapeRelation;
private final ShapeRelation shapeRelation;
private final GeometryParser parser;
GeoMatchProcessor(
String tag,
@ -37,10 +34,12 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor {
boolean ignoreMissing,
String matchField,
int maxMatches,
ShapeRelation shapeRelation
ShapeRelation shapeRelation,
ShapeBuilder.Orientation orientation
) {
super(tag, description, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches);
this.shapeRelation = shapeRelation;
parser = new GeometryParser(orientation.getAsBoolean(), true, true);
}
/** used in tests **/
@ -55,38 +54,17 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor {
boolean ignoreMissing,
String matchField,
int maxMatches,
ShapeRelation shapeRelation
ShapeRelation shapeRelation,
ShapeBuilder.Orientation orientation
) {
super(tag, description, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches);
this.shapeRelation = shapeRelation;
parser = new GeometryParser(orientation.getAsBoolean(), true, true);
}
@Override
public QueryBuilder getQueryBuilder(Object fieldValue) {
List<Point> points = new ArrayList<>();
if (fieldValue instanceof List) {
List<?> values = (List<?>) fieldValue;
if (values.size() == 2 && values.get(0) instanceof Number) {
GeoPoint geoPoint = GeoUtils.parseGeoPoint(values, true);
points.add(new Point(geoPoint.lon(), geoPoint.lat()));
} else {
for (Object value : values) {
GeoPoint geoPoint = GeoUtils.parseGeoPoint(value, true);
points.add(new Point(geoPoint.lon(), geoPoint.lat()));
}
}
} else {
GeoPoint geoPoint = GeoUtils.parseGeoPoint(fieldValue, true);
points.add(new Point(geoPoint.lon(), geoPoint.lat()));
}
final Geometry queryGeometry;
if (points.isEmpty()) {
throw new IllegalArgumentException("no geopoints found");
} else if (points.size() == 1) {
queryGeometry = points.get(0);
} else {
queryGeometry = new MultiPoint(points);
}
final Geometry queryGeometry = parser.parseGeometry(fieldValue);
GeoShapeQueryBuilder shapeQuery = new GeoShapeQueryBuilder(matchField, queryGeometry);
shapeQuery.relation(shapeRelation);
return shapeQuery;

47
x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java
View File

@ -14,12 +14,16 @@ import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.cluster.routing.Preference;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.MultiPoint;
import org.elasticsearch.geometry.GeometryCollection;
import org.elasticsearch.geometry.Line;
import org.elasticsearch.geometry.LinearRing;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.Polygon;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
@ -51,16 +55,48 @@ import static org.hamcrest.Matchers.nullValue;
public class GeoMatchProcessorTests extends ESTestCase {
public void testBasics() {
// point
Point expectedPoint = new Point(-122.084110, 37.386637);
testBasicsForFieldValue(mapOf("lat", 37.386637, "lon", -122.084110), expectedPoint);
testBasicsForFieldValue("37.386637, -122.084110", expectedPoint);
testBasicsForFieldValue("POINT (-122.084110 37.386637)", expectedPoint);
testBasicsForFieldValue(Arrays.asList(-122.084110, 37.386637), expectedPoint);
testBasicsForFieldValue(mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)), expectedPoint);
// line
Line expectedLine = new Line(new double[] { 0, 1 }, new double[] { 0, 1 });
testBasicsForFieldValue("LINESTRING(0 0, 1 1)", expectedLine);
testBasicsForFieldValue(
Arrays.asList(Arrays.asList(-122.084110, 37.386637), "37.386637, -122.084110", "POINT (-122.084110 37.386637)"),
new MultiPoint(Arrays.asList(expectedPoint, expectedPoint, expectedPoint))
mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))),
expectedLine
);
// polygon
Polygon expectedPolygon = new Polygon(new LinearRing(new double[] { 0, 1, 1, 0, 0 }, new double[] { 0, 0, 1, 1, 0 }));
testBasicsForFieldValue("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon);
testBasicsForFieldValue(
mapOf(
"type",
"Polygon",
"coordinates",
Arrays.asList(
Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 0), Arrays.asList(1, 1), Arrays.asList(0, 1), Arrays.asList(0, 0))
)
),
expectedPolygon
);
// geometry collection
testBasicsForFieldValue(
Arrays.asList(
Arrays.asList(-122.084110, 37.386637),
"37.386637, -122.084110",
"POINT (-122.084110 37.386637)",
mapOf("type", "Point", "coordinates", Arrays.asList(-122.084110, 37.386637)),
mapOf("type", "LineString", "coordinates", Arrays.asList(Arrays.asList(0, 0), Arrays.asList(1, 1))),
"POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"
),
new GeometryCollection<>(
Arrays.asList(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon)
)
);
testBasicsForFieldValue("not a point", null);
}
@ -78,7 +114,8 @@ public class GeoMatchProcessorTests extends ESTestCase {
false,
"shape",
maxMatches,
ShapeRelation.INTERSECTS
ShapeRelation.INTERSECTS,
ShapeBuilder.Orientation.CCW
);
IngestDocument ingestDocument = new IngestDocument(
"_index",