honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Geo: move indexShape to AbstractGeometryFieldMapper.Indexer (#44979) 

Move indexShape functionality into AbstractGeometryFieldMapper to make
it more unit testable.

Relates to #43644
This commit is contained in:
Igor Motov 2019-07-30 12:36:39 -04:00
parent e4b7ae211b
commit 1a1bb4707d
11 changed files with 171 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.query.QueryShardException;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -77,6 +78,7 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
Class<Processed> processedClass();
List<IndexableField> indexShape(ParseContext context, Processed shape);
}
/**
@ -395,8 +397,6 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
return ((AbstractGeometryFieldType)fieldType).orientation();
}
protected abstract void indexShape(ParseContext context, Processed shape);
/** parsing logic for geometry indexing */
@Override
public void parse(ParseContext context) throws IOException {
@ -413,7 +413,13 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
}
shape = geometryIndexer.prepareForIndexing(geometry);
}
indexShape(context, shape);
List<IndexableField> fields = new ArrayList<>();
fields.addAll(geometryIndexer.indexShape(context, shape));
createFieldNamesField(context, fields);
for (IndexableField field : fields) {
context.doc().add(field);
}
} catch (Exception e) {
if (ignoreMalformed.value() == false) {
throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),

129
server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View File

@ -18,29 +18,14 @@
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.geo.geometry.Circle;
import org.elasticsearch.geo.geometry.Geometry;
import org.elasticsearch.geo.geometry.GeometryCollection;
import org.elasticsearch.geo.geometry.GeometryVisitor;
import org.elasticsearch.geo.geometry.LinearRing;
import org.elasticsearch.geo.geometry.MultiLine;
import org.elasticsearch.geo.geometry.MultiPoint;
import org.elasticsearch.geo.geometry.MultiPolygon;
import org.elasticsearch.geo.geometry.Point;
import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;
import java.util.ArrayList;
import java.util.Arrays;
/**
* FieldMapper for indexing {@link LatLonShape}s.
* <p>
@ -80,12 +65,14 @@ public class GeoShapeFieldMapper extends AbstractGeometryFieldMapper<Geometry, G
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
GeometryParser geometryParser = new GeometryParser(orientation == ShapeBuilder.Orientation.RIGHT, coerce(context).value(),
ignoreZValue().value());
GeoShapeFieldType fieldType = (GeoShapeFieldType)fieldType();
boolean orientation = fieldType.orientation == ShapeBuilder.Orientation.RIGHT;
((GeoShapeFieldType)fieldType()).setGeometryIndexer(new GeoShapeIndexer(orientation == ShapeBuilder.Orientation.RIGHT));
((GeoShapeFieldType)fieldType()).setGeometryParser( (parser, mapper) -> geometryParser.parse(parser));
((GeoShapeFieldType)fieldType()).setGeometryQueryBuilder(new VectorGeoShapeQueryProcessor());
GeometryParser geometryParser = new GeometryParser(orientation, coerce(context).value(), ignoreZValue().value());
fieldType.setGeometryIndexer(new GeoShapeIndexer(orientation, fieldType.name()));
fieldType.setGeometryParser( (parser, mapper) -> geometryParser.parse(parser));
fieldType.setGeometryQueryBuilder(new VectorGeoShapeQueryProcessor());
}
}
@ -107,11 +94,6 @@ public class GeoShapeFieldMapper extends AbstractGeometryFieldMapper<Geometry, G
public String typeName() {
return CONTENT_TYPE;
}
@Override
protected Indexer<Geometry, Geometry> geometryIndexer() {
return new GeoShapeIndexer(orientation == ShapeBuilder.Orientation.RIGHT);
}
}
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@ -127,103 +109,6 @@ public class GeoShapeFieldMapper extends AbstractGeometryFieldMapper<Geometry, G
return (GeoShapeFieldType) super.fieldType();
}
@Override
protected void indexShape(ParseContext context, Geometry luceneShape) {
luceneShape.visit(new LuceneGeometryIndexer(context));
}
private class LuceneGeometryIndexer implements GeometryVisitor<Void, RuntimeException> {
private ParseContext context;
private LuceneGeometryIndexer(ParseContext context) {
this.context = context;
}
@Override
public Void visit(Circle circle) {
throw new IllegalArgumentException("invalid shape type found [Circle] while indexing shape");
}
@Override
public Void visit(GeometryCollection<?> collection) {
for (Geometry geometry : collection) {
geometry.visit(this);
}
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Line line) {
indexFields(context, LatLonShape.createIndexableFields(name(), new Line(line.getLats(), line.getLons())));
return null;
}
@Override
public Void visit(LinearRing ring) {
throw new IllegalArgumentException("invalid shape type found [LinearRing] while indexing shape");
}
@Override
public Void visit(MultiLine multiLine) {
for (org.elasticsearch.geo.geometry.Line line : multiLine) {
visit(line);
}
return null;
}
@Override
public Void visit(MultiPoint multiPoint) {
for(Point point : multiPoint) {
visit(point);
}
return null;
}
@Override
public Void visit(MultiPolygon multiPolygon) {
for(org.elasticsearch.geo.geometry.Polygon polygon : multiPolygon) {
visit(polygon);
}
return null;
}
@Override
public Void visit(Point point) {
indexFields(context, LatLonShape.createIndexableFields(name(), point.getLat(), point.getLon()));
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Polygon polygon) {
indexFields(context, LatLonShape.createIndexableFields(name(), toLucenePolygon(polygon)));
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Rectangle r) {
Polygon p = new Polygon(new double[]{r.getMinLat(), r.getMinLat(), r.getMaxLat(), r.getMaxLat(), r.getMinLat()},
new double[]{r.getMinLon(), r.getMaxLon(), r.getMaxLon(), r.getMinLon(), r.getMinLon()});
indexFields(context, LatLonShape.createIndexableFields(name(), p));
return null;
}
}
public static Polygon toLucenePolygon(org.elasticsearch.geo.geometry.Polygon polygon) {
Polygon[] holes = new Polygon[polygon.getNumberOfHoles()];
for(int i = 0; i<holes.length; i++) {
holes[i] = new Polygon(polygon.getHole(i).getLats(), polygon.getHole(i).getLons());
}
return new Polygon(polygon.getPolygon().getLats(), polygon.getPolygon().getLons(), holes);
}
private void indexFields(ParseContext context, Field[] fields) {
ArrayList<IndexableField> flist = new ArrayList<>(Arrays.asList(fields));
createFieldNamesField(context, flist);
for (IndexableField f : flist) {
context.doc().add(f);
}
}
@Override
protected String contentType() {
return CONTENT_TYPE;

109
server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.geo.geometry.Circle;
import org.elasticsearch.geo.geometry.Geometry;
@ -60,8 +62,11 @@ public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexe
private final boolean orientation;
public GeoShapeIndexer(boolean orientation) {
private final String name;
public GeoShapeIndexer(boolean orientation, String name) {
this.orientation = orientation;
this.name = name;
}
public Geometry prepareForIndexing(Geometry geometry) {
@ -181,6 +186,13 @@ public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexe
return Geometry.class;
}
@Override
public List<IndexableField> indexShape(ParseContext context, Geometry shape) {
LuceneGeometryIndexer visitor = new LuceneGeometryIndexer(name);
shape.visit(visitor);
return visitor.fields();
}
/**
* Calculate the intersection of a line segment and a vertical dateline.
*
@ -936,4 +948,99 @@ public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexe
return points;
}
private static class LuceneGeometryIndexer implements GeometryVisitor<Void, RuntimeException> {
private List<IndexableField> fields = new ArrayList<>();
private String name;
private LuceneGeometryIndexer(String name) {
this.name = name;
}
List<IndexableField> fields() {
return fields;
}
@Override
public Void visit(Circle circle) {
throw new IllegalArgumentException("invalid shape type found [Circle] while indexing shape");
}
@Override
public Void visit(GeometryCollection<?> collection) {
for (Geometry geometry : collection) {
geometry.visit(this);
}
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Line line) {
addFields(LatLonShape.createIndexableFields(name, new org.apache.lucene.geo.Line(line.getLats(), line.getLons())));
return null;
}
@Override
public Void visit(LinearRing ring) {
throw new IllegalArgumentException("invalid shape type found [LinearRing] while indexing shape");
}
@Override
public Void visit(MultiLine multiLine) {
for (org.elasticsearch.geo.geometry.Line line : multiLine) {
visit(line);
}
return null;
}
@Override
public Void visit(MultiPoint multiPoint) {
for(Point point : multiPoint) {
visit(point);
}
return null;
}
@Override
public Void visit(MultiPolygon multiPolygon) {
for(org.elasticsearch.geo.geometry.Polygon polygon : multiPolygon) {
visit(polygon);
}
return null;
}
@Override
public Void visit(Point point) {
addFields(LatLonShape.createIndexableFields(name, point.getLat(), point.getLon()));
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Polygon polygon) {
addFields(LatLonShape.createIndexableFields(name, toLucenePolygon(polygon)));
return null;
}
@Override
public Void visit(org.elasticsearch.geo.geometry.Rectangle r) {
org.apache.lucene.geo.Polygon p = new org.apache.lucene.geo.Polygon(
new double[]{r.getMinLat(), r.getMinLat(), r.getMaxLat(), r.getMaxLat(), r.getMinLat()},
new double[]{r.getMinLon(), r.getMaxLon(), r.getMaxLon(), r.getMinLon(), r.getMinLon()});
addFields(LatLonShape.createIndexableFields(name, p));
return null;
}
private void addFields(IndexableField[] fields) {
this.fields.addAll(Arrays.asList(fields));
}
}
public static org.apache.lucene.geo.Polygon toLucenePolygon(org.elasticsearch.geo.geometry.Polygon polygon) {
org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[polygon.getNumberOfHoles()];
for(int i = 0; i<holes.length; i++) {
holes[i] = new org.apache.lucene.geo.Polygon(polygon.getHole(i).getLats(), polygon.getHole(i).getLons());
}
return new org.apache.lucene.geo.Polygon(polygon.getPolygon().getLats(), polygon.getPolygon().getLons(), holes);
}
}

36
server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
@ -36,7 +35,6 @@ import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.ShapesAvailability;
import org.elasticsearch.common.geo.SpatialStrategy;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation;
import org.elasticsearch.common.geo.parsers.ShapeParser;
@ -47,13 +45,9 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor;
import org.locationtech.spatial4j.shape.Point;
import org.locationtech.spatial4j.shape.Shape;
import org.locationtech.spatial4j.shape.jts.JtsGeometry;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@ -272,7 +266,7 @@ public class LegacyGeoShapeFieldMapper extends AbstractGeometryFieldMapper<Shape
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
fieldType().setGeometryIndexer(new LegacyGeoShapeIndexer());
fieldType().setGeometryIndexer(new LegacyGeoShapeIndexer(fieldType()));
fieldType().setGeometryParser(ShapeParser::parse);
fieldType().setGeometryQueryBuilder(new LegacyGeoShapeQueryProcessor(fieldType()));
@ -489,34 +483,6 @@ public class LegacyGeoShapeFieldMapper extends AbstractGeometryFieldMapper<Shape
return (GeoShapeFieldType) super.fieldType();
}
@Override
protected void indexShape(ParseContext context, Shape shape) {
if (fieldType().pointsOnly() == true) {
// index configured for pointsOnly
if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) {
// MULTIPOINT data: index each point separately
@SuppressWarnings("unchecked") List<Shape> shapes = ((XShapeCollection) shape).getShapes();
for (Shape s : shapes) {
doIndexShape(context, s);
}
return;
} else if (shape instanceof Point == false) {
throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a "
+ ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass())
+ " was found");
}
}
doIndexShape(context, shape);
}
private void doIndexShape(ParseContext context, Shape shape) {
List<IndexableField> fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape)));
createFieldNamesField(context, fields);
for (IndexableField field : fields) {
context.doc().add(field);
}
}
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);

37
server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java
View File

@ -19,10 +19,26 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.locationtech.spatial4j.shape.Point;
import org.locationtech.spatial4j.shape.Shape;
import org.locationtech.spatial4j.shape.jts.JtsGeometry;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class LegacyGeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<ShapeBuilder<?, ?, ?>, Shape> {
private LegacyGeoShapeFieldMapper.GeoShapeFieldType fieldType;
public LegacyGeoShapeIndexer(LegacyGeoShapeFieldMapper.GeoShapeFieldType fieldType) {
this.fieldType = fieldType;
}
@Override
public Shape prepareForIndexing(ShapeBuilder<?, ?, ?> shapeBuilder) {
return shapeBuilder.buildS4J();
@ -32,4 +48,25 @@ public class LegacyGeoShapeIndexer implements AbstractGeometryFieldMapper.Indexe
public Class<Shape> processedClass() {
return Shape.class;
}
@Override
public List<IndexableField> indexShape(ParseContext context, Shape shape) {
if (fieldType.pointsOnly() == true) {
// index configured for pointsOnly
if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) {
// MULTIPOINT data: index each point separately
@SuppressWarnings("unchecked") List<Shape> shapes = ((XShapeCollection) shape).getShapes();
List<IndexableField> fields = new ArrayList<>();
for (Shape s : shapes) {
fields.addAll(Arrays.asList(fieldType.defaultPrefixTreeStrategy().createIndexableFields(s)));
}
return fields;
} else if (shape instanceof Point == false) {
throw new MapperParsingException("[{" + fieldType.name() + "}] is configured for points only but a "
+ ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass())
+ " was found");
}
}
return Arrays.asList(fieldType.defaultPrefixTreeStrategy().createIndexableFields(shape));
}
}

4
server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
View File

@ -43,7 +43,7 @@ import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
import org.elasticsearch.index.mapper.GeoShapeIndexer;
import org.elasticsearch.index.mapper.MappedFieldType;
import static org.elasticsearch.index.mapper.GeoShapeFieldMapper.toLucenePolygon;
import static org.elasticsearch.index.mapper.GeoShapeIndexer.toLucenePolygon;
public class VectorGeoShapeQueryProcessor implements AbstractGeometryFieldMapper.QueryProcessor {
@ -59,7 +59,7 @@ public class VectorGeoShapeQueryProcessor implements AbstractGeometryFieldMapper
}
protected Query getVectorQueryFromShape(Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
GeoShapeIndexer geometryIndexer = new GeoShapeIndexer(true);
GeoShapeIndexer geometryIndexer = new GeoShapeIndexer(true, fieldName);
Geometry processedShape = geometryIndexer.prepareForIndexing(queryShape);

2
server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
View File

@ -67,7 +67,7 @@ abstract class BaseGeoParsingTestCase extends ESTestCase {
} else {
GeometryParser geometryParser = new GeometryParser(true, true, true);
org.elasticsearch.geo.geometry.Geometry shape = geometryParser.parse(parser);
shape = new GeoShapeIndexer(true).prepareForIndexing(shape);
shape = new GeoShapeIndexer(true, "name").prepareForIndexing(shape);
ElasticsearchGeoAssertions.assertEquals(expected, shape);
}
}

2
server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
View File

@ -1426,7 +1426,7 @@ public class GeoJsonShapeParserTests extends BaseGeoParsingTestCase {
public Geometry parse(XContentParser parser) throws IOException, ParseException {
GeometryParser geometryParser = new GeometryParser(true, true, true);
GeoShapeIndexer indexer = new GeoShapeIndexer(true);
GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name");
return indexer.prepareForIndexing(geometryParser.parse(parser));
}
}

2
server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java
View File

@ -471,7 +471,7 @@ public class GeoWKTShapeParserTests extends BaseGeoParsingTestCase {
} else {
GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random());
assertExpected(gcb.buildS4J(), gcb, true);
assertExpected(new GeoShapeIndexer(true).prepareForIndexing(gcb.buildGeometry()), gcb, false);
assertExpected(new GeoShapeIndexer(true, "name").prepareForIndexing(gcb.buildGeometry()), gcb, false);
}
}

6
server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
View File

@ -43,7 +43,7 @@ import java.util.Collections;
public class GeometryIndexerTests extends ESTestCase {
GeoShapeIndexer indexer = new GeoShapeIndexer(true);
GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test");
private static final WellKnownText WKT = new WellKnownText(true, geometry -> {
});
@ -209,13 +209,13 @@ public class GeometryIndexerTests extends ESTestCase {
private Geometry actual(String wkt, boolean rightOrientation) throws IOException, ParseException {
Geometry shape = parseGeometry(wkt, rightOrientation);
return new GeoShapeIndexer(true).prepareForIndexing(shape);
return new GeoShapeIndexer(true, "test").prepareForIndexing(shape);
}
private Geometry actual(XContentBuilder geoJson, boolean rightOrientation) throws IOException, ParseException {
Geometry shape = parseGeometry(geoJson, rightOrientation);
return new GeoShapeIndexer(true).prepareForIndexing(shape);
return new GeoShapeIndexer(true, "test").prepareForIndexing(shape);
}
private Geometry parseGeometry(String wkt, boolean rightOrientation) throws IOException, ParseException {

2
server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
View File

@ -779,6 +779,6 @@ public class ShapeBuilderTests extends ESTestCase {
}
public Object buildGeometry(ShapeBuilder<?, ?, ?> builder) {
return new GeoShapeIndexer(true).prepareForIndexing(builder.buildGeometry());
return new GeoShapeIndexer(true, "name").prepareForIndexing(builder.buildGeometry());
}
}