Add ignore_malformed to geo_shape fields

This commit adds ignore_malformed support to geo_shape field types to skip malformed geoJson fields.

closes #23747

core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
@@ -54,6 +53,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED;
+
 /**
  * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
  * <p>
@@ -96,6 +97,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
         public static final Orientation ORIENTATION = Orientation.RIGHT;
         public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d;
         public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
+        public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
 
         public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType();
 
@@ -115,6 +117,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
     public static class Builder extends FieldMapper.Builder<Builder, GeoShapeFieldMapper> {
 
         private Boolean coerce;
+        private Boolean ignoreMalformed;
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -145,6 +148,21 @@ public class GeoShapeFieldMapper extends FieldMapper {
             return Defaults.COERCE;
         }
 
+        public Builder ignoreMalformed(boolean ignoreMalformed) {
+            this.ignoreMalformed = ignoreMalformed;
+            return builder;
+        }
+
+        protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
+            if (ignoreMalformed != null) {
+                return new Explicit<>(ignoreMalformed, true);
+            }
+            if (context.indexSettings() != null) {
+                return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
+            }
+            return Defaults.IGNORE_MALFORMED;
+        }
+
         @Override
         public GeoShapeFieldMapper build(BuilderContext context) {
             GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType;
@@ -154,8 +172,8 @@ public class GeoShapeFieldMapper extends FieldMapper {
             }
             setupFieldType(context);
 
-            return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this,
+            return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(),
-                    context), copyTo);
+                    multiFieldsBuilder.build(this, context), copyTo);
         }
     }
 
@@ -186,6 +204,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
                 } else if (Names.STRATEGY.equals(fieldName)) {
                     builder.fieldType().setStrategyName(fieldNode.toString());
                     iterator.remove();
+                } else if (IGNORE_MALFORMED.equals(fieldName)) {
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
+                    iterator.remove();
                 } else if (Names.COERCE.equals(fieldName)) {
                     builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
                     iterator.remove();
@@ -428,11 +449,13 @@ public class GeoShapeFieldMapper extends FieldMapper {
     }
 
     protected Explicit<Boolean> coerce;
+    protected Explicit<Boolean> ignoreMalformed;
 
-    public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> coerce, Settings indexSettings,
+    public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> ignoreMalformed,
-                               MultiFields multiFields, CopyTo copyTo) {
+                               Explicit<Boolean> coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
         this.coerce = coerce;
+        this.ignoreMalformed = ignoreMalformed;
     }
 
     @Override
@@ -461,7 +484,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
                 context.doc().add(field);
             }
         } catch (Exception e) {
-            throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+            if (ignoreMalformed.value() == false) {
+                throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+            }
         }
         return null;
     }
@@ -478,6 +503,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
         if (gsfm.coerce.explicit()) {
             this.coerce = gsfm.coerce;
         }
+        if (gsfm.ignoreMalformed.explicit()) {
+            this.ignoreMalformed = gsfm.ignoreMalformed;
+        }
     }
 
     @Override
@@ -506,7 +534,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
             builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly());
         }
         if (includeDefaults || coerce.explicit()) {
-            builder.field("coerce", coerce.value());
+            builder.field(Names.COERCE, coerce.value());
+        }
+        if (includeDefaults || ignoreMalformed.explicit()) {
+            builder.field(IGNORE_MALFORMED, ignoreMalformed.value());
         }
     }
 
@@ -514,6 +545,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
         return coerce;
     }
 
+    public Explicit<Boolean> ignoreMalformed() {
+        return ignoreMalformed;
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java

@@ -22,6 +22,7 @@ import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -103,7 +104,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
     }
 
     /**
-     * Test that orientation parameter correctly parses
+     * Test that coerce parameter correctly parses
      */
     public void testCoerceParsing() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
@@ -136,6 +137,41 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
         assertThat(coerce, equalTo(false));
     }
 
+    /**
+     * Test that ignore_malformed parameter correctly parses
+     */
+    public void testIgnoreMalformedParsing() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+            .startObject("properties").startObject("location")
+            .field("type", "geo_shape")
+            .field("ignore_malformed", "true")
+            .endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
+
+        Explicit<Boolean> ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
+        assertThat(ignoreMalformed.value(), equalTo(true));
+
+        // explicit false ignore_malformed test
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+            .startObject("properties").startObject("location")
+            .field("type", "geo_shape")
+            .field("ignore_malformed", "false")
+            .endObject().endObject()
+            .endObject().endObject().string();
+
+        defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        fieldMapper = defaultMapper.mappers().getMapper("location");
+        assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
+
+        ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
+        assertThat(ignoreMalformed.explicit(), equalTo(true));
+        assertThat(ignoreMalformed.value(), equalTo(false));
+    }
+
     public void testGeohashConfiguration() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location")

core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.geo;
 
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -29,6 +30,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -88,6 +90,36 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase {
         assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW));
     }
 
+    /**
+     * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document
+     */
+    public void testIgnoreMalformed() throws Exception {
+        // create index
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .addMapping("geometry", "shape", "type=geo_shape,ignore_malformed=true").get());
+        ensureGreen();
+
+        // test self crossing ccw poly not crossing dateline
+        String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
+            .startArray("coordinates")
+            .startArray()
+            .startArray().value(176.0).value(15.0).endArray()
+            .startArray().value(-177.0).value(10.0).endArray()
+            .startArray().value(-177.0).value(-10.0).endArray()
+            .startArray().value(176.0).value(-15.0).endArray()
+            .startArray().value(-177.0).value(15.0).endArray()
+            .startArray().value(172.0).value(0.0).endArray()
+            .startArray().value(176.0).value(15.0).endArray()
+            .endArray()
+            .endArray()
+            .endObject().string();
+
+        indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape",
+            polygonGeoJson));
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get();
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
+    }
+
     private String findNodeName(String index) {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0);

docs/reference/mapping/types/geo-shape.asciidoc

@@ -86,6 +86,10 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie
 optimal on a point only field.
 | `false`
 
+|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default),
+malformed geojson shapes throw an exception and reject the whole document.
+| `false`
+
 
 |=======================================================================