honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ignore_malformed to geo_shape fields 

This commit adds ignore_malformed support to geo_shape field types to skip malformed geoJson fields.

closes #23747
This commit is contained in:
Nicholas Knize 2017-05-12 16:05:50 -05:00
parent ac5fd6a7d9
commit 06ff92d237
4 changed files with 115 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
@ -54,6 +53,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED;
/**
* FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
* <p>
@ -96,6 +97,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
public static final Orientation ORIENTATION = Orientation.RIGHT;
public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d;
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType();
@ -115,6 +117,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
public static class Builder extends FieldMapper.Builder<Builder, GeoShapeFieldMapper> {
private Boolean coerce;
private Boolean ignoreMalformed;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@ -145,6 +148,21 @@ public class GeoShapeFieldMapper extends FieldMapper {
return Defaults.COERCE;
}
public Builder ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
return builder;
}
protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
if (ignoreMalformed != null) {
return new Explicit<>(ignoreMalformed, true);
}
if (context.indexSettings() != null) {
return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
}
return Defaults.IGNORE_MALFORMED;
}
@Override
public GeoShapeFieldMapper build(BuilderContext context) {
GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType;
@ -154,8 +172,8 @@ public class GeoShapeFieldMapper extends FieldMapper {
}
setupFieldType(context);
return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this,
context), copyTo);
return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}
}
@ -186,6 +204,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
} else if (Names.STRATEGY.equals(fieldName)) {
builder.fieldType().setStrategyName(fieldNode.toString());
iterator.remove();
} else if (IGNORE_MALFORMED.equals(fieldName)) {
builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
iterator.remove();
} else if (Names.COERCE.equals(fieldName)) {
builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
iterator.remove();
@ -428,11 +449,13 @@ public class GeoShapeFieldMapper extends FieldMapper {
}
protected Explicit<Boolean> coerce;
protected Explicit<Boolean> ignoreMalformed;
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> coerce, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
}
@Override
@ -461,7 +484,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
context.doc().add(field);
}
} catch (Exception e) {
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
if (ignoreMalformed.value() == false) {
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
}
}
return null;
}
@ -478,6 +503,9 @@ public class GeoShapeFieldMapper extends FieldMapper {
if (gsfm.coerce.explicit()) {
this.coerce = gsfm.coerce;
}
if (gsfm.ignoreMalformed.explicit()) {
this.ignoreMalformed = gsfm.ignoreMalformed;
}
}
@Override
@ -506,7 +534,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly());
}
if (includeDefaults || coerce.explicit()) {
builder.field("coerce", coerce.value());
builder.field(Names.COERCE, coerce.value());
}
if (includeDefaults || ignoreMalformed.explicit()) {
builder.field(IGNORE_MALFORMED, ignoreMalformed.value());
}
}
@ -514,6 +545,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
return coerce;
}
public Explicit<Boolean> ignoreMalformed() {
return ignoreMalformed;
}
@Override
protected String contentType() {
return CONTENT_TYPE;

38
core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
View File

@ -22,6 +22,7 @@ import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@ -103,7 +104,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
}
/**
* Test that orientation parameter correctly parses
* Test that coerce parameter correctly parses
*/
public void testCoerceParsing() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
@ -136,6 +137,41 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
assertThat(coerce, equalTo(false));
}
/**
* Test that ignore_malformed parameter correctly parses
*/
public void testIgnoreMalformedParsing() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("ignore_malformed", "true")
.endObject().endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
Explicit<Boolean> ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
assertThat(ignoreMalformed.value(), equalTo(true));
// explicit false ignore_malformed test
mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("ignore_malformed", "false")
.endObject().endObject()
.endObject().endObject().string();
defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
assertThat(ignoreMalformed.explicit(), equalTo(true));
assertThat(ignoreMalformed.value(), equalTo(false));
}
public void testGeohashConfiguration() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")

32
core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.geo;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@ -29,6 +30,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ESIntegTestCase;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
@ -88,6 +90,36 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase {
assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW));
}
/**
* Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document
*/
public void testIgnoreMalformed() throws Exception {
// create index
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("geometry", "shape", "type=geo_shape,ignore_malformed=true").get());
ensureGreen();
// test self crossing ccw poly not crossing dateline
String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
.startArray("coordinates")
.startArray()
.startArray().value(176.0).value(15.0).endArray()
.startArray().value(-177.0).value(10.0).endArray()
.startArray().value(-177.0).value(-10.0).endArray()
.startArray().value(176.0).value(-15.0).endArray()
.startArray().value(-177.0).value(15.0).endArray()
.startArray().value(172.0).value(0.0).endArray()
.startArray().value(176.0).value(15.0).endArray()
.endArray()
.endArray()
.endObject().string();
indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape",
polygonGeoJson));
SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
}
private String findNodeName(String index) {
ClusterState state = client().admin().cluster().prepareState().get().getState();
IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0);

4
docs/reference/mapping/types/geo-shape.asciidoc
View File

@ -86,6 +86,10 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie
optimal on a point only field.
| `false`
|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default),
malformed geojson shapes throw an exception and reject the whole document.
| `false`
|=======================================================================