honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add null_value support to geo_point type (#29451) 

Adds support for null_value attribute to the geo_point types.

Closes #12998
This commit is contained in:
Igor Motov 2018-04-17 10:19:54 -04:00 committed by GitHub
parent 3367948be6
commit 983d6c15a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 180 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/reference/mapping/types/geo-point.asciidoc
View File

@ -122,6 +122,11 @@ The following parameters are accepted by `geo_point` fields:
ignored. If `false`, geo-points containing any more than latitude and longitude
(two dimensions) values throw an exception and reject the whole document.
<<null-value,`null_value`>>::
Accepts an geopoint value which is substituted for any explicit `null` values.
Defaults to `null`, which means the field is treated as missing.
==== Using geo-points in scripts
When accessing the value of a geo-point in a script, the value is returned as

36
server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
View File

@ -24,9 +24,14 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.apache.lucene.util.SloppyMath;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.GeoPointValues;
@ -36,6 +41,7 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import java.io.IOException;
import java.io.InputStream;
public class GeoUtils {
@ -351,6 +357,36 @@ public class GeoUtils {
return parseGeoPoint(parser, point, false);
}
/**
* Parses the value as a geopoint. The following types of values are supported:
* <p>
* Object: has to contain either lat and lon or geohash fields
* <p>
* String: expected to be in "latitude, longitude" format or a geohash
* <p>
* Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true
*/
public static GeoPoint parseGeoPoint(Object value, final boolean ignoreZValue) throws ElasticsearchParseException {
try {
XContentBuilder content = JsonXContent.contentBuilder();
content.startObject();
content.field("null_value", value);
content.endObject();
try (InputStream stream = BytesReference.bytes(content).streamInput();
XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value
return parseGeoPoint(parser, new GeoPoint(), ignoreZValue);
}
} catch (IOException ex) {
throw new ElasticsearchParseException("error parsing geopoint", ex);
}
}
/**
* Parse a {@link GeoPoint} with a {@link XContentParser}. A geopoint has one of the following forms:
*

37
server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
View File

@ -60,6 +60,7 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
public static class Names {
public static final String IGNORE_MALFORMED = "ignore_malformed";
public static final ParseField IGNORE_Z_VALUE = new ParseField("ignore_z_value");
public static final String NULL_VALUE = "null_value";
}
public static class Defaults {
@ -134,7 +135,7 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
throws MapperParsingException {
Builder builder = new GeoPointFieldMapper.Builder(name);
parseField(builder, name, node, parserContext);
Object nullValue = null;
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
@ -147,9 +148,31 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
builder.ignoreZValue(XContentMapValues.nodeBooleanValue(propNode,
name + "." + Names.IGNORE_Z_VALUE.getPreferredName()));
iterator.remove();
} else if (propName.equals(Names.NULL_VALUE)) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
nullValue = propNode;
iterator.remove();
}
}
if (nullValue != null) {
boolean ignoreZValue = builder.ignoreZValue == null ? Defaults.IGNORE_Z_VALUE.value() : builder.ignoreZValue;
boolean ignoreMalformed = builder.ignoreMalformed == null ? Defaults.IGNORE_MALFORMED.value() : builder.ignoreZValue;
GeoPoint point = GeoUtils.parseGeoPoint(nullValue, ignoreZValue);
if (ignoreMalformed == false) {
if (point.lat() > 90.0 || point.lat() < -90.0) {
throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "]");
}
if (point.lon() > 180.0 || point.lon() < -180) {
throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "]");
}
} else {
GeoUtils.normalizePoint(point);
}
builder.nullValue(point);
}
return builder;
}
}
@ -318,7 +341,11 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
}
} else if (token == XContentParser.Token.VALUE_STRING) {
parse(context, sparse.resetFromString(context.parser().text(), ignoreZValue.value()));
} else if (token != XContentParser.Token.VALUE_NULL) {
} else if (token == XContentParser.Token.VALUE_NULL) {
if (fieldType.nullValue() != null) {
parse(context, (GeoPoint) fieldType.nullValue());
}
} else {
try {
parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse));
} catch (ElasticsearchParseException e) {
@ -337,11 +364,15 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || ignoreMalformed.explicit()) {
builder.field(GeoPointFieldMapper.Names.IGNORE_MALFORMED, ignoreMalformed.value());
builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value());
}
if (includeDefaults || ignoreZValue.explicit()) {
builder.field(Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
}
if (includeDefaults || fieldType().nullValue() != null) {
builder.field(Names.NULL_VALUE, fieldType().nullValue());
}
}
public Explicit<Boolean> ignoreZValue() {

49
server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.Priority;
@ -41,10 +42,12 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDI
import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE;
import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.NULL_VALUE;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
@ -349,4 +352,50 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
);
assertThat(e.getMessage(), containsString("name cannot be empty string"));
}
public void testNullValue() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("location")
.field("type", "geo_point")
.field(NULL_VALUE, "1,2")
.endObject().endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoPointFieldMapper.class));
Object nullValue = fieldMapper.fieldType().nullValue();
assertThat(nullValue, equalTo(new GeoPoint(1, 2)));
ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.nullField("location")
.endObject()),
XContentType.JSON));
assertThat(doc.rootDoc().getField("location"), notNullValue());
BytesRef defaultValue = doc.rootDoc().getField("location").binaryValue();
doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("location", "1, 2")
.endObject()),
XContentType.JSON));
// Shouldn't matter if we specify the value explicitly or use null value
assertThat(defaultValue, equalTo(doc.rootDoc().getField("location").binaryValue()));
doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("location", "3, 4")
.endObject()),
XContentType.JSON));
// Shouldn't matter if we specify the value explicitly or use null value
assertThat(defaultValue, not(equalTo(doc.rootDoc().getField("location").binaryValue())));
}
}

2
server/src/test/java/org/elasticsearch/index/mapper/NullValueTests.java
View File

@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.equalTo;
public class NullValueTests extends ESSingleNodeTestCase {
public void testNullNullValue() throws Exception {
IndexService indexService = createIndex("test", Settings.builder().build());
String[] typesToTest = {"integer", "long", "double", "float", "short", "date", "ip", "keyword", "boolean", "byte"};
String[] typesToTest = {"integer", "long", "double", "float", "short", "date", "ip", "keyword", "boolean", "byte", "geo_point"};
for (String type : typesToTest) {
String mapping = Strings.toString(XContentFactory.jsonBuilder()

57
server/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
View File

@ -76,14 +76,26 @@ public class GeoPointParsingTests extends ESTestCase {
GeoPoint point = GeoUtils.parseGeoPoint(objectLatLon(randomPt.lat(), randomPt.lon()));
assertPointsEqual(point, randomPt);
GeoUtils.parseGeoPoint(toObject(objectLatLon(randomPt.lat(), randomPt.lon())), randomBoolean());
assertPointsEqual(point, randomPt);
GeoUtils.parseGeoPoint(arrayLatLon(randomPt.lat(), randomPt.lon()), point);
assertPointsEqual(point, randomPt);
GeoUtils.parseGeoPoint(toObject(arrayLatLon(randomPt.lat(), randomPt.lon())), randomBoolean());
assertPointsEqual(point, randomPt);
GeoUtils.parseGeoPoint(geohash(randomPt.lat(), randomPt.lon()), point);
assertCloseTo(point, randomPt.lat(), randomPt.lon());
GeoUtils.parseGeoPoint(toObject(geohash(randomPt.lat(), randomPt.lon())), randomBoolean());
assertCloseTo(point, randomPt.lat(), randomPt.lon());
GeoUtils.parseGeoPoint(stringLatLon(randomPt.lat(), randomPt.lon()), point);
assertCloseTo(point, randomPt.lat(), randomPt.lon());
GeoUtils.parseGeoPoint(toObject(stringLatLon(randomPt.lat(), randomPt.lon())), randomBoolean());
assertCloseTo(point, randomPt.lat(), randomPt.lon());
}
// Based on #5390
@ -99,6 +111,12 @@ public class GeoPointParsingTests extends ESTestCase {
parser.nextToken();
Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
XContentParser parser2 = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser2.nextToken();
e = expectThrows(ElasticsearchParseException.class, () ->
GeoUtils.parseGeoPoint(toObject(parser2), randomBoolean()));
assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
}
public void testInvalidPointLatHashMix() throws IOException {
@ -109,9 +127,14 @@ public class GeoPointParsingTests extends ESTestCase {
XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser.nextToken();
Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
XContentParser parser2 = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser2.nextToken();
e = expectThrows(ElasticsearchParseException.class, () ->
GeoUtils.parseGeoPoint(toObject(parser2), randomBoolean()));
assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
}
public void testInvalidPointLonHashMix() throws IOException {
@ -125,6 +148,12 @@ public class GeoPointParsingTests extends ESTestCase {
Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
XContentParser parser2 = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser2.nextToken();
e = expectThrows(ElasticsearchParseException.class, () ->
GeoUtils.parseGeoPoint(toObject(parser2), randomBoolean()));
assertThat(e.getMessage(), is("field must be either lat/lon or geohash"));
}
public void testInvalidField() throws IOException {
@ -135,9 +164,15 @@ public class GeoPointParsingTests extends ESTestCase {
XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser.nextToken();
Exception e = expectThrows(ElasticsearchParseException.class, () -> GeoUtils.parseGeoPoint(parser));
assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
XContentParser parser2 = createParser(JsonXContent.jsonXContent, BytesReference.bytes(content));
parser2.nextToken();
e = expectThrows(ElasticsearchParseException.class, () ->
GeoUtils.parseGeoPoint(toObject(parser2), randomBoolean()));
assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]"));
}
private XContentParser objectLatLon(double lat, double lon) throws IOException {
@ -183,4 +218,22 @@ public class GeoPointParsingTests extends ESTestCase {
assertEquals(point.lat(), lat, TOLERANCE);
assertEquals(point.lon(), lon, TOLERANCE);
}
public static Object toObject(XContentParser parser) throws IOException {
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
return null;
} else if (token == XContentParser.Token.VALUE_STRING) {
return parser.text();
} else if (token == XContentParser.Token.VALUE_NUMBER) {
return parser.numberValue();
} else if (token == XContentParser.Token.START_OBJECT) {
return parser.map();
} else if (token == XContentParser.Token.START_ARRAY) {
return parser.list();
} else {
fail("Unexpected token " + token);
}
return null;
}
}