honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support externalValue() in mappers 

Some mappers do not support externalValue() to be set. So plugin developers can't use it while building their own mappers.

Support added in this PR for:

* `BinaryFieldMapper`
* `BooleanFieldMapper`
* `GeoPointFieldMapper`
* `GeoShapeFieldMapper`

Closes #4986.
Relative to #4154.
This commit is contained in:
David Pilato 2014-03-14 16:21:11 +01:00
parent 6f80b7737a
commit 84b5b45644
10 changed files with 490 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
@ -33,7 +34,6 @@ import org.elasticsearch.common.lucene.all.AllEntries;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import java.util.*;
@ -396,6 +396,23 @@ public class ParseContext {
return externalValue;
}
/**
* Try to parse an externalValue if any
* @param clazz Expected class for external value
* @return null if no external value has been set or the value
*/
public <T> T parseExternalValue(Class<T> clazz) {
if (!externalValueSet() || externalValue() == null) {
return null;
}
if (!clazz.isInstance(externalValue())) {
throw new ElasticsearchIllegalArgumentException("illegal external value class ["
+ externalValue().getClass().getName() + "]. Should be " + clazz.getName());
}
return (T) externalValue();
}
public float docBoost() {
return this.docBoost;
}

28
src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View File

@ -174,24 +174,26 @@ public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
if (!fieldType().stored()) {
return;
}
byte[] value;
if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
return;
} else {
value = context.parser().binaryValue();
if (compress != null && compress && !CompressorFactory.isCompressed(value, 0, value.length)) {
if (compressThreshold == -1 || value.length > compressThreshold) {
BytesStreamOutput bStream = new BytesStreamOutput();
StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream);
stream.writeBytes(value, 0, value.length);
stream.close();
value = bStream.bytes().toBytes();
}
byte[] value = context.parseExternalValue(byte[].class);
if (value == null) {
if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
return;
} else {
value = context.parser().binaryValue();
}
}
if (value == null) {
return;
}
if (compress != null && compress && !CompressorFactory.isCompressed(value, 0, value.length)) {
if (compressThreshold == -1 || value.length > compressThreshold) {
BytesStreamOutput bStream = new BytesStreamOutput();
StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream);
stream.writeBytes(value, 0, value.length);
stream.close();
value = bStream.bytes().toBytes();
}
}
fields.add(new Field(names.indexName(), value, fieldType));
}

20
src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
View File

@ -207,19 +207,23 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
if (!fieldType().indexed() && !fieldType().stored()) {
return;
}
XContentParser.Token token = context.parser().currentToken();
String value = null;
if (token == XContentParser.Token.VALUE_NULL) {
if (nullValue != null) {
value = nullValue ? "T" : "F";
Boolean value = context.parseExternalValue(Boolean.class);
if (value == null) {
XContentParser.Token token = context.parser().currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
if (nullValue != null) {
value = nullValue;
}
} else {
value = context.parser().booleanValue();
}
} else {
value = context.parser().booleanValue() ? "T" : "F";
}
if (value == null) {
return;
}
fields.add(new Field(names.indexName(), value, fieldType));
fields.add(new Field(names.indexName(), value ? "T" : "F", fieldType));
}
@Override

73
src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
View File

@ -480,47 +480,52 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
context.path().pathType(pathType);
context.path().add(name());
XContentParser.Token token = context.parser().currentToken();
if (token == XContentParser.Token.START_ARRAY) {
token = context.parser().nextToken();
GeoPoint value = context.parseExternalValue(GeoPoint.class);
if (value != null) {
parseLatLon(context, value.lat(), value.lon());
} else {
XContentParser.Token token = context.parser().currentToken();
if (token == XContentParser.Token.START_ARRAY) {
// its an array of array of lon/lat [ [1.2, 1.3], [1.4, 1.5] ]
while (token != XContentParser.Token.END_ARRAY) {
token = context.parser().nextToken();
double lon = context.parser().doubleValue();
token = context.parser().nextToken();
double lat = context.parser().doubleValue();
while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY) {
}
parseLatLon(context, lat, lon);
token = context.parser().nextToken();
}
} else {
// its an array of other possible values
if (token == XContentParser.Token.VALUE_NUMBER) {
double lon = context.parser().doubleValue();
token = context.parser().nextToken();
double lat = context.parser().doubleValue();
while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY) {
}
parseLatLon(context, lat, lon);
} else {
token = context.parser().nextToken();
if (token == XContentParser.Token.START_ARRAY) {
// its an array of array of lon/lat [ [1.2, 1.3], [1.4, 1.5] ]
while (token != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.START_OBJECT) {
parseObjectLatLon(context);
} else if (token == XContentParser.Token.VALUE_STRING) {
parseStringLatLon(context);
token = context.parser().nextToken();
double lon = context.parser().doubleValue();
token = context.parser().nextToken();
double lat = context.parser().doubleValue();
while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY) {
}
parseLatLon(context, lat, lon);
token = context.parser().nextToken();
}
} else {
// its an array of other possible values
if (token == XContentParser.Token.VALUE_NUMBER) {
double lon = context.parser().doubleValue();
token = context.parser().nextToken();
double lat = context.parser().doubleValue();
while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY) {
}
parseLatLon(context, lat, lon);
} else {
while (token != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.START_OBJECT) {
parseObjectLatLon(context);
} else if (token == XContentParser.Token.VALUE_STRING) {
parseStringLatLon(context);
}
token = context.parser().nextToken();
}
}
}
} else if (token == XContentParser.Token.START_OBJECT) {
parseObjectLatLon(context);
} else if (token == XContentParser.Token.VALUE_STRING) {
parseStringLatLon(context);
}
} else if (token == XContentParser.Token.START_OBJECT) {
parseObjectLatLon(context);
} else if (token == XContentParser.Token.VALUE_STRING) {
parseStringLatLon(context);
}
context.path().remove();

11
src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.mapper.geo;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.FieldInfo;
@ -224,11 +225,15 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
@Override
public void parse(ParseContext context) throws IOException {
try {
ShapeBuilder shape = ShapeBuilder.parse(context.parser());
Shape shape = context.parseExternalValue(Shape.class);
if (shape == null) {
return;
ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser());
if (shapeBuilder == null) {
return;
}
shape = shapeBuilder.build();
}
Field[] fields = defaultStrategy.createIndexableFields(shape.build());
Field[] fields = defaultStrategy.createIndexableFields(shape);
if (fields == null || fields.length == 0) {
return;
}

33
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalIndexModule.java Executable file
View File

@ -0,0 +1,33 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import org.elasticsearch.common.inject.AbstractModule;
/**
*
*/
public class ExternalIndexModule extends AbstractModule {
@Override
protected void configure() {
bind(RegisterExternalTypes.class).asEagerSingleton();
}
}

189
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java Executable file
View File

@ -0,0 +1,189 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import com.spatial4j.core.shape.Point;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Map;
/**
* This mapper add a new sub fields
* .bin Binary type
* .bool Boolean type
* .point GeoPoint type
* .shape GeoShape type
*/
public class ExternalMapper implements Mapper {
public static class Names {
public static final String FIELD_BIN = "bin";
public static final String FIELD_BOOL = "bool";
public static final String FIELD_POINT = "point";
public static final String FIELD_SHAPE = "shape";
}
public static class Builder extends Mapper.Builder<Builder, ExternalMapper> {
private BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN);
private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL);
private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT);
private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE);
public Builder(String name) {
super(name);
this.builder = this;
}
@Override
public ExternalMapper build(BuilderContext context) {
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(ContentPath.Type.FULL);
context.path().add(name);
BinaryFieldMapper binMapper = binBuilder.build(context);
BooleanFieldMapper boolMapper = boolBuilder.build(context);
GeoPointFieldMapper pointMapper = pointBuilder.build(context);
GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context);
context.path().remove();
context.path().pathType(origPathType);
return new ExternalMapper(name, binMapper, boolMapper, pointMapper, shapeMapper);
}
}
public static class TypeParser implements Mapper.TypeParser {
@SuppressWarnings({"unchecked"})
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
ExternalMapper.Builder builder = new ExternalMapper.Builder(name);
return builder;
}
}
private final String name;
private final BinaryFieldMapper binMapper;
private final BooleanFieldMapper boolMapper;
private final GeoPointFieldMapper pointMapper;
private final GeoShapeFieldMapper shapeMapper;
public ExternalMapper(String name,
BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, GeoShapeFieldMapper shapeMapper) {
this.name = name;
this.binMapper = binMapper;
this.boolMapper = boolMapper;
this.pointMapper = pointMapper;
this.shapeMapper = shapeMapper;
}
@Override
public String name() {
return name;
}
@Override
public void parse(ParseContext context) throws IOException {
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(ContentPath.Type.FULL);
context.path().add(name);
// Let's add a Dummy Binary content
context.path().add(Names.FIELD_BIN);
byte[] bytes = "Hello world".getBytes(Charset.defaultCharset());
context.externalValue(bytes);
binMapper.parse(context);
context.path().remove();
// Let's add a Dummy Boolean content
context.path().add(Names.FIELD_BOOL);
context.externalValue(true);
boolMapper.parse(context);
context.path().remove();
// Let's add a Dummy Point
Double lat = 42.0;
Double lng = 51.0;
context.path().add(Names.FIELD_POINT);
GeoPoint point = new GeoPoint(lat, lng);
context.externalValue(point);
pointMapper.parse(context);
context.path().remove();
// Let's add a Dummy Shape
context.path().add(Names.FIELD_SHAPE);
Point shape = ShapeBuilder.newPoint(-100, 45).build();
context.externalValue(shape);
shapeMapper.parse(context);
context.path().remove();
context.path().pathType(origPathType);
}
@Override
public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
// ignore this for now
}
@Override
public void traverse(FieldMapperListener fieldMapperListener) {
binMapper.traverse(fieldMapperListener);
boolMapper.traverse(fieldMapperListener);
pointMapper.traverse(fieldMapperListener);
shapeMapper.traverse(fieldMapperListener);
}
@Override
public void traverse(ObjectMapperListener objectMapperListener) {
}
@Override
public void close() {
binMapper.close();
boolMapper.close();
pointMapper.close();
shapeMapper.close();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field("type", RegisterExternalTypes.EXTERNAL);
builder.startObject("fields");
binMapper.toXContent(builder, params);
boolMapper.toXContent(builder, params);
pointMapper.toXContent(builder, params);
shapeMapper.toXContent(builder, params);
builder.endObject();
builder.endObject();
return builder;
}
}

52
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.plugins.AbstractPlugin;
import java.util.Collection;
import static com.google.common.collect.Lists.newArrayList;
public class ExternalMapperPlugin extends AbstractPlugin {
/**
* The name of the plugin.
*/
@Override
public String name() {
return "external-mappers";
}
/**
* The description of the plugin.
*/
@Override
public String description() {
return "External Mappers Plugin";
}
@Override
public Collection<Class<? extends Module>> indexModules() {
Collection<Class<? extends Module>> modules = newArrayList();
modules.add(ExternalIndexModule.class);
return modules;
}
}

86
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import java.io.IOException;
import static org.hamcrest.Matchers.equalTo;
/**
*/
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE)
public class ExternalValuesMapperIntegrationTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return ImmutableSettings.settingsBuilder()
.put("plugin.types", ExternalMapperPlugin.class.getName())
.put(super.nodeSettings(nodeOrdinal))
.build();
}
@Test
public void testExternalGeoPoint() throws Exception {
prepareCreate("test-idx").addMapping("doc", createMapping()).execute().get();
ensureYellow("test-idx");
index("test-idx", "doc", "1", "external", "dummy");
refresh();
SearchResponse response;
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.termFilter("external.bool", "T"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.geoDistanceRangeFilter("external.point").point(42.0, 51.0).to("1km"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.geoShapeFilter("external.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
}
private XContentBuilder createMapping() throws IOException {
return XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties")
.startObject("external").field("type", RegisterExternalTypes.EXTERNAL).endObject()
.endObject().endObject().endObject();
}
}

38
src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java Executable file
View File

@ -0,0 +1,38 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
public class RegisterExternalTypes extends AbstractIndexComponent {
public static final String EXTERNAL = "external";
@Inject
public RegisterExternalTypes(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) {
super(index, indexSettings);
mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser());
}
}