Support externalValue() in mappers

Some mappers do not support externalValue() to be set. So plugin developers can't use it while building their own mappers.

Support added in this PR for:

* `BinaryFieldMapper`
* `BooleanFieldMapper`
* `GeoPointFieldMapper`
* `GeoShapeFieldMapper`

Closes #4986.
Relative to #4154.

src/main/java/org/elasticsearch/index/mapper/ParseContext.java

@@ -26,6 +26,7 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -33,7 +34,6 @@ import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
 
 import java.util.*;
@@ -396,6 +396,23 @@ public class ParseContext {
         return externalValue;
     }
 
+    /**
+     * Try to parse an externalValue if any
+     * @param clazz Expected class for external value
+     * @return null if no external value has been set or the value
+     */
+    public <T> T parseExternalValue(Class<T> clazz) {
+        if (!externalValueSet() || externalValue() == null) {
+            return null;
+        }
+
+        if (!clazz.isInstance(externalValue())) {
+            throw new ElasticsearchIllegalArgumentException("illegal external value class ["
+                    + externalValue().getClass().getName() + "]. Should be " + clazz.getName());
+        }
+        return (T) externalValue();
+    }
+
     public float docBoost() {
         return this.docBoost;
     }

src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java

@@ -174,11 +174,17 @@ public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
         if (!fieldType().stored()) {
             return;
         }
-        byte[] value;
+        byte[] value = context.parseExternalValue(byte[].class);
+        if (value == null) {
             if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
                 return;
             } else {
                 value = context.parser().binaryValue();
+            }
+        }
+        if (value == null) {
+            return;
+        }
         if (compress != null && compress && !CompressorFactory.isCompressed(value, 0, value.length)) {
             if (compressThreshold == -1 || value.length > compressThreshold) {
                 BytesStreamOutput bStream = new BytesStreamOutput();
@@ -188,10 +194,6 @@ public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
                 value = bStream.bytes().toBytes();
             }
         }
-        }
-        if (value == null) {
-            return;
-        }
         fields.add(new Field(names.indexName(), value, fieldType));
     }
 

src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java

@@ -207,19 +207,23 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
         if (!fieldType().indexed() && !fieldType().stored()) {
             return;
         }
+
+        Boolean value = context.parseExternalValue(Boolean.class);
+        if (value == null) {
             XContentParser.Token token = context.parser().currentToken();
-        String value = null;
             if (token == XContentParser.Token.VALUE_NULL) {
                 if (nullValue != null) {
-                value = nullValue ? "T" : "F";
+                    value = nullValue;
                 }
             } else {
-            value = context.parser().booleanValue() ? "T" : "F";
+                value = context.parser().booleanValue();
             }
+        }
+
         if (value == null) {
             return;
         }
-        fields.add(new Field(names.indexName(), value, fieldType));
+        fields.add(new Field(names.indexName(), value ? "T" : "F", fieldType));
     }
 
     @Override

src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java

@@ -480,6 +480,10 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
         context.path().pathType(pathType);
         context.path().add(name());
 
+        GeoPoint value = context.parseExternalValue(GeoPoint.class);
+        if (value != null) {
+            parseLatLon(context, value.lat(), value.lon());
+        } else {
             XContentParser.Token token = context.parser().currentToken();
             if (token == XContentParser.Token.START_ARRAY) {
                 token = context.parser().nextToken();
@@ -522,6 +526,7 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
             } else if (token == XContentParser.Token.VALUE_STRING) {
                 parseStringLatLon(context);
             }
+        }
 
         context.path().remove();
         context.path().pathType(origPathType);

src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.mapper.geo;
 
+import com.spatial4j.core.shape.Shape;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.FieldInfo;
@@ -224,11 +225,15 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
     @Override
     public void parse(ParseContext context) throws IOException {
         try {
-            ShapeBuilder shape = ShapeBuilder.parse(context.parser());
+            Shape shape = context.parseExternalValue(Shape.class);
             if (shape == null) {
+                ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser());
+                if (shapeBuilder == null) {
                     return;
                 }
-            Field[] fields = defaultStrategy.createIndexableFields(shape.build());
+                shape = shapeBuilder.build();
+            }
+            Field[] fields = defaultStrategy.createIndexableFields(shape);
             if (fields == null || fields.length == 0) {
                 return;
             }

src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalIndexModule.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import org.elasticsearch.common.inject.AbstractModule;
+
+/**
+ *
+ */
+public class ExternalIndexModule extends AbstractModule {
+
+    @Override
+    protected void configure() {
+        bind(RegisterExternalTypes.class).asEagerSingleton();
+    }
+}

src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java

@@ -0,0 +1,189 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import com.spatial4j.core.shape.Point;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.*;
+import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
+import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
+import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
+import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+/**
+ * This mapper add a new sub fields
+ *  .bin Binary type
+ *  .bool Boolean type
+ *  .point GeoPoint type
+ *  .shape GeoShape type
+ */
+public class ExternalMapper implements Mapper {
+    public static class Names {
+        public static final String FIELD_BIN = "bin";
+        public static final String FIELD_BOOL = "bool";
+        public static final String FIELD_POINT = "point";
+        public static final String FIELD_SHAPE = "shape";
+    }
+
+    public static class Builder extends Mapper.Builder<Builder, ExternalMapper> {
+
+        private BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN);
+        private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL);
+        private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT);
+        private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE);
+
+        public Builder(String name) {
+            super(name);
+            this.builder = this;
+        }
+
+        @Override
+        public ExternalMapper build(BuilderContext context) {
+            ContentPath.Type origPathType = context.path().pathType();
+            context.path().pathType(ContentPath.Type.FULL);
+
+            context.path().add(name);
+            BinaryFieldMapper binMapper = binBuilder.build(context);
+            BooleanFieldMapper boolMapper = boolBuilder.build(context);
+            GeoPointFieldMapper pointMapper = pointBuilder.build(context);
+            GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context);
+            context.path().remove();
+
+            context.path().pathType(origPathType);
+
+            return new ExternalMapper(name, binMapper, boolMapper, pointMapper, shapeMapper);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+
+        @SuppressWarnings({"unchecked"})
+        @Override
+        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            ExternalMapper.Builder builder = new ExternalMapper.Builder(name);
+            return builder;
+        }
+    }
+
+    private final String name;
+
+    private final BinaryFieldMapper binMapper;
+    private final BooleanFieldMapper boolMapper;
+    private final GeoPointFieldMapper pointMapper;
+    private final GeoShapeFieldMapper shapeMapper;
+
+    public ExternalMapper(String name,
+                          BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, GeoShapeFieldMapper shapeMapper) {
+        this.name = name;
+        this.binMapper = binMapper;
+        this.boolMapper = boolMapper;
+        this.pointMapper = pointMapper;
+        this.shapeMapper = shapeMapper;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public void parse(ParseContext context) throws IOException {
+        ContentPath.Type origPathType = context.path().pathType();
+        context.path().pathType(ContentPath.Type.FULL);
+        context.path().add(name);
+
+        // Let's add a Dummy Binary content
+        context.path().add(Names.FIELD_BIN);
+        byte[] bytes = "Hello world".getBytes(Charset.defaultCharset());
+        context.externalValue(bytes);
+        binMapper.parse(context);
+        context.path().remove();
+
+        // Let's add a Dummy Boolean content
+        context.path().add(Names.FIELD_BOOL);
+        context.externalValue(true);
+        boolMapper.parse(context);
+        context.path().remove();
+
+        // Let's add a Dummy Point
+        Double lat = 42.0;
+        Double lng = 51.0;
+        context.path().add(Names.FIELD_POINT);
+        GeoPoint point = new GeoPoint(lat, lng);
+        context.externalValue(point);
+        pointMapper.parse(context);
+        context.path().remove();
+
+        // Let's add a Dummy Shape
+        context.path().add(Names.FIELD_SHAPE);
+        Point shape = ShapeBuilder.newPoint(-100, 45).build();
+        context.externalValue(shape);
+        shapeMapper.parse(context);
+        context.path().remove();
+
+        context.path().pathType(origPathType);
+    }
+
+    @Override
+    public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
+        // ignore this for now
+    }
+
+    @Override
+    public void traverse(FieldMapperListener fieldMapperListener) {
+        binMapper.traverse(fieldMapperListener);
+        boolMapper.traverse(fieldMapperListener);
+        pointMapper.traverse(fieldMapperListener);
+        shapeMapper.traverse(fieldMapperListener);
+    }
+
+    @Override
+    public void traverse(ObjectMapperListener objectMapperListener) {
+    }
+
+    @Override
+    public void close() {
+        binMapper.close();
+        boolMapper.close();
+        pointMapper.close();
+        shapeMapper.close();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(name);
+	        builder.field("type", RegisterExternalTypes.EXTERNAL);
+	        builder.startObject("fields");
+                binMapper.toXContent(builder, params);
+                boolMapper.toXContent(builder, params);
+                pointMapper.toXContent(builder, params);
+                shapeMapper.toXContent(builder, params);
+	        builder.endObject();
+        
+        builder.endObject();
+        return builder;
+    }
+}

src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.plugins.AbstractPlugin;
+
+import java.util.Collection;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+public class ExternalMapperPlugin extends AbstractPlugin {
+    /**
+     * The name of the plugin.
+     */
+    @Override
+    public String name() {
+        return "external-mappers";
+    }
+
+    /**
+     * The description of the plugin.
+     */
+    @Override
+    public String description() {
+        return "External Mappers Plugin";
+    }
+
+    @Override
+    public Collection<Class<? extends Module>> indexModules() {
+        Collection<Class<? extends Module>> modules = newArrayList();
+        modules.add(ExternalIndexModule.class);
+        return modules;
+    }
+}

src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ */
+@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE)
+public class ExternalValuesMapperIntegrationTests extends ElasticsearchIntegrationTest {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.settingsBuilder()
+                .put("plugin.types", ExternalMapperPlugin.class.getName())
+                .put(super.nodeSettings(nodeOrdinal))
+                .build();
+    }
+
+    @Test
+    public void testExternalGeoPoint() throws Exception {
+        prepareCreate("test-idx").addMapping("doc", createMapping()).execute().get();
+        ensureYellow("test-idx");
+
+        index("test-idx", "doc", "1", "external", "dummy");
+        refresh();
+
+        SearchResponse response;
+
+        response = client().prepareSearch("test-idx")
+                .setPostFilter(FilterBuilders.termFilter("external.bool", "T"))
+                .execute().actionGet();
+
+        assertThat(response.getHits().totalHits(), equalTo((long) 1));
+
+        response = client().prepareSearch("test-idx")
+                .setPostFilter(FilterBuilders.geoDistanceRangeFilter("external.point").point(42.0, 51.0).to("1km"))
+                .execute().actionGet();
+
+        assertThat(response.getHits().totalHits(), equalTo((long) 1));
+
+        response = client().prepareSearch("test-idx")
+                .setPostFilter(FilterBuilders.geoShapeFilter("external.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN))
+                        .execute().actionGet();
+
+        assertThat(response.getHits().totalHits(), equalTo((long) 1));
+    }
+
+
+    private XContentBuilder createMapping() throws IOException {
+        return XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties")
+                .startObject("external").field("type", RegisterExternalTypes.EXTERNAL).endObject()
+                .endObject().endObject().endObject();
+    }
+
+}

src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.settings.IndexSettings;
+
+public class RegisterExternalTypes extends AbstractIndexComponent {
+    public static final String EXTERNAL = "external";
+
+    @Inject
+    public RegisterExternalTypes(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) {
+        super(index, indexSettings);
+
+        mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser());
+    }
+}