honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add multi_field support for Mapper externalValue (plugins) 

In context of mapper attachment and other mapper plugins, when dealing with multi fields, sub fields never get the `externalValue` although it was set.

Here is a full script which reproduce the issue when used with mapper attachment plugin:

```
DELETE /test

PUT /test
{
    "mappings": {
        "test": {
            "properties": {
                "f": {
                    "type": "attachment",
                    "fields": {
                        "f": {
                            "analyzer": "english",
                            "fields": {
                                "no_stemming": {
                                    "type": "string",
                                    "store": "yes",
                                    "analyzer": "standard"
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

PUT /test/test/1
{
    "f": "VGhlIHF1aWNrIGJyb3duIGZveGVz"
}

GET /test/_search
{
    "query": {
        "match": {
           "f": "quick"
        }
    }
}

GET /test/_search
{
    "query": {
        "match": {
           "f.no_stemming": "quick"
        }
    }
}

GET /test/test/1?fields=f.no_stemming
```

Related to https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/57

Closes #5402.
This commit is contained in:
David Pilato 2014-06-14 16:11:39 +02:00
parent 655157c83a
commit 11eced01da
9 changed files with 885 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -248,10 +248,10 @@ public class DocumentMapper implements ToXContent {
}
private CloseableThreadLocal<ParseContext> cache = new CloseableThreadLocal<ParseContext>() {
private CloseableThreadLocal<ParseContext.InternalParseContext> cache = new CloseableThreadLocal<ParseContext.InternalParseContext>() {
@Override
protected ParseContext initialValue() {
return new ParseContext(index, indexSettings, docMapperParser, DocumentMapper.this, new ContentPath(0));
protected ParseContext.InternalParseContext initialValue() {
return new ParseContext.InternalParseContext(index, indexSettings, docMapperParser, DocumentMapper.this, new ContentPath(0));
}
};
@ -484,7 +484,7 @@ public class DocumentMapper implements ToXContent {
}
public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener) throws MapperParsingException {
ParseContext context = cache.get();
ParseContext.InternalParseContext context = cache.get();
if (source.type() != null && !source.type().equals(this.type)) {
throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]");

750
src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.common.lucene.all.AllEntries;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.DocumentMapper.ParseListener;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import java.util.*;
@ -41,7 +42,7 @@ import java.util.*;
/**
*
*/
public class ParseContext {
public abstract class ParseContext {
/** Fork of {@link org.apache.lucene.document.Document} with additional functionality. */
public static class Document implements Iterable<IndexableField> {
@ -121,242 +122,572 @@ public class ParseContext {
}
private final DocumentMapper docMapper;
private static class FilterParseContext extends ParseContext {
private final DocumentMapperParser docMapperParser;
private final ParseContext in;
private final ContentPath path;
private XContentParser parser;
private Document document;
private List<Document> documents = Lists.newArrayList();
private Analyzer analyzer;
private final String index;
@Nullable
private final Settings indexSettings;
private SourceToParse sourceToParse;
private BytesReference source;
private String id;
private DocumentMapper.ParseListener listener;
private Field uid, version;
private StringBuilder stringBuilder = new StringBuilder();
private Map<String, String> ignoredValues = new HashMap<>();
private boolean mappingsModified = false;
private boolean withinNewMapper = false;
private boolean withinCopyTo = false;
private boolean withinMultiFields = false;
private boolean externalValueSet;
private Object externalValue;
private AllEntries allEntries = new AllEntries();
private float docBoost = 1.0f;
public ParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
this.index = index;
this.indexSettings = indexSettings;
this.docMapper = docMapper;
this.docMapperParser = docMapperParser;
this.path = path;
}
public void reset(XContentParser parser, Document document, SourceToParse source, DocumentMapper.ParseListener listener) {
this.parser = parser;
this.document = document;
if (document != null) {
this.documents = Lists.newArrayList();
this.documents.add(document);
} else {
this.documents = null;
private FilterParseContext(ParseContext in) {
this.in = in;
}
this.analyzer = null;
this.uid = null;
this.version = null;
this.id = null;
this.sourceToParse = source;
this.source = source == null ? null : sourceToParse.source();
this.path.reset();
this.mappingsModified = false;
this.withinNewMapper = false;
this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener;
this.allEntries = new AllEntries();
this.ignoredValues.clear();
this.docBoost = 1.0f;
@Override
public boolean flyweight() {
return in.flyweight();
}
@Override
public DocumentMapperParser docMapperParser() {
return in.docMapperParser();
}
@Override
public boolean mappingsModified() {
return in.mappingsModified();
}
@Override
public void setMappingsModified() {
in.setMappingsModified();
}
@Override
public void setWithinNewMapper() {
in.setWithinNewMapper();
}
@Override
public void clearWithinNewMapper() {
in.clearWithinNewMapper();
}
@Override
public boolean isWithinNewMapper() {
return in.isWithinNewMapper();
}
@Override
public boolean isWithinCopyTo() {
return in.isWithinCopyTo();
}
@Override
public boolean isWithinMultiFields() {
return in.isWithinMultiFields();
}
@Override
public String index() {
return in.index();
}
@Override
public Settings indexSettings() {
return in.indexSettings();
}
@Override
public String type() {
return in.type();
}
@Override
public SourceToParse sourceToParse() {
return in.sourceToParse();
}
@Override
public BytesReference source() {
return in.source();
}
@Override
public void source(BytesReference source) {
in.source(source);
}
@Override
public ContentPath path() {
return in.path();
}
@Override
public XContentParser parser() {
return in.parser();
}
@Override
public ParseListener listener() {
return in.listener();
}
@Override
public Document rootDoc() {
return in.rootDoc();
}
@Override
public List<Document> docs() {
return in.docs();
}
@Override
public Document doc() {
return in.doc();
}
@Override
public void addDoc(Document doc) {
in.addDoc(doc);
}
@Override
public Document switchDoc(Document doc) {
return in.switchDoc(doc);
}
@Override
public RootObjectMapper root() {
return in.root();
}
@Override
public DocumentMapper docMapper() {
return in.docMapper();
}
@Override
public AnalysisService analysisService() {
return in.analysisService();
}
@Override
public String id() {
return in.id();
}
@Override
public void ignoredValue(String indexName, String value) {
in.ignoredValue(indexName, value);
}
@Override
public String ignoredValue(String indexName) {
return in.ignoredValue(indexName);
}
@Override
public void id(String id) {
in.id(id);
}
@Override
public Field uid() {
return in.uid();
}
@Override
public void uid(Field uid) {
in.uid(uid);
}
@Override
public Field version() {
return in.version();
}
@Override
public void version(Field version) {
in.version(version);
}
@Override
public AllEntries allEntries() {
return in.allEntries();
}
@Override
public Analyzer analyzer() {
return in.analyzer();
}
@Override
public void analyzer(Analyzer analyzer) {
in.analyzer(analyzer);
}
@Override
public boolean externalValueSet() {
return in.externalValueSet();
}
@Override
public Object externalValue() {
return in.externalValue();
}
@Override
public float docBoost() {
return in.docBoost();
}
@Override
public void docBoost(float docBoost) {
in.docBoost(docBoost);
}
@Override
public StringBuilder stringBuilder() {
return in.stringBuilder();
}
}
public boolean flyweight() {
return sourceToParse.flyweight();
public static class InternalParseContext extends ParseContext {
private final DocumentMapper docMapper;
private final DocumentMapperParser docMapperParser;
private final ContentPath path;
private XContentParser parser;
private Document document;
private List<Document> documents = Lists.newArrayList();
private Analyzer analyzer;
private final String index;
@Nullable
private final Settings indexSettings;
private SourceToParse sourceToParse;
private BytesReference source;
private String id;
private DocumentMapper.ParseListener listener;
private Field uid, version;
private StringBuilder stringBuilder = new StringBuilder();
private Map<String, String> ignoredValues = new HashMap<>();
private boolean mappingsModified = false;
private boolean withinNewMapper = false;
private AllEntries allEntries = new AllEntries();
private float docBoost = 1.0f;
public InternalParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
this.index = index;
this.indexSettings = indexSettings;
this.docMapper = docMapper;
this.docMapperParser = docMapperParser;
this.path = path;
}
public void reset(XContentParser parser, Document document, SourceToParse source, DocumentMapper.ParseListener listener) {
this.parser = parser;
this.document = document;
if (document != null) {
this.documents = Lists.newArrayList();
this.documents.add(document);
} else {
this.documents = null;
}
this.analyzer = null;
this.uid = null;
this.version = null;
this.id = null;
this.sourceToParse = source;
this.source = source == null ? null : sourceToParse.source();
this.path.reset();
this.mappingsModified = false;
this.withinNewMapper = false;
this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener;
this.allEntries = new AllEntries();
this.ignoredValues.clear();
this.docBoost = 1.0f;
}
public boolean flyweight() {
return sourceToParse.flyweight();
}
public DocumentMapperParser docMapperParser() {
return this.docMapperParser;
}
public boolean mappingsModified() {
return this.mappingsModified;
}
public void setMappingsModified() {
this.mappingsModified = true;
}
public void setWithinNewMapper() {
this.withinNewMapper = true;
}
public void clearWithinNewMapper() {
this.withinNewMapper = false;
}
public boolean isWithinNewMapper() {
return withinNewMapper;
}
public String index() {
return this.index;
}
@Nullable
public Settings indexSettings() {
return this.indexSettings;
}
public String type() {
return sourceToParse.type();
}
public SourceToParse sourceToParse() {
return this.sourceToParse;
}
public BytesReference source() {
return source;
}
// only should be used by SourceFieldMapper to update with a compressed source
public void source(BytesReference source) {
this.source = source;
}
public ContentPath path() {
return this.path;
}
public XContentParser parser() {
return this.parser;
}
public DocumentMapper.ParseListener listener() {
return this.listener;
}
public Document rootDoc() {
return documents.get(0);
}
public List<Document> docs() {
return this.documents;
}
public Document doc() {
return this.document;
}
public void addDoc(Document doc) {
this.documents.add(doc);
}
public Document switchDoc(Document doc) {
Document prev = this.document;
this.document = doc;
return prev;
}
public RootObjectMapper root() {
return docMapper.root();
}
public DocumentMapper docMapper() {
return this.docMapper;
}
public AnalysisService analysisService() {
return docMapperParser.analysisService;
}
public String id() {
return id;
}
public void ignoredValue(String indexName, String value) {
ignoredValues.put(indexName, value);
}
public String ignoredValue(String indexName) {
return ignoredValues.get(indexName);
}
/**
* Really, just the id mapper should set this.
*/
public void id(String id) {
this.id = id;
}
public Field uid() {
return this.uid;
}
/**
* Really, just the uid mapper should set this.
*/
public void uid(Field uid) {
this.uid = uid;
}
public Field version() {
return this.version;
}
public void version(Field version) {
this.version = version;
}
public AllEntries allEntries() {
return this.allEntries;
}
public Analyzer analyzer() {
return this.analyzer;
}
public void analyzer(Analyzer analyzer) {
this.analyzer = analyzer;
}
public float docBoost() {
return this.docBoost;
}
public void docBoost(float docBoost) {
this.docBoost = docBoost;
}
/**
* A string builder that can be used to construct complex names for example.
* Its better to reuse the.
*/
public StringBuilder stringBuilder() {
stringBuilder.setLength(0);
return this.stringBuilder;
}
}
public DocumentMapperParser docMapperParser() {
return this.docMapperParser;
}
public abstract boolean flyweight();
public boolean mappingsModified() {
return this.mappingsModified;
}
public abstract DocumentMapperParser docMapperParser();
public void setMappingsModified() {
this.mappingsModified = true;
}
public abstract boolean mappingsModified();
public void setWithinNewMapper() {
this.withinNewMapper = true;
}
public abstract void setMappingsModified();
public void clearWithinNewMapper() {
this.withinNewMapper = false;
}
public abstract void setWithinNewMapper();
public boolean isWithinNewMapper() {
return withinNewMapper;
}
public abstract void clearWithinNewMapper();
public void setWithinCopyTo() {
this.withinCopyTo = true;
}
public abstract boolean isWithinNewMapper();
public void clearWithinCopyTo() {
this.withinCopyTo = false;
/**
* Return a new context that will be within a copy-to operation.
*/
public final ParseContext createCopyToContext() {
return new FilterParseContext(this) {
@Override
public boolean isWithinCopyTo() {
return true;
}
};
}
public boolean isWithinCopyTo() {
return withinCopyTo;
return false;
}
public void setWithinMultiFields() {
this.withinMultiFields = true;
/**
* Return a new context that will be within multi-fields.
*/
public final ParseContext createMultiFieldContext() {
return new FilterParseContext(this) {
@Override
public boolean isWithinMultiFields() {
return true;
}
};
}
public void clearWithinMultiFields() {
this.withinMultiFields = false;
public boolean isWithinMultiFields() {
return false;
}
public String index() {
return this.index;
}
public abstract String index();
@Nullable
public Settings indexSettings() {
return this.indexSettings;
}
public abstract Settings indexSettings();
public String type() {
return sourceToParse.type();
}
public abstract String type();
public SourceToParse sourceToParse() {
return this.sourceToParse;
}
public abstract SourceToParse sourceToParse();
public BytesReference source() {
return source;
}
public abstract BytesReference source();
// only should be used by SourceFieldMapper to update with a compressed source
public void source(BytesReference source) {
this.source = source;
}
public abstract void source(BytesReference source);
public ContentPath path() {
return this.path;
}
public abstract ContentPath path();
public XContentParser parser() {
return this.parser;
}
public abstract XContentParser parser();
public DocumentMapper.ParseListener listener() {
return this.listener;
}
public abstract DocumentMapper.ParseListener listener();
public Document rootDoc() {
return documents.get(0);
}
public abstract Document rootDoc();
public List<Document> docs() {
return this.documents;
}
public abstract List<Document> docs();
public Document doc() {
return this.document;
}
public abstract Document doc();
public void addDoc(Document doc) {
this.documents.add(doc);
}
public abstract void addDoc(Document doc);
public Document switchDoc(Document doc) {
Document prev = this.document;
this.document = doc;
return prev;
}
public abstract Document switchDoc(Document doc);
public RootObjectMapper root() {
return docMapper.root();
}
public abstract RootObjectMapper root();
public DocumentMapper docMapper() {
return this.docMapper;
}
public abstract DocumentMapper docMapper();
public AnalysisService analysisService() {
return docMapperParser.analysisService;
}
public abstract AnalysisService analysisService();
public String id() {
return id;
}
public abstract String id();
public void ignoredValue(String indexName, String value) {
ignoredValues.put(indexName, value);
}
public abstract void ignoredValue(String indexName, String value);
public String ignoredValue(String indexName) {
return ignoredValues.get(indexName);
}
public abstract String ignoredValue(String indexName);
/**
* Really, just the id mapper should set this.
*/
public void id(String id) {
this.id = id;
}
public abstract void id(String id);
public Field uid() {
return this.uid;
}
public abstract Field uid();
/**
* Really, just the uid mapper should set this.
*/
public void uid(Field uid) {
this.uid = uid;
}
public abstract void uid(Field uid);
public Field version() {
return this.version;
}
public abstract Field version();
public void version(Field version) {
this.version = version;
}
public abstract void version(Field version);
public boolean includeInAll(Boolean includeInAll, FieldMapper mapper) {
public final boolean includeInAll(Boolean includeInAll, FieldMapper mapper) {
return includeInAll(includeInAll, mapper.fieldType().indexed());
}
@ -366,13 +697,13 @@ public class ParseContext {
* its actual value (so, if not set, defaults to "true") and the field is indexed.
*/
private boolean includeInAll(Boolean specificIncludeInAll, boolean indexed) {
if (withinCopyTo) {
if (isWithinCopyTo()) {
return false;
}
if (withinMultiFields) {
if (isWithinMultiFields()) {
return false;
}
if (!docMapper.allFieldMapper().enabled()) {
if (!docMapper().allFieldMapper().enabled()) {
return false;
}
// not explicitly set
@ -382,30 +713,34 @@ public class ParseContext {
return specificIncludeInAll;
}
public AllEntries allEntries() {
return this.allEntries;
}
public abstract AllEntries allEntries();
public Analyzer analyzer() {
return this.analyzer;
}
public abstract Analyzer analyzer();
public void analyzer(Analyzer analyzer) {
this.analyzer = analyzer;
}
public abstract void analyzer(Analyzer analyzer);
public void externalValue(Object externalValue) {
this.externalValueSet = true;
this.externalValue = externalValue;
/**
* Return a new context that will have the external value set.
*/
public final ParseContext createExternalValueContext(final Object externalValue) {
return new FilterParseContext(this) {
@Override
public boolean externalValueSet() {
return true;
}
@Override
public Object externalValue() {
return externalValue;
}
};
}
public boolean externalValueSet() {
return this.externalValueSet;
return false;
}
public Object externalValue() {
externalValueSet = false;
return externalValue;
throw new ElasticsearchIllegalStateException("External value is not set");
}
/**
@ -413,7 +748,7 @@ public class ParseContext {
* @param clazz Expected class for external value
* @return null if no external value has been set or the value
*/
public <T> T parseExternalValue(Class<T> clazz) {
public final <T> T parseExternalValue(Class<T> clazz) {
if (!externalValueSet() || externalValue() == null) {
return null;
}
@ -422,24 +757,17 @@ public class ParseContext {
throw new ElasticsearchIllegalArgumentException("illegal external value class ["
+ externalValue().getClass().getName() + "]. Should be " + clazz.getName());
}
return (T) externalValue();
return clazz.cast(externalValue());
}
public float docBoost() {
return this.docBoost;
}
public abstract float docBoost();
public void docBoost(float docBoost) {
this.docBoost = docBoost;
}
public abstract void docBoost(float docBoost);
/**
* A string builder that can be used to construct complex names for example.
* Its better to reuse the.
*/
public StringBuilder stringBuilder() {
stringBuilder.setLength(0);
return this.stringBuilder;
}
public abstract StringBuilder stringBuilder();
}

7
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -916,7 +916,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
return;
}
context.setWithinMultiFields();
context = context.createMultiFieldContext();
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
@ -927,8 +927,6 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
}
context.path().remove();
context.path().pathType(origPathType);
context.clearWithinMultiFields();
}
// No need for locking, because locking is taken care of in ObjectMapper#merge and DocumentMapper#merge
@ -1055,7 +1053,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
* Creates an copy of the current field with given field name and boost
*/
public void parse(String field, ParseContext context) throws IOException {
context.setWithinCopyTo();
context = context.createCopyToContext();
FieldMappers mappers = context.docMapper().mappers().indexName(field);
if (mappers != null && !mappers.isEmpty()) {
mappers.mapper().parse(context);
@ -1109,7 +1107,6 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
}
}
context.clearWithinCopyTo();
}

3
src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java
View File

@ -102,8 +102,7 @@ public class Murmur3FieldMapper extends LongFieldMapper {
if (value != null) {
final BytesRef bytes = new BytesRef(value.toString());
final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1;
context.externalValue(hash);
super.innerParseCreateField(context, fields);
super.innerParseCreateField(context.createExternalValueContext(hash), fields);
}
}

9
src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
View File

@ -539,9 +539,8 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
int min = enableGeohashPrefix ? 1 : geohash.length();
for (int i = len; i >= min; i--) {
context.externalValue(geohash.substring(0, i));
// side effect of this call is adding the field
geohashMapper.parse(context);
geohashMapper.parse(context.createExternalValueContext(geohash.substring(0, i)));
}
}
@ -580,10 +579,8 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
parseGeohashField(context, geohash);
}
if (enableLatLon) {
context.externalValue(point.lat());
latMapper.parse(context);
context.externalValue(point.lon());
lonMapper.parse(context);
latMapper.parse(context.createExternalValueContext(point.lat()));
lonMapper.parse(context.createExternalValueContext(point.lon()));
}
if (hasDocValues()) {
CustomGeoPointDocValuesField field = (CustomGeoPointDocValuesField) context.doc().getByKey(names().indexName());

147
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
View File

@ -20,10 +20,16 @@
package org.elasticsearch.index.mapper.externalvalues;
import com.spatial4j.core.shape.Point;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
@ -31,8 +37,13 @@ import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.stringField;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
/**
* This mapper add a new sub fields
* .bin Binary type
@ -40,7 +51,17 @@ import java.util.Map;
* .point GeoPoint type
* .shape GeoShape type
*/
public class ExternalMapper implements Mapper {
public class ExternalMapper extends AbstractFieldMapper<Object> {
/**
* Returns the actual value of the field.
*
* @param value
*/
@Override
public Object value(Object value) {
return null;
}
public static class Names {
public static final String FIELD_BIN = "bin";
public static final String FIELD_BOOL = "bool";
@ -48,16 +69,27 @@ public class ExternalMapper implements Mapper {
public static final String FIELD_SHAPE = "shape";
}
public static class Builder extends Mapper.Builder<Builder, ExternalMapper> {
public static class Builder extends AbstractFieldMapper.Builder<Builder, ExternalMapper> {
private BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN);
private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL);
private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT);
private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE);
private Mapper.Builder stringBuilder;
private String generatedValue;
private String mapperName;
public Builder(String name) {
super(name);
public Builder(String name, String generatedValue, String mapperName) {
super(name, new FieldType(Defaults.FIELD_TYPE));
this.builder = this;
this.stringBuilder = stringField(name).store(false);
this.generatedValue = generatedValue;
this.mapperName = mapperName;
}
public Builder string(Mapper.Builder content) {
this.stringBuilder = content;
return this;
}
@Override
@ -70,81 +102,107 @@ public class ExternalMapper implements Mapper {
BooleanFieldMapper boolMapper = boolBuilder.build(context);
GeoPointFieldMapper pointMapper = pointBuilder.build(context);
GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context);
Mapper stringMapper = stringBuilder.build(context);
context.path().remove();
context.path().pathType(origPathType);
return new ExternalMapper(name, binMapper, boolMapper, pointMapper, shapeMapper);
return new ExternalMapper(buildNames(context), generatedValue, mapperName, binMapper, boolMapper, pointMapper, shapeMapper, stringMapper,
multiFieldsBuilder.build(this, context), copyTo);
}
}
public static class TypeParser implements Mapper.TypeParser {
private String generatedValue;
private String mapperName;
TypeParser(String mapperName, String generatedValue) {
this.mapperName = mapperName;
this.generatedValue = generatedValue;
}
@SuppressWarnings({"unchecked"})
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
ExternalMapper.Builder builder = new ExternalMapper.Builder(name);
ExternalMapper.Builder builder = new ExternalMapper.Builder(name, generatedValue, mapperName);
parseField(builder, name, node, parserContext);
for (Map.Entry<String, Object> entry : node.entrySet()) {
String propName = Strings.toUnderscoreCase(entry.getKey());
Object propNode = entry.getValue();
parseMultiField(builder, name, node, parserContext, propName, propNode);
}
return builder;
}
}
private final String name;
private final String generatedValue;
private final String mapperName;
private final BinaryFieldMapper binMapper;
private final BooleanFieldMapper boolMapper;
private final GeoPointFieldMapper pointMapper;
private final GeoShapeFieldMapper shapeMapper;
private final Mapper stringMapper;
public ExternalMapper(String name,
BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, GeoShapeFieldMapper shapeMapper) {
this.name = name;
public ExternalMapper(FieldMapper.Names names,
String generatedValue, String mapperName,
BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper,
GeoShapeFieldMapper shapeMapper, Mapper stringMapper, MultiFields multiFields, CopyTo copyTo) {
super(names, 1.0f, Defaults.FIELD_TYPE, false, null, null, null, null, null, null, null, ImmutableSettings.EMPTY,
multiFields, copyTo);
this.generatedValue = generatedValue;
this.mapperName = mapperName;
this.binMapper = binMapper;
this.boolMapper = boolMapper;
this.pointMapper = pointMapper;
this.shapeMapper = shapeMapper;
this.stringMapper = stringMapper;
}
@Override
public String name() {
return name;
public FieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
}
@Override
public FieldDataType defaultFieldDataType() {
return null;
}
@Override
public void parse(ParseContext context) throws IOException {
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(ContentPath.Type.FULL);
context.path().add(name);
// Let's add a Dummy Binary content
context.path().add(Names.FIELD_BIN);
byte[] bytes = "Hello world".getBytes(Charset.defaultCharset());
context.externalValue(bytes);
binMapper.parse(context);
context.path().remove();
binMapper.parse(context.createExternalValueContext(bytes));
// Let's add a Dummy Boolean content
context.path().add(Names.FIELD_BOOL);
context.externalValue(true);
boolMapper.parse(context);
context.path().remove();
boolMapper.parse(context.createExternalValueContext(true));
// Let's add a Dummy Point
Double lat = 42.0;
Double lng = 51.0;
context.path().add(Names.FIELD_POINT);
GeoPoint point = new GeoPoint(lat, lng);
context.externalValue(point);
pointMapper.parse(context);
context.path().remove();
pointMapper.parse(context.createExternalValueContext(point));
// Let's add a Dummy Shape
context.path().add(Names.FIELD_SHAPE);
Point shape = ShapeBuilder.newPoint(-100, 45).build();
context.externalValue(shape);
shapeMapper.parse(context);
context.path().remove();
shapeMapper.parse(context.createExternalValueContext(shape));
context.path().pathType(origPathType);
context = context.createExternalValueContext(generatedValue);
// Let's add a Original String
stringMapper.parse(context);
multiFields.parse(this, context);
if (copyTo != null) {
copyTo.parse(context);
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
throw new UnsupportedOperationException();
}
@Override
@ -158,6 +216,7 @@ public class ExternalMapper implements Mapper {
boolMapper.traverse(fieldMapperListener);
pointMapper.traverse(fieldMapperListener);
shapeMapper.traverse(fieldMapperListener);
stringMapper.traverse(fieldMapperListener);
}
@Override
@ -170,20 +229,20 @@ public class ExternalMapper implements Mapper {
boolMapper.close();
pointMapper.close();
shapeMapper.close();
stringMapper.close();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field("type", RegisterExternalTypes.EXTERNAL);
builder.startObject("fields");
binMapper.toXContent(builder, params);
boolMapper.toXContent(builder, params);
pointMapper.toXContent(builder, params);
shapeMapper.toXContent(builder, params);
builder.endObject();
builder.startObject(name());
builder.field("type", mapperName);
multiFields.toXContent(builder, params);
builder.endObject();
return builder;
}
@Override
protected String contentType() {
return mapperName;
}
}

65
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java
View File

@ -24,14 +24,12 @@ import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import java.io.IOException;
import static org.hamcrest.Matchers.equalTo;
/**
@ -48,39 +46,78 @@ public class ExternalValuesMapperIntegrationTests extends ElasticsearchIntegrati
}
@Test
public void testExternalGeoPoint() throws Exception {
prepareCreate("test-idx").addMapping("doc", createMapping()).execute().get();
public void testExternalValues() throws Exception {
prepareCreate("test-idx").addMapping("type",
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field").field("type", RegisterExternalTypes.EXTERNAL).endObject()
.endObject()
.endObject().endObject()).execute().get();
ensureYellow("test-idx");
index("test-idx", "doc", "1", "external", "dummy");
index("test-idx", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject());
refresh();
SearchResponse response;
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.termFilter("external.bool", "T"))
.setPostFilter(FilterBuilders.termFilter("field.bool", "T"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.geoDistanceRangeFilter("external.point").point(42.0, 51.0).to("1km"))
.setPostFilter(FilterBuilders.geoDistanceRangeFilter("field.point").point(42.0, 51.0).to("1km"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.geoShapeFilter("external.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN))
.setPostFilter(FilterBuilders.geoShapeFilter("field.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
.setPostFilter(FilterBuilders.termFilter("field.field", "foo"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
}
@Test
public void testExternalValuesWithMultifield() throws Exception {
prepareCreate("test-idx").addMapping("doc",
XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties")
.startObject("f")
.field("type", RegisterExternalTypes.EXTERNAL_UPPER)
.startObject("fields")
.startObject("f")
.field("type", "string")
.field("stored", "yes")
.startObject("fields")
.startObject("raw")
.field("type", "string")
.field("index", "not_analyzed")
.field("stored", "yes")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()).execute().get();
ensureYellow("test-idx");
private XContentBuilder createMapping() throws IOException {
return XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties")
.startObject("external").field("type", RegisterExternalTypes.EXTERNAL).endObject()
.endObject().endObject().endObject();
index("test-idx", "doc", "1", "f", "This is my text");
refresh();
SearchResponse response = client().prepareSearch("test-idx")
.setQuery(QueryBuilders.termQuery("f.f.raw", "FOO BAR"))
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo((long) 1));
}
}

6
src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java
View File

@ -28,11 +28,15 @@ import org.elasticsearch.index.settings.IndexSettings;
public class RegisterExternalTypes extends AbstractIndexComponent {
public static final String EXTERNAL = "external";
public static final String EXTERNAL_BIS = "external_bis";
public static final String EXTERNAL_UPPER = "external_upper";
@Inject
public RegisterExternalTypes(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) {
super(index, indexSettings);
mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser());
mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo"));
mapperService.documentMapperParser().putTypeParser(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar"));
mapperService.documentMapperParser().putTypeParser(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR"));
}
}

177
src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java Normal file
View File

@ -0,0 +1,177 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.externalvalues;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.test.ElasticsearchSingleNodeLuceneTestCase;
import org.junit.Test;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
/**
*/
public class SimpleExternalMappingTests extends ElasticsearchSingleNodeLuceneTestCase {
@Test
public void testExternalValues() throws Exception {
MapperService mapperService = createIndex("test").mapperService();
mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL,
new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo"));
DocumentMapper documentMapper = mapperService.documentMapperParser().parse(
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field").field("type", "external").endObject()
.endObject()
.endObject().endObject().string()
);
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
assertThat(doc.rootDoc().getField("field.point"), notNullValue());
assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
assertThat(doc.rootDoc().getField("field.field"), notNullValue());
assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo"));
}
@Test
public void testExternalValuesWithMultifield() throws Exception {
MapperService mapperService = createIndex("test").mapperService();
mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL,
new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo"));
DocumentMapper documentMapper = mapperService.documentMapperParser().parse(
XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", RegisterExternalTypes.EXTERNAL)
.startObject("fields")
.startObject("field")
.field("type", "string")
.field("stored", "yes")
.startObject("fields")
.startObject("raw")
.field("type", "string")
.field("index", "not_analyzed")
.field("stored", "yes")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()
.string());
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
assertThat(doc.rootDoc().getField("field.point"), notNullValue());
assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
assertThat(doc.rootDoc().getField("field.field"), notNullValue());
assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo"));
assertThat(doc.rootDoc().getField("field.field.raw"), notNullValue());
assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo"));
}
@Test
public void testExternalValuesWithMultifieldTwoLevels() throws Exception {
MapperService mapperService = createIndex("test").mapperService();
mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL,
new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo"));
mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL_BIS,
new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL_BIS, "bar"));
DocumentMapper documentMapper = mapperService.documentMapperParser().parse(
XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", RegisterExternalTypes.EXTERNAL)
.startObject("fields")
.startObject("field")
.field("type", "string")
.startObject("fields")
.startObject("generated")
.field("type", RegisterExternalTypes.EXTERNAL_BIS)
.endObject()
.startObject("raw")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.startObject("raw")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()
.string());
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
assertThat(doc.rootDoc().getField("field.point"), notNullValue());
assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
assertThat(doc.rootDoc().getField("field.field"), notNullValue());
assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo"));
assertThat(doc.rootDoc().getField("field.field.generated.generated"), notNullValue());
assertThat(doc.rootDoc().getField("field.field.generated.generated").stringValue(), is("bar"));
assertThat(doc.rootDoc().getField("field.field.raw"), notNullValue());
assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo"));
assertThat(doc.rootDoc().getField("field.raw"), notNullValue());
assertThat(doc.rootDoc().getField("field.raw").stringValue(), is("foo"));
}
}