honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more refactoring to generalize root level field mappers

This commit is contained in:
Shay Banon 2011-08-18 00:57:53 +03:00
parent 4395202ebc
commit 821c3524a2
13 changed files with 410 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -48,7 +48,10 @@ import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
@ -137,23 +140,14 @@ public class DocumentMapper implements ToXContent {
private IdFieldMapper idFieldMapper = new IdFieldMapper();
private TypeFieldMapper typeFieldMapper = new TypeFieldMapper();
private IndexFieldMapper indexFieldMapper = new IndexFieldMapper();
private SourceFieldMapper sourceFieldMapper = new SourceFieldMapper();
private SizeFieldMapper sizeFieldMapper = new SizeFieldMapper();
private RoutingFieldMapper routingFieldMapper = new RoutingFieldMapper();
private BoostFieldMapper boostFieldMapper = new BoostFieldMapper();
private AllFieldMapper allFieldMapper = new AllFieldMapper();
private AnalyzerMapper analyzerMapper = new AnalyzerMapper();
private ParentFieldMapper parentFieldMapper = null;
private Map<String, RootMapper> rootMappers = new HashMap<String, RootMapper>();
private NamedAnalyzer indexAnalyzer;
private NamedAnalyzer searchAnalyzer;
@ -175,6 +169,13 @@ public class DocumentMapper implements ToXContent {
idFieldMapper = new IdFieldMapper(Field.Index.NOT_ANALYZED);
}
}
// add default mappers
this.rootMappers.put(SizeFieldMapper.NAME, new SizeFieldMapper());
this.rootMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper());
this.rootMappers.put(SourceFieldMapper.NAME, new SourceFieldMapper());
this.rootMappers.put(TypeFieldMapper.NAME, new TypeFieldMapper());
this.rootMappers.put(AllFieldMapper.NAME, new AllFieldMapper());
this.rootMappers.put(AnalyzerMapper.NAME, new AnalyzerMapper());
}
public Builder meta(ImmutableMap<String, Object> meta) {
@ -182,13 +183,8 @@ public class DocumentMapper implements ToXContent {
return this;
}
public Builder sourceField(SourceFieldMapper.Builder builder) {
this.sourceFieldMapper = builder.build(builderContext);
return this;
}
public Builder sizeField(SizeFieldMapper.Builder builder) {
this.sizeFieldMapper = builder.build(builderContext);
public Builder put(RootMapper.Builder mapper) {
rootMappers.put(mapper.name(), (RootMapper) mapper.build(builderContext));
return this;
}
@ -202,16 +198,6 @@ public class DocumentMapper implements ToXContent {
return this;
}
public Builder typeField(TypeFieldMapper.Builder builder) {
this.typeFieldMapper = builder.build(builderContext);
return this;
}
public Builder indexField(IndexFieldMapper.Builder builder) {
this.indexFieldMapper = builder.build(builderContext);
return this;
}
public Builder routingField(RoutingFieldMapper.Builder builder) {
this.routingFieldMapper = builder.build(builderContext);
return this;
@ -227,16 +213,6 @@ public class DocumentMapper implements ToXContent {
return this;
}
public Builder allField(AllFieldMapper.Builder builder) {
this.allFieldMapper = builder.build(builderContext);
return this;
}
public Builder analyzerField(AnalyzerMapper.Builder builder) {
this.analyzerMapper = builder.build(builderContext);
return this;
}
public Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) {
this.indexAnalyzer = indexAnalyzer;
return this;
@ -257,8 +233,9 @@ public class DocumentMapper implements ToXContent {
public DocumentMapper build(DocumentMapperParser docMapperParser) {
Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
return new DocumentMapper(index, docMapperParser, rootObjectMapper, meta, uidFieldMapper, idFieldMapper, typeFieldMapper, indexFieldMapper,
sourceFieldMapper, sizeFieldMapper, parentFieldMapper, routingFieldMapper, allFieldMapper, analyzerMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper);
return new DocumentMapper(index, docMapperParser, rootObjectMapper, meta, uidFieldMapper, idFieldMapper,
parentFieldMapper, routingFieldMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper,
rootMappers);
}
}
@ -283,25 +260,18 @@ public class DocumentMapper implements ToXContent {
private final IdFieldMapper idFieldMapper;
private final TypeFieldMapper typeFieldMapper;
private final IndexFieldMapper indexFieldMapper;
private final SourceFieldMapper sourceFieldMapper;
private final SizeFieldMapper sizeFieldMapper;
private final RoutingFieldMapper routingFieldMapper;
private final ParentFieldMapper parentFieldMapper;
private final BoostFieldMapper boostFieldMapper;
private final AllFieldMapper allFieldMapper;
private final AnalyzerMapper analyzerMapper;
private final RootObjectMapper rootObjectMapper;
private final ImmutableMap<String, RootMapper> rootMappers;
private final RootMapper[] rootMappersOrdered;
private final RootMapper[] rootMappersNotIncludedInObject;
private final NamedAnalyzer indexAnalyzer;
private final NamedAnalyzer searchAnalyzer;
@ -325,16 +295,11 @@ public class DocumentMapper implements ToXContent {
ImmutableMap<String, Object> meta,
UidFieldMapper uidFieldMapper,
IdFieldMapper idFieldMapper,
TypeFieldMapper typeFieldMapper,
IndexFieldMapper indexFieldMapper,
SourceFieldMapper sourceFieldMapper,
SizeFieldMapper sizeFieldMapper,
@Nullable ParentFieldMapper parentFieldMapper,
RoutingFieldMapper routingFieldMapper,
AllFieldMapper allFieldMapper,
AnalyzerMapper analyzerMapper,
NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
@Nullable BoostFieldMapper boostFieldMapper) {
@Nullable BoostFieldMapper boostFieldMapper,
Map<String, RootMapper> rootMappers) {
this.index = index;
this.type = rootObjectMapper.name();
this.docMapperParser = docMapperParser;
@ -342,16 +307,20 @@ public class DocumentMapper implements ToXContent {
this.rootObjectMapper = rootObjectMapper;
this.uidFieldMapper = uidFieldMapper;
this.idFieldMapper = idFieldMapper;
this.typeFieldMapper = typeFieldMapper;
this.indexFieldMapper = indexFieldMapper;
this.sourceFieldMapper = sourceFieldMapper;
this.sizeFieldMapper = sizeFieldMapper;
this.parentFieldMapper = parentFieldMapper;
this.routingFieldMapper = routingFieldMapper;
this.allFieldMapper = allFieldMapper;
this.analyzerMapper = analyzerMapper;
this.boostFieldMapper = boostFieldMapper;
this.rootMappers = ImmutableMap.copyOf(rootMappers);
this.rootMappersOrdered = rootMappers.values().toArray(new RootMapper[rootMappers.values().size()]);
List<RootMapper> rootMappersNotIncludedInObjectLst = newArrayList();
for (RootMapper rootMapper : rootMappersOrdered) {
if (!rootMapper.includeInObject()) {
rootMappersNotIncludedInObjectLst.add(rootMapper);
}
}
this.rootMappersNotIncludedInObject = rootMappersNotIncludedInObjectLst.toArray(new RootMapper[rootMappersNotIncludedInObjectLst.size()]);
this.indexAnalyzer = indexAnalyzer;
this.searchAnalyzer = searchAnalyzer;
@ -369,15 +338,18 @@ public class DocumentMapper implements ToXContent {
rootObjectMapper.putMapper(routingFieldMapper);
final List<FieldMapper> tempFieldMappers = newArrayList();
// add the basic ones
if (indexFieldMapper.enabled()) {
tempFieldMappers.add(indexFieldMapper);
for (RootMapper rootMapper : rootMappersOrdered) {
if (rootMapper.includeInObject()) {
rootObjectMapper.putMapper(rootMapper);
} else {
if (rootMapper instanceof FieldMapper) {
tempFieldMappers.add((FieldMapper) rootMapper);
}
}
}
tempFieldMappers.add(typeFieldMapper);
tempFieldMappers.add(sourceFieldMapper);
tempFieldMappers.add(sizeFieldMapper);
// add the basic ones
tempFieldMappers.add(uidFieldMapper);
tempFieldMappers.add(allFieldMapper);
// now traverse and get all the statically defined ones
rootObjectMapper.traverse(new FieldMapperListener() {
@Override public void fieldMapper(FieldMapper fieldMapper) {
@ -423,28 +395,20 @@ public class DocumentMapper implements ToXContent {
return this.uidFieldMapper;
}
public IdFieldMapper idMapper() {
return this.idFieldMapper;
}
public IndexFieldMapper indexMapper() {
return this.indexFieldMapper;
@SuppressWarnings({"unchecked"}) public <T extends RootMapper> T rootMapper(String name) {
return (T) rootMappers.get(name);
}
public TypeFieldMapper typeMapper() {
return this.typeFieldMapper;
return rootMapper(TypeFieldMapper.NAME);
}
public SourceFieldMapper sourceMapper() {
return this.sourceFieldMapper;
}
public BoostFieldMapper boostMapper() {
return this.boostFieldMapper;
return rootMapper(SourceFieldMapper.NAME);
}
public AllFieldMapper allFieldMapper() {
return this.allFieldMapper;
return rootMapper(AllFieldMapper.NAME);
}
public RoutingFieldMapper routingFieldMapper() {
@ -542,27 +506,20 @@ public class DocumentMapper implements ToXContent {
// }
}
if (sizeFieldMapper.enabled()) {
context.externalValue(source.source().length);
sizeFieldMapper.parse(context);
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.preParse(context);
}
if (sourceFieldMapper.enabled()) {
sourceFieldMapper.parse(context);
}
// set the id if we have it so we can validate it later on, also, add the uid if we can
if (source.id() != null) {
context.id(source.id());
uidFieldMapper.parse(context);
}
typeFieldMapper.parse(context);
if (source.routing() != null) {
context.externalValue(source.routing());
routingFieldMapper.parse(context);
}
indexFieldMapper.parse(context);
if (!emptyDoc) {
rootObjectMapper.parse(context);
}
@ -601,11 +558,20 @@ public class DocumentMapper implements ToXContent {
idFieldMapper.parse(context);
}
}
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.postParse(context);
}
if (parentFieldMapper != null) {
parentFieldMapper.parse(context);
}
analyzerMapper.parse(context);
allFieldMapper.parse(context);
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.validate(context);
}
// validate aggregated mappers (TODO: need to be added as a phase to any field mapper)
routingFieldMapper.validate(context);
} catch (IOException e) {
@ -639,14 +605,12 @@ public class DocumentMapper implements ToXContent {
public void addFieldMapperListener(FieldMapperListener fieldMapperListener, boolean includeExisting) {
fieldMapperListeners.add(fieldMapperListener);
if (includeExisting) {
if (indexFieldMapper.enabled()) {
fieldMapperListener.fieldMapper(indexFieldMapper);
}
fieldMapperListener.fieldMapper(sourceFieldMapper);
fieldMapperListener.fieldMapper(sizeFieldMapper);
fieldMapperListener.fieldMapper(typeFieldMapper);
fieldMapperListener.fieldMapper(uidFieldMapper);
fieldMapperListener.fieldMapper(allFieldMapper);
for (RootMapper rootMapper : rootMappersOrdered) {
if (!rootMapper.includeInObject() && rootMapper instanceof FieldMapper) {
fieldMapperListener.fieldMapper((FieldMapper) rootMapper);
}
}
rootObjectMapper.traverse(fieldMapperListener);
}
}
@ -674,10 +638,16 @@ public class DocumentMapper implements ToXContent {
MergeContext mergeContext = new MergeContext(this, mergeFlags);
rootObjectMapper.merge(mergeWith.rootObjectMapper, mergeContext);
allFieldMapper.merge(mergeWith.allFieldMapper, mergeContext);
analyzerMapper.merge(mergeWith.analyzerMapper, mergeContext);
sourceFieldMapper.merge(mergeWith.sourceFieldMapper, mergeContext);
sizeFieldMapper.merge(mergeWith.sizeFieldMapper, mergeContext);
for (Map.Entry<String, RootMapper> entry : rootMappers.entrySet()) {
// root mappers included in root object will get merge in the rootObjectMapper
if (entry.getValue().includeInObject()) {
continue;
}
RootMapper mergeWithRootMapper = mergeWith.rootMappers.get(entry.getKey());
if (mergeWithRootMapper != null) {
entry.getValue().merge(mergeWithRootMapper, mergeContext);
}
}
if (!mergeFlags.simulate()) {
// let the merge with attributes to override the attributes
@ -704,15 +674,14 @@ public class DocumentMapper implements ToXContent {
cache.remove();
rootObjectMapper.close();
idFieldMapper.close();
indexFieldMapper.close();
typeFieldMapper.close();
allFieldMapper.close();
analyzerMapper.close();
sourceFieldMapper.close();
sizeFieldMapper.close();
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.close();
}
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
List<Mapper> additional = new ArrayList<Mapper>();
additional.addAll(Arrays.asList(rootMappersNotIncludedInObject));
rootObjectMapper.toXContent(builder, params, new ToXContent() {
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (indexAnalyzer != null && searchAnalyzer != null && indexAnalyzer.name().equals(searchAnalyzer.name()) && !indexAnalyzer.name().startsWith("_")) {
@ -740,7 +709,7 @@ public class DocumentMapper implements ToXContent {
}
// no need to pass here id and boost, since they are added to the root object mapper
// in the constructor
}, indexFieldMapper, typeFieldMapper, allFieldMapper, analyzerMapper, sourceFieldMapper, sizeFieldMapper);
}, additional.toArray(new Mapper[additional.size()]));
return builder;
}
}

123
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
@ -62,6 +61,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
private final Object typeParsersMutex = new Object();
private volatile ImmutableMap<String, Mapper.TypeParser> typeParsers;
private volatile ImmutableMap<String, Mapper.TypeParser> rootTypeParsers;
public DocumentMapperParser(Index index, AnalysisService analysisService) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService);
@ -87,6 +87,15 @@ public class DocumentMapperParser extends AbstractIndexComponent {
.put(MultiFieldMapper.CONTENT_TYPE, new MultiFieldMapper.TypeParser())
.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser())
.immutableMap();
rootTypeParsers = new MapBuilder<String, Mapper.TypeParser>()
.put(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())
.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser())
.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser())
.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser())
.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser())
.put(AnalyzerMapper.NAME, new AnalyzerMapper.TypeParser())
.immutableMap();
}
public void putTypeParser(String type, Mapper.TypeParser typeParser) {
@ -98,6 +107,15 @@ public class DocumentMapperParser extends AbstractIndexComponent {
}
}
public void putRootTypeParser(String type, Mapper.TypeParser typeParser) {
synchronized (typeParsersMutex) {
rootTypeParsers = new MapBuilder<String, Mapper.TypeParser>()
.putAll(typeParsers)
.put(type, typeParser)
.immutableMap();
}
}
public Mapper.TypeParser.ParserContext parserContext() {
return new Mapper.TypeParser.ParserContext(analysisService, typeParsers);
}
@ -141,16 +159,8 @@ public class DocumentMapperParser extends AbstractIndexComponent {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (SourceFieldMapper.CONTENT_TYPE.equals(fieldName) || "sourceField".equals(fieldName)) {
docBuilder.sourceField(parseSourceField((Map<String, Object>) fieldNode, parserContext));
} else if (SizeFieldMapper.CONTENT_TYPE.equals(fieldName)) {
docBuilder.sizeField(parseSizeField((Map<String, Object>) fieldNode, parserContext));
} else if (IdFieldMapper.CONTENT_TYPE.equals(fieldName) || "idField".equals(fieldName)) {
if (IdFieldMapper.CONTENT_TYPE.equals(fieldName) || "idField".equals(fieldName)) {
docBuilder.idField(parseIdField((Map<String, Object>) fieldNode, parserContext));
} else if (IndexFieldMapper.CONTENT_TYPE.equals(fieldName) || "indexField".equals(fieldName)) {
docBuilder.indexField(parseIndexField((Map<String, Object>) fieldNode, parserContext));
} else if (TypeFieldMapper.CONTENT_TYPE.equals(fieldName) || "typeField".equals(fieldName)) {
docBuilder.typeField(parseTypeField((Map<String, Object>) fieldNode, parserContext));
} else if (UidFieldMapper.CONTENT_TYPE.equals(fieldName) || "uidField".equals(fieldName)) {
docBuilder.uidField(parseUidField((Map<String, Object>) fieldNode, parserContext));
} else if (RoutingFieldMapper.CONTENT_TYPE.equals(fieldName)) {
@ -159,10 +169,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
docBuilder.parentFiled(parseParentField((Map<String, Object>) fieldNode, parserContext));
} else if (BoostFieldMapper.CONTENT_TYPE.equals(fieldName) || "boostField".equals(fieldName)) {
docBuilder.boostField(parseBoostField((Map<String, Object>) fieldNode, parserContext));
} else if (AllFieldMapper.CONTENT_TYPE.equals(fieldName) || "allField".equals(fieldName)) {
docBuilder.allField(parseAllField((Map<String, Object>) fieldNode, parserContext));
} else if (AnalyzerMapper.CONTENT_TYPE.equals(fieldName)) {
docBuilder.analyzerField(parseAnalyzerField((Map<String, Object>) fieldNode, parserContext));
} else if ("index_analyzer".equals(fieldName)) {
docBuilder.indexAnalyzer(analysisService.analyzer(fieldNode.toString()));
} else if ("search_analyzer".equals(fieldName)) {
@ -170,6 +176,11 @@ public class DocumentMapperParser extends AbstractIndexComponent {
} else if ("analyzer".equals(fieldName)) {
docBuilder.indexAnalyzer(analysisService.analyzer(fieldNode.toString()));
docBuilder.searchAnalyzer(analysisService.analyzer(fieldNode.toString()));
} else {
Mapper.TypeParser typeParser = rootTypeParsers.get(fieldName);
if (typeParser != null) {
docBuilder.put(typeParser.parse(fieldName, (Map<String, Object>) fieldNode, parserContext));
}
}
}
@ -211,13 +222,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
return builder;
}
private TypeFieldMapper.Builder parseTypeField(Map<String, Object> typeNode, Mapper.TypeParser.ParserContext parserContext) {
TypeFieldMapper.Builder builder = type();
parseField(builder, builder.name, typeNode, parserContext);
return builder;
}
private IdFieldMapper.Builder parseIdField(Map<String, Object> idNode, Mapper.TypeParser.ParserContext parserContext) {
IdFieldMapper.Builder builder = id();
parseField(builder, builder.name, idNode, parserContext);
@ -252,83 +256,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
return builder;
}
private AnalyzerMapper.Builder parseAnalyzerField(Map<String, Object> analyzerNode, Mapper.TypeParser.ParserContext parserContext) {
AnalyzerMapper.Builder builder = analyzer();
for (Map.Entry<String, Object> entry : analyzerNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("path")) {
builder.field(fieldNode.toString());
}
}
return builder;
}
private AllFieldMapper.Builder parseAllField(Map<String, Object> allNode, Mapper.TypeParser.ParserContext parserContext) {
AllFieldMapper.Builder builder = all();
parseField(builder, builder.name, allNode, parserContext);
for (Map.Entry<String, Object> entry : allNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
}
}
return builder;
}
private SizeFieldMapper.Builder parseSizeField(Map<String, Object> node, Mapper.TypeParser.ParserContext parserContext) {
SizeFieldMapper.Builder builder = new SizeFieldMapper.Builder();
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("store")) {
builder.store(parseStore(fieldName, fieldNode.toString()));
}
}
return builder;
}
private SourceFieldMapper.Builder parseSourceField(Map<String, Object> sourceNode, Mapper.TypeParser.ParserContext parserContext) {
SourceFieldMapper.Builder builder = source();
for (Map.Entry<String, Object> entry : sourceNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress") && fieldNode != null) {
builder.compress(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress_threshold") && fieldNode != null) {
if (fieldNode instanceof Number) {
builder.compressThreshold(((Number) fieldNode).longValue());
builder.compress(true);
} else {
builder.compressThreshold(ByteSizeValue.parseBytesSizeValue(fieldNode.toString()).bytes());
builder.compress(true);
}
}
}
return builder;
}
private IndexFieldMapper.Builder parseIndexField(Map<String, Object> indexNode, Mapper.TypeParser.ParserContext parserContext) {
IndexFieldMapper.Builder builder = MapperBuilders.index();
parseField(builder, builder.name, indexNode, parserContext);
for (Map.Entry<String, Object> entry : indexNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
}
}
return builder;
}
@SuppressWarnings({"unchecked"})
private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
Map<String, Object> root;

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -53,7 +53,7 @@ public interface Mapper extends ToXContent {
@NotThreadSafe
public static abstract class Builder<T extends Builder, Y extends Mapper> {
protected String name;
public String name;
protected T builder;

39
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/RootMapper.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import java.io.IOException;
/**
* A mapper that exists only as a mapper within a root object.
*/
public interface RootMapper extends Mapper {
void preParse(ParseContext context) throws IOException;
void postParse(ParseContext context) throws IOException;
void validate(ParseContext context) throws MapperParsingException;
/**
* Should the mapper be included in the root {@link org.elasticsearch.index.mapper.object.ObjectMapper}.
*/
boolean includeInObject();
}

43
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.all.AllField;
import org.elasticsearch.common.lucene.all.AllTermQuery;
@ -31,18 +32,25 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.*;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* @author kimchy (shay.banon)
*/
public class AllFieldMapper extends AbstractFieldMapper<Void> implements InternalMapper {
public class AllFieldMapper extends AbstractFieldMapper<Void> implements InternalMapper, RootMapper {
public interface IncludeInAll extends Mapper {
@ -98,6 +106,21 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
AllFieldMapper.Builder builder = all();
parseField(builder, builder.name, node, parserContext);
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
}
}
return builder;
}
}
private boolean enabled;
@ -124,6 +147,24 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
return new AllTermQuery(termFactory.createTerm(value));
}
@Override public void preParse(ParseContext context) throws IOException {
}
@Override public void postParse(ParseContext context) throws IOException {
super.parse(context);
}
@Override public void parse(ParseContext context) throws IOException {
// we parse in post parse
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;

50
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AnalyzerMapper.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.FieldMapperListener;
import org.elasticsearch.index.mapper.InternalMapper;
@ -29,14 +30,19 @@ import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ObjectMapperListener;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
/**
* @author kimchy (shay.banon)
*/
public class AnalyzerMapper implements Mapper, InternalMapper {
public class AnalyzerMapper implements Mapper, InternalMapper, RootMapper {
public static final String NAME = "_analyzer";
public static final String CONTENT_TYPE = "_analyzer";
public static class Defaults {
@ -62,20 +68,19 @@ public class AnalyzerMapper implements Mapper, InternalMapper {
}
}
// for now, it is parsed directly in the document parser, need to move this internal types parsing to be done here as well...
// public static class TypeParser implements XContentMapper.TypeParser {
// @Override public XContentMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
// AnalyzerMapper.Builder builder = analyzer();
// for (Map.Entry<String, Object> entry : node.entrySet()) {
// String fieldName = Strings.toUnderscoreCase(entry.getKey());
// Object fieldNode = entry.getValue();
// if ("path".equals(fieldName)) {
// builder.field(fieldNode.toString());
// }
// }
// return builder;
// }
// }
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
AnalyzerMapper.Builder builder = analyzer();
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("path")) {
builder.field(fieldNode.toString());
}
}
return builder;
}
}
private final String path;
@ -91,7 +96,10 @@ public class AnalyzerMapper implements Mapper, InternalMapper {
return CONTENT_TYPE;
}
@Override public void parse(ParseContext context) throws IOException {
@Override public void preParse(ParseContext context) throws IOException {
}
@Override public void postParse(ParseContext context) throws IOException {
Analyzer analyzer = context.docMapper().mappers().indexAnalyzer();
if (path != null) {
String value = context.doc().get(path);
@ -109,6 +117,16 @@ public class AnalyzerMapper implements Mapper, InternalMapper {
context.analyzer(analyzer);
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override public void parse(ParseContext context) throws IOException {
}
@Override public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
}

45
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
View File

@ -23,21 +23,29 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* @author kimchy (shay.banon)
*/
public class IndexFieldMapper extends AbstractFieldMapper<String> implements InternalMapper {
public class IndexFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
public static final String NAME = "_index";
@ -76,6 +84,22 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
IndexFieldMapper.Builder builder = MapperBuilders.index();
parseField(builder, builder.name, node, parserContext);
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
}
}
return builder;
}
}
private final boolean enabled;
public IndexFieldMapper() {
@ -123,6 +147,25 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
return termFactory.createTerm(value);
}
@Override public void preParse(ParseContext context) throws IOException {
// we pre parse it and not in parse, since its not part of the root object
super.parse(context);
}
@Override public void postParse(ParseContext context) throws IOException {
}
@Override public void parse(ParseContext context) throws IOException {
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;

47
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java
View File

@ -21,17 +21,25 @@ package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.IntegerFieldMapper;
import java.io.IOException;
import java.util.Map;
public class SizeFieldMapper extends IntegerFieldMapper {
import static org.elasticsearch.common.xcontent.support.XContentMapValues.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
public static final String NAME = "_size";
public static final String CONTENT_TYPE = "_size";
public static class Defaults extends IntegerFieldMapper.Defaults {
@ -65,6 +73,22 @@ public class SizeFieldMapper extends IntegerFieldMapper {
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
SizeFieldMapper.Builder builder = new SizeFieldMapper.Builder();
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("store")) {
builder.store(parseStore(fieldName, fieldNode.toString()));
}
}
return builder;
}
}
private final boolean enabled;
public SizeFieldMapper() {
@ -84,11 +108,30 @@ public class SizeFieldMapper extends IntegerFieldMapper {
return this.enabled;
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public void preParse(ParseContext context) throws IOException {
}
@Override public void postParse(ParseContext context) throws IOException {
// we post parse it so we get the size stored, possibly compressed (source will be preParse)
super.parse(context);
}
@Override public void parse(ParseContext context) throws IOException {
// nothing to do here, we call the parent in postParse
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
return new CustomIntegerNumericField(this, ((Number) context.externalValue()).intValue());
return new CustomIntegerNumericField(this, context.source().length);
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {

53
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.lzf.LZF;
import org.elasticsearch.common.compress.lzf.LZFDecoder;
import org.elasticsearch.common.compress.lzf.LZFEncoder;
@ -32,17 +33,23 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.*;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
/**
* @author kimchy (shay.banon)
*/
public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements InternalMapper {
public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements InternalMapper, RootMapper {
public static final String NAME = "_source";
@ -90,6 +97,32 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
SourceFieldMapper.Builder builder = source();
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress") && fieldNode != null) {
builder.compress(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress_threshold") && fieldNode != null) {
if (fieldNode instanceof Number) {
builder.compressThreshold(((Number) fieldNode).longValue());
builder.compress(true);
} else {
builder.compressThreshold(ByteSizeValue.parseBytesSizeValue(fieldNode.toString()).bytes());
builder.compress(true);
}
}
}
return builder;
}
}
private final boolean enabled;
private Boolean compress;
@ -116,6 +149,24 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
return SourceFieldSelector.INSTANCE;
}
@Override public void preParse(ParseContext context) throws IOException {
super.parse(context);
}
@Override public void postParse(ParseContext context) throws IOException {
}
@Override public void parse(ParseContext context) throws IOException {
// nothing to do here, we will call it in pre parse
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;

35
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
View File

@ -32,19 +32,25 @@ import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* @author kimchy (shay.banon)
*/
public class TypeFieldMapper extends AbstractFieldMapper<String> implements InternalMapper {
public class TypeFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
public static final String NAME = "_type";
@ -77,6 +83,15 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
TypeFieldMapper.Builder builder = type();
parseField(builder, builder.name, node, parserContext);
return builder;
}
}
public TypeFieldMapper() {
this(Defaults.NAME, Defaults.INDEX_NAME);
}
@ -132,6 +147,24 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
return true;
}
@Override public void preParse(ParseContext context) throws IOException {
super.parse(context);
}
@Override public void postParse(ParseContext context) throws IOException {
}
@Override public void parse(ParseContext context) throws IOException {
// we parse in pre parse
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
if (index == Field.Index.NO && store == Field.Store.NO) {
return null;

15
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java
View File

@ -40,8 +40,9 @@ public class IndexTypeMapperTests {
.startObject("_index").field("enabled", true).field("store", "yes").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
assertThat(docMapper.indexMapper().enabled(), equalTo(true));
assertThat(docMapper.indexMapper().store(), equalTo(Field.Store.YES));
IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.NAME);
assertThat(indexMapper.enabled(), equalTo(true));
assertThat(indexMapper.store(), equalTo(Field.Store.YES));
assertThat(docMapper.mappers().indexName("_index").mapper(), instanceOf(IndexFieldMapper.class));
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
@ -59,8 +60,9 @@ public class IndexTypeMapperTests {
.startObject("_index").field("enabled", false).field("store", "yes").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
assertThat(docMapper.indexMapper().enabled(), equalTo(false));
assertThat(docMapper.indexMapper().store(), equalTo(Field.Store.YES));
IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.NAME);
assertThat(indexMapper.enabled(), equalTo(false));
assertThat(indexMapper.store(), equalTo(Field.Store.YES));
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
@ -76,8 +78,9 @@ public class IndexTypeMapperTests {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
assertThat(docMapper.indexMapper().enabled(), equalTo(false));
assertThat(docMapper.indexMapper().store(), equalTo(Field.Store.NO));
IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.NAME);
assertThat(indexMapper.enabled(), equalTo(false));
assertThat(indexMapper.store(), equalTo(Field.Store.NO));
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()

2
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java
View File

@ -43,7 +43,7 @@ public class SimpleMapperTests {
DocumentMapper docMapper = doc("test",
rootObject("person")
.add(object("name").add(stringField("first").store(YES).index(Field.Index.NO)))
).sourceField(source()).build(mapperParser);
).build(mapperParser);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json");
Document doc = docMapper.parse("person", "1", json).rootDoc();

4
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java
View File

@ -75,7 +75,7 @@ public class DefaultSourceMappingTests {
MapperService mapperService = MapperTests.newMapperService();
mapperService.add(MapperService.DEFAULT_MAPPING, defaultMapping);
DocumentMapper mapper = (DocumentMapper) mapperService.documentMapperWithAutoCreate("my_type");
DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type");
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().enabled(), equalTo(false));
}
@ -93,7 +93,7 @@ public class DefaultSourceMappingTests {
.endObject().endObject().string();
mapperService.add("my_type", mapping);
DocumentMapper mapper = (DocumentMapper) mapperService.documentMapper("my_type");
DocumentMapper mapper = mapperService.documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
}