honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more work on simplifying mapper parsing code

This commit is contained in:
Shay Banon 2011-08-19 01:34:22 +03:00
parent 8d2123a452
commit 5fa072263d
7 changed files with 203 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.common.compress.lzf.LZF;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.CachedStreamInput;
import org.elasticsearch.common.io.stream.LZFStreamInput;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -133,10 +132,6 @@ public class DocumentMapper implements ToXContent {
public static class Builder {
private UidFieldMapper uidFieldMapper = new UidFieldMapper();
private IdFieldMapper idFieldMapper = new IdFieldMapper();
private Map<Class<? extends RootMapper>, RootMapper> rootMappers = Maps.newHashMap();
private NamedAnalyzer indexAnalyzer;
@ -154,12 +149,14 @@ public class DocumentMapper implements ToXContent {
public Builder(String index, @Nullable Settings indexSettings, RootObjectMapper.Builder builder) {
this.index = index;
this.rootObjectMapper = builder.build(builderContext);
IdFieldMapper idFieldMapper = new IdFieldMapper();
if (indexSettings != null) {
String idIndexed = indexSettings.get("index.mapping._id.indexed");
if (idIndexed != null && Booleans.parseBoolean(idIndexed, false)) {
idFieldMapper = new IdFieldMapper(Field.Index.NOT_ANALYZED);
}
}
this.rootMappers.put(IdFieldMapper.class, idFieldMapper);
// add default mappers
this.rootMappers.put(SizeFieldMapper.class, new SizeFieldMapper());
this.rootMappers.put(IndexFieldMapper.class, new IndexFieldMapper());
@ -169,6 +166,7 @@ public class DocumentMapper implements ToXContent {
this.rootMappers.put(AnalyzerMapper.class, new AnalyzerMapper());
this.rootMappers.put(BoostFieldMapper.class, new BoostFieldMapper());
this.rootMappers.put(RoutingFieldMapper.class, new RoutingFieldMapper());
this.rootMappers.put(UidFieldMapper.class, new UidFieldMapper());
// don't add parent field, by default its "null"
}
@ -183,16 +181,6 @@ public class DocumentMapper implements ToXContent {
return this;
}
public Builder idField(IdFieldMapper.Builder builder) {
this.idFieldMapper = builder.build(builderContext);
return this;
}
public Builder uidField(UidFieldMapper.Builder builder) {
this.uidFieldMapper = builder.build(builderContext);
return this;
}
public Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) {
this.indexAnalyzer = indexAnalyzer;
return this;
@ -213,7 +201,7 @@ public class DocumentMapper implements ToXContent {
public DocumentMapper build(DocumentMapperParser docMapperParser) {
Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
return new DocumentMapper(index, docMapperParser, rootObjectMapper, meta, uidFieldMapper, idFieldMapper,
return new DocumentMapper(index, docMapperParser, rootObjectMapper, meta,
indexAnalyzer, searchAnalyzer,
rootMappers);
}
@ -236,10 +224,6 @@ public class DocumentMapper implements ToXContent {
private volatile CompressedString mappingSource;
private final UidFieldMapper uidFieldMapper;
private final IdFieldMapper idFieldMapper;
private final RootObjectMapper rootObjectMapper;
private final ImmutableMap<Class<? extends RootMapper>, RootMapper> rootMappers;
@ -267,8 +251,6 @@ public class DocumentMapper implements ToXContent {
public DocumentMapper(String index, DocumentMapperParser docMapperParser,
RootObjectMapper rootObjectMapper,
ImmutableMap<String, Object> meta,
UidFieldMapper uidFieldMapper,
IdFieldMapper idFieldMapper,
NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
Map<Class<? extends RootMapper>, RootMapper> rootMappers) {
this.index = index;
@ -276,8 +258,6 @@ public class DocumentMapper implements ToXContent {
this.docMapperParser = docMapperParser;
this.meta = meta;
this.rootObjectMapper = rootObjectMapper;
this.uidFieldMapper = uidFieldMapper;
this.idFieldMapper = idFieldMapper;
this.rootMappers = ImmutableMap.copyOf(rootMappers);
this.rootMappersOrdered = rootMappers.values().toArray(new RootMapper[rootMappers.values().size()]);
@ -294,7 +274,6 @@ public class DocumentMapper implements ToXContent {
this.typeFilter = typeMapper().fieldFilter(type);
rootObjectMapper.putMapper(idFieldMapper);
if (rootMapper(ParentFieldMapper.class) != null) {
// mark the routing field mapper as required
rootMapper(RoutingFieldMapper.class).markAsRequired();
@ -311,8 +290,6 @@ public class DocumentMapper implements ToXContent {
}
}
// add the basic ones
tempFieldMappers.add(uidFieldMapper);
// now traverse and get all the statically defined ones
rootObjectMapper.traverse(new FieldMapperListener() {
@Override public void fieldMapper(FieldMapper fieldMapper) {
@ -355,7 +332,7 @@ public class DocumentMapper implements ToXContent {
}
public UidFieldMapper uidMapper() {
return this.uidFieldMapper;
return rootMapper(UidFieldMapper.class);
}
@SuppressWarnings({"unchecked"}) public <T extends RootMapper> T rootMapper(Class<T> type) {
@ -473,12 +450,6 @@ public class DocumentMapper implements ToXContent {
rootMapper.preParse(context);
}
// set the id if we have it so we can validate it later on, also, add the uid if we can
if (source.id() != null) {
context.id(source.id());
uidFieldMapper.parse(context);
}
if (!emptyDoc) {
rootObjectMapper.parse(context);
}
@ -487,37 +458,6 @@ public class DocumentMapper implements ToXContent {
parser.nextToken();
}
// if we did not get the id, we need to parse the uid into the document now, after it was added
if (source.id() == null) {
if (context.id() == null) {
if (!source.flyweight()) {
throw new MapperParsingException("No id found while parsing the content source");
}
} else {
uidFieldMapper.parse(context);
if (context.docs().size() > 1) {
UidField uidField = (UidField) context.doc().getFieldable(UidFieldMapper.NAME);
assert uidField != null;
// we need to go over the docs and add it...
for (int i = 1; i < context.docs().size(); i++) {
// we don't need to add it as a full uid field in nested docs, since we don't need versioning
context.docs().get(i).add(new Field(UidFieldMapper.NAME, uidField.uid(), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
}
if (context.parsedIdState() != ParseContext.ParsedIdState.PARSED) {
if (context.id() == null) {
if (!source.flyweight()) {
throw new MapperParsingException("No id mapping with [_id] found in the content, and not explicitly set");
}
} else {
// mark it as external, so we can parse it
context.parsedId(ParseContext.ParsedIdState.EXTERNAL);
idFieldMapper.parse(context);
}
}
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.postParse(context);
}
@ -556,7 +496,6 @@ public class DocumentMapper implements ToXContent {
public void addFieldMapperListener(FieldMapperListener fieldMapperListener, boolean includeExisting) {
fieldMapperListeners.add(fieldMapperListener);
if (includeExisting) {
fieldMapperListener.fieldMapper(uidFieldMapper);
for (RootMapper rootMapper : rootMappersOrdered) {
if (!rootMapper.includeInObject() && rootMapper instanceof FieldMapper) {
fieldMapperListener.fieldMapper((FieldMapper) rootMapper);
@ -624,7 +563,6 @@ public class DocumentMapper implements ToXContent {
public void close() {
cache.remove();
rootObjectMapper.close();
idFieldMapper.close();
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.close();
}

20
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -46,7 +46,6 @@ import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* @author kimchy (shay.banon)
@ -97,6 +96,8 @@ public class DocumentMapperParser extends AbstractIndexComponent {
.put(BoostFieldMapper.NAME, new BoostFieldMapper.TypeParser())
.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser())
.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser())
.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser())
.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser())
.immutableMap();
}
@ -161,11 +162,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (IdFieldMapper.CONTENT_TYPE.equals(fieldName) || "idField".equals(fieldName)) {
docBuilder.idField(parseIdField((Map<String, Object>) fieldNode, parserContext));
} else if (UidFieldMapper.CONTENT_TYPE.equals(fieldName) || "uidField".equals(fieldName)) {
docBuilder.uidField(parseUidField((Map<String, Object>) fieldNode, parserContext));
} else if ("index_analyzer".equals(fieldName)) {
if ("index_analyzer".equals(fieldName)) {
docBuilder.indexAnalyzer(analysisService.analyzer(fieldNode.toString()));
} else if ("search_analyzer".equals(fieldName)) {
docBuilder.searchAnalyzer(analysisService.analyzer(fieldNode.toString()));
@ -199,17 +196,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
return documentMapper;
}
private UidFieldMapper.Builder parseUidField(Map<String, Object> uidNode, Mapper.TypeParser.ParserContext parserContext) {
UidFieldMapper.Builder builder = uid();
return builder;
}
private IdFieldMapper.Builder parseIdField(Map<String, Object> idNode, Mapper.TypeParser.ParserContext parserContext) {
IdFieldMapper.Builder builder = id();
parseField(builder, builder.name, idNode, parserContext);
return builder;
}
@SuppressWarnings({"unchecked"})
private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
Map<String, Object> root;

17
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -67,8 +67,6 @@ public class ParseContext {
private Map<String, String> ignoredValues = new HashMap<String, String>();
private ParsedIdState parsedIdState;
private boolean mappersAdded = false;
private boolean externalValueSet;
@ -99,7 +97,6 @@ public class ParseContext {
this.sourceToParse = source;
this.source = source == null ? null : sourceToParse.source();
this.path.reset();
this.parsedIdState = ParsedIdState.NO;
this.mappersAdded = false;
this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener;
this.allEntries = new AllEntries();
@ -193,14 +190,6 @@ public class ParseContext {
return id;
}
public void parsedId(ParsedIdState parsedIdState) {
this.parsedIdState = parsedIdState;
}
public ParsedIdState parsedIdState() {
return this.parsedIdState;
}
public void ignoredValue(String indexName, String value) {
ignoredValues.put(indexName, value);
}
@ -273,10 +262,4 @@ public class ParseContext {
stringBuilder.setLength(0);
return this.stringBuilder;
}
public static enum ParsedIdState {
NO,
PARSED,
EXTERNAL
}
}

62
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
View File

@ -24,20 +24,26 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* @author kimchy (shay.banon)
*/
public class IdFieldMapper extends AbstractFieldMapper<String> implements InternalMapper {
public class IdFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
public static final String NAME = "_id";
@ -68,6 +74,14 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
IdFieldMapper.Builder builder = id();
parseField(builder, builder.name, node, parserContext);
return builder;
}
}
public IdFieldMapper() {
this(Defaults.NAME, Defaults.INDEX_NAME, Defaults.INDEX);
}
@ -108,28 +122,50 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
return value;
}
@Override public void preParse(ParseContext context) throws IOException {
if (context.sourceToParse().id() != null) {
context.id(context.sourceToParse().id());
super.parse(context);
}
}
@Override public void postParse(ParseContext context) throws IOException {
if (context.id() == null && !context.sourceToParse().flyweight()) {
throw new MapperParsingException("No id found while parsing the content source");
}
// it either get built in the preParse phase, or get parsed...
}
@Override public void parse(ParseContext context) throws IOException {
super.parse(context);
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return true;
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
if (context.parsedIdState() == ParseContext.ParsedIdState.NO) {
String id = context.parser().text();
XContentParser parser = context.parser();
if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
// we are in the parse Phase
String id = parser.text();
if (context.id() != null && !context.id().equals(id)) {
throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
}
context.id(id);
context.parsedId(ParseContext.ParsedIdState.PARSED);
if (index == Field.Index.NO && store == Field.Store.NO) {
return null;
}
return new Field(names.indexName(), false, context.id(), store, index, termVector);
} else if (context.parsedIdState() == ParseContext.ParsedIdState.EXTERNAL) {
if (context.id() == null) {
throw new MapperParsingException("No id mapping with [" + names.name() + "] found in the content, and not explicitly set");
}
if (index == Field.Index.NO && store == Field.Store.NO) {
return null;
}
return new Field(names.indexName(), false, context.id(), store, index, termVector);
} else {
throw new MapperParsingException("Illegal parsed id state");
// we are in the pre/post parse phase
if (index == Field.Index.NO && store == Field.Store.NO) {
return null;
}
return new Field(names.indexName(), false, context.id(), store, index, termVector);
}
}

54
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
View File

@ -31,15 +31,19 @@ import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
/**
* @author kimchy (shay.banon)
*/
public class UidFieldMapper extends AbstractFieldMapper<Uid> implements InternalMapper {
public class UidFieldMapper extends AbstractFieldMapper<Uid> implements InternalMapper, RootMapper {
public static final String NAME = "_uid";
@ -68,6 +72,12 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
return uid();
}
}
private ThreadLocal<UidField> fieldCache = new ThreadLocal<UidField>() {
@Override protected UidField initialValue() {
return new UidField(names().indexName(), "", 0);
@ -87,10 +97,48 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER);
}
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (context.id() == null) {
@Override public void preParse(ParseContext context) throws IOException {
// if we have the id provided, fill it, and parse now
if (context.sourceToParse().id() != null) {
context.id(context.sourceToParse().id());
super.parse(context);
}
}
@Override public void postParse(ParseContext context) throws IOException {
if (context.id() == null && !context.sourceToParse().flyweight()) {
throw new MapperParsingException("No id found while parsing the content source");
}
// if we did not have the id as part of the sourceToParse, then we need to parse it here
// it would have been filled in the _id parse phase
if (context.sourceToParse().id() == null) {
super.parse(context);
// since we did not have the uid in the pre phase, we did not add it automatically to the nested docs
// as they were created we need to make sure we add it to all the nested docs...
if (context.docs().size() > 1) {
UidField uidField = (UidField) context.rootDoc().getFieldable(UidFieldMapper.NAME);
assert uidField != null;
// we need to go over the docs and add it...
for (int i = 1; i < context.docs().size(); i++) {
// we don't need to add it as a full uid field in nested docs, since we don't need versioning
context.docs().get(i).add(new Field(UidFieldMapper.NAME, uidField.uid(), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
}
@Override public void parse(ParseContext context) throws IOException {
// nothing to do here, we either do it in post parse, or in pre parse.
}
@Override public void validate(ParseContext context) throws MapperParsingException {
}
@Override public boolean includeInObject() {
return false;
}
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
context.uid(Uid.createUid(context.stringBuilder(), context.type(), context.id()));
// so, caching uid stream and field is fine
// since we don't do any mapping parsing without immediate indexing

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java
View File

@ -389,7 +389,7 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
if (nested.isNested()) {
Document nestedDoc = new Document();
// pre add the uid field if possible (id was already provided)
Fieldable uidField = (Fieldable) context.doc().getFieldable(UidFieldMapper.NAME);
Fieldable uidField = context.doc().getFieldable(UidFieldMapper.NAME);
if (uidField != null) {
// we don't need to add it as a full uid field in nested docs, since we don't need versioning
// we also rely on this for UidField#loadVersion

94
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.id;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperTests;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.internal.IdFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
*/
@Test
public class IdMappingTests {
@Test public void simpleIdTests() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.endObject()
.copiedBytes());
assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
assertThat(doc.rootDoc().get(IdFieldMapper.NAME), nullValue());
try {
docMapper.parse("type", null, XContentFactory.jsonBuilder()
.startObject()
.endObject()
.copiedBytes());
assert false;
} catch (MapperParsingException e) {
}
doc = docMapper.parse("type", null, XContentFactory.jsonBuilder()
.startObject()
.field("_id", 1)
.endObject()
.copiedBytes());
assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
assertThat(doc.rootDoc().get(IdFieldMapper.NAME), nullValue());
}
@Test public void testIdIndexed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_id").field("index", "not_analyzed").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.endObject()
.copiedBytes());
assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
assertThat(doc.rootDoc().get(IdFieldMapper.NAME), notNullValue());
doc = docMapper.parse("type", null, XContentFactory.jsonBuilder()
.startObject()
.field("_id", 1)
.endObject()
.copiedBytes());
assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
assertThat(doc.rootDoc().get(IdFieldMapper.NAME), notNullValue());
}
}