honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 18:35:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

second phase at optimizing merging/parsing large new mappings

Browse Source 
apply the new mappings only after the parsing/merging of a full doc/mapping is done
This commit is contained in:
Shay Banon 2012-11-19 17:40:13 +01:00
parent 303752d78a
commit 4e8a9008b7
8 changed files with 62 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -310,7 +310,7 @@ public class DocumentMapper implements ToXContent {
rootObjectMapper.putMapper(rootMapper);
} else {
if (rootMapper instanceof FieldMapper) {
fieldMappersAgg.fieldMappers.add((FieldMapper) rootMapper);
fieldMappersAgg.mappers.add((FieldMapper) rootMapper);
}
}
}
@ -318,7 +318,7 @@ public class DocumentMapper implements ToXContent {
// now traverse and get all the statically defined ones
rootObjectMapper.traverse(fieldMappersAgg);
this.fieldMappers = new DocumentFieldMappers(this, fieldMappersAgg.fieldMappers);
this.fieldMappers = new DocumentFieldMappers(this, fieldMappersAgg.mappers);
final Map<String, ObjectMapper> objectMappers = Maps.newHashMap();
rootObjectMapper.traverse(new ObjectMapperListener() {
@ -529,6 +529,14 @@ public class DocumentMapper implements ToXContent {
}
}
// fire up any new mappers if exists
if (!context.newFieldMappers().mappers.isEmpty()) {
addFieldMappers(context.newFieldMappers().mappers);
}
if (!context.newObjectMappers().mappers.isEmpty()) {
addObjectMappers(context.newObjectMappers().mappers);
}
ParsedDocument doc = new ParsedDocument(context.uid(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.analyzer(),
context.source(), context.mappingsModified()).parent(source.parent());
// reset the context to free up memory
@ -536,11 +544,11 @@ public class DocumentMapper implements ToXContent {
return doc;
}
public void addFieldMappers(Collection<FieldMapper> fieldMappers) {
private void addFieldMappers(Collection<FieldMapper> fieldMappers) {
addFieldMappers(fieldMappers.toArray(new FieldMapper[fieldMappers.size()]));
}
public void addFieldMappers(FieldMapper... fieldMappers) {
private void addFieldMappers(FieldMapper... fieldMappers) {
synchronized (mutex) {
this.fieldMappers = this.fieldMappers.concat(this, fieldMappers);
}
@ -565,11 +573,11 @@ public class DocumentMapper implements ToXContent {
rootObjectMapper.traverse(listener);
}
public void addObjectMappers(Collection<ObjectMapper> objectMappers) {
private void addObjectMappers(Collection<ObjectMapper> objectMappers) {
addObjectMappers(objectMappers.toArray(new ObjectMapper[objectMappers.size()]));
}
public void addObjectMappers(ObjectMapper... objectMappers) {
private void addObjectMappers(ObjectMapper... objectMappers) {
synchronized (mutex) {
MapBuilder<String, ObjectMapper> builder = MapBuilder.newMapBuilder(this.objectMappers);
for (ObjectMapper objectMapper : objectMappers) {
@ -612,6 +620,12 @@ public class DocumentMapper implements ToXContent {
}
if (!mergeFlags.simulate()) {
if (!mergeContext.newFieldMappers().mappers.isEmpty()) {
addFieldMappers(mergeContext.newFieldMappers().mappers);
}
if (!mergeContext.newObjectMappers().mappers.isEmpty()) {
addObjectMappers(mergeContext.newObjectMappers().mappers);
}
// let the merge with attributes to override the attributes
meta = mergeWith.meta();
// update the source of the merged one

4
src/main/java/org/elasticsearch/index/mapper/FieldMapperListener.java
View File

@ -28,11 +28,11 @@ import java.util.List;
public abstract class FieldMapperListener {
public static class Aggregator extends FieldMapperListener {
public final List<FieldMapper> fieldMappers = new ArrayList<FieldMapper>();
public final List<FieldMapper> mappers = new ArrayList<FieldMapper>();
@Override
public void fieldMapper(FieldMapper fieldMapper) {
fieldMappers.add(fieldMapper);
mappers.add(fieldMapper);
}
}

4
src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -214,12 +214,12 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
mapper.traverse(fieldMappersAgg);
addFieldMappers(fieldMappersAgg.fieldMappers.toArray(new FieldMapper[fieldMappersAgg.fieldMappers.size()]));
addFieldMappers(fieldMappersAgg.mappers.toArray(new FieldMapper[fieldMappersAgg.mappers.size()]));
mapper.addFieldMapperListener(fieldMapperListener, false);
ObjectMapperListener.Aggregator objectMappersAgg = new ObjectMapperListener.Aggregator();
mapper.traverse(objectMappersAgg);
addObjectMappers(objectMappersAgg.objectMappers.toArray(new ObjectMapper[objectMappersAgg.objectMappers.size()]));
addObjectMappers(objectMappersAgg.mappers.toArray(new ObjectMapper[objectMappersAgg.mappers.size()]));
mapper.addObjectMapperListener(objectMapperListener, false);
mappers = newMapBuilder(mappers).put(mapper.type(), mapper).map();

13
src/main/java/org/elasticsearch/index/mapper/MergeContext.java
View File

@ -29,11 +29,12 @@ import java.util.List;
public class MergeContext {
private final DocumentMapper documentMapper;
private final DocumentMapper.MergeFlags mergeFlags;
private final List<String> mergeConflicts = Lists.newArrayList();
private final FieldMapperListener.Aggregator newFieldMappers = new FieldMapperListener.Aggregator();
private final ObjectMapperListener.Aggregator newObjectMappers = new ObjectMapperListener.Aggregator();
public MergeContext(DocumentMapper documentMapper, DocumentMapper.MergeFlags mergeFlags) {
this.documentMapper = documentMapper;
this.mergeFlags = mergeFlags;
@ -47,6 +48,14 @@ public class MergeContext {
return mergeFlags;
}
public FieldMapperListener.Aggregator newFieldMappers() {
return newFieldMappers;
}
public ObjectMapperListener.Aggregator newObjectMappers() {
return newObjectMappers;
}
public void addConflict(String mergeFailure) {
mergeConflicts.add(mergeFailure);
}

4
src/main/java/org/elasticsearch/index/mapper/ObjectMapperListener.java
View File

@ -30,11 +30,11 @@ import java.util.List;
public abstract class ObjectMapperListener {
public static class Aggregator extends ObjectMapperListener {
public final List<ObjectMapper> objectMappers = new ArrayList<ObjectMapper>();
public final List<ObjectMapper> mappers = new ArrayList<ObjectMapper>();
@Override
public void objectMapper(ObjectMapper objectMapper) {
objectMappers.add(objectMapper);
mappers.add(objectMapper);
}
}

13
src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -82,6 +82,9 @@ public class ParseContext {
private float docBoost = 1.0f;
private FieldMapperListener.Aggregator newFieldMappers = new FieldMapperListener.Aggregator();
private ObjectMapperListener.Aggregator newObjectMappers = new ObjectMapperListener.Aggregator();
public ParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
this.index = index;
this.indexSettings = indexSettings;
@ -110,12 +113,22 @@ public class ParseContext {
this.allEntries = new AllEntries();
this.ignoredValues.clear();
this.docBoost = 1.0f;
this.newFieldMappers.mappers.clear();
this.newObjectMappers.mappers.clear();
}
public boolean flyweight() {
return sourceToParse.flyweight();
}
public FieldMapperListener.Aggregator newFieldMappers() {
return newFieldMappers;
}
public ObjectMapperListener.Aggregator newObjectMappers() {
return newObjectMappers;
}
public DocumentMapperParser docMapperParser() {
return this.docMapperParser;
}

14
src/main/java/org/elasticsearch/index/mapper/multifield/MultiFieldMapper.java
View File

@ -27,7 +27,10 @@ import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import java.io.IOException;
import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
@ -218,14 +221,13 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
mergeContext.addConflict("Can't merge a non multi_field / non simple mapping [" + mergeWith.name() + "] with a multi_field mapping [" + name() + "]");
return;
}
List<FieldMapper> mappersToAddToDocMapper = new ArrayList<FieldMapper>();
synchronized (mutex) {
if (mergeWith instanceof AbstractFieldMapper) {
// its a single field mapper, upgraded into a multi field mapper, just update the default mapper
if (defaultMapper == null) {
if (!mergeContext.mergeFlags().simulate()) {
defaultMapper = mergeWith;
mappersToAddToDocMapper.add((FieldMapper) defaultMapper);
mergeContext.newFieldMappers().mappers.add((FieldMapper) defaultMapper);
}
}
} else {
@ -235,7 +237,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
if (mergeWithMultiField.defaultMapper != null) {
if (!mergeContext.mergeFlags().simulate()) {
defaultMapper = mergeWithMultiField.defaultMapper;
mappersToAddToDocMapper.add((FieldMapper) defaultMapper);
mergeContext.newFieldMappers().mappers.add((FieldMapper) defaultMapper);
}
}
} else {
@ -256,7 +258,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
}
mappers = newMapBuilder(mappers).put(mergeWithMapper.name(), mergeWithMapper).immutableMap();
if (mergeWithMapper instanceof AbstractFieldMapper) {
mappersToAddToDocMapper.add((FieldMapper) mergeWithMapper);
mergeContext.newFieldMappers().mappers.add((FieldMapper) mergeWithMapper);
}
}
} else {
@ -265,8 +267,6 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
}
}
}
// call it outside of the mutex
mergeContext.docMapper().addFieldMappers(mappersToAddToDocMapper);
}
@Override

21
src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java
View File

@ -539,13 +539,8 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
if (newMapper) {
// we need to traverse in case we have a dynamic template and need to add field mappers
// introduced by it
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
objectMapper.traverse(fieldMappersAgg);
context.docMapper().addFieldMappers(fieldMappersAgg.fieldMappers);
ObjectMapperListener.Aggregator objectMappersAgg = new ObjectMapperListener.Aggregator();
objectMapper.traverse(objectMappersAgg);
context.docMapper().addObjectMappers(objectMappersAgg.objectMappers);
objectMapper.traverse(context.newFieldMappers());
objectMapper.traverse(context.newObjectMappers());
}
// now, parse it
objectMapper.parse(context);
@ -769,9 +764,7 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
}
}
if (newMapper) {
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
mapper.traverse(fieldMappersAgg);
context.docMapper().addFieldMappers(fieldMappersAgg.fieldMappers);
mapper.traverse(context.newFieldMappers());
}
mapper.parse(context);
}
@ -814,14 +807,10 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
}
}
// call this outside of the mutex
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
ObjectMapperListener.Aggregator objectMappersAgg = new ObjectMapperListener.Aggregator();
for (Mapper mapper : mappersToTraverse) {
mapper.traverse(fieldMappersAgg);
mapper.traverse(objectMappersAgg);
mapper.traverse(mergeContext.newFieldMappers());
mapper.traverse(mergeContext.newObjectMappers());
}
mergeContext.docMapper().addFieldMappers(fieldMappersAgg.fieldMappers);
mergeContext.docMapper().addObjectMappers(objectMappersAgg.objectMappers);
}
protected void doMerge(ObjectMapper mergeWith, MergeContext mergeContext) {