From 6f90a3e39ae59fcb91334a99e3881198d6b53e65 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 26 Nov 2013 09:48:33 +0100 Subject: [PATCH] allow to parse directly the compressed mapping --- .../metadata/MetaDataCreateIndexService.java | 4 +-- .../metadata/MetaDataIndexAliasesService.java | 4 +-- .../metadata/MetaDataMappingService.java | 16 +++++----- .../index/mapper/DocumentMapperParser.java | 30 ++++++++++++++++++- .../index/mapper/MapperService.java | 18 +++++++---- .../cluster/IndicesClusterStateService.java | 14 ++++++--- .../index/cache/id/SimpleIdCacheTests.java | 3 +- .../source/DefaultSourceMappingTests.java | 7 +++-- .../query/SimpleIndexQueryParserTests.java | 7 +++-- .../ChildrenConstantScoreQueryTests.java | 11 +++---- 10 files changed, 79 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 65acb4c696b..a7f1d841bc3 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -294,7 +294,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { // first, add the default mapping if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) { try { - mapperService.merge(MapperService.DEFAULT_MAPPING, XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string(), false); + mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), false); } catch (Exception e) { failureReason = "failed on parsing default mapping on index creation"; throw new MapperParsingException("mapping [" + MapperService.DEFAULT_MAPPING + "]", e); @@ -306,7 +306,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { } try { // apply the default here, its the first time we parse it - mapperService.merge(entry.getKey(), XContentFactory.jsonBuilder().map(entry.getValue()).string(), true); + mapperService.merge(entry.getKey(), new CompressedString(XContentFactory.jsonBuilder().map(entry.getValue()).string()), true); } catch (Exception e) { failureReason = "failed on parsing mappings on index creation"; throw new MapperParsingException("mapping [" + entry.getKey() + "]", e); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java index 99b39c0b3dc..c2d09237edc 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java @@ -140,11 +140,11 @@ public class MetaDataIndexAliasesService extends AbstractComponent { try { indexService = indicesService.createIndex(indexMetaData.index(), indexMetaData.settings(), clusterService.localNode().id()); if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) { - indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source().string(), false); + indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false); } for (ObjectCursor cursor : indexMetaData.mappings().values()) { MappingMetaData mappingMetaData = cursor.value; - indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source().string(), false); + indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source(), false); } } catch (Exception e) { logger.warn("[{}] failed to temporary create in order to apply alias action", e, indexMetaData.index()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 6a1ee702856..412fd86a8cf 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -172,7 +172,7 @@ public class MetaDataMappingService extends AbstractComponent { // only add the current relevant mapping (if exists) if (indexMetaData.mappings().containsKey(type)) { // don't apply the default mapping, it has been applied when the mapping was created - indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source().string(), false); + indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source(), false); } } } @@ -215,11 +215,11 @@ public class MetaDataMappingService extends AbstractComponent { removeIndex = true; // only add the current relevant mapping (if exists) if (indexMetaData.mappings().containsKey(type)) { - indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source().string(), false); + indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source(), false); } } - DocumentMapper updatedMapper = indexService.mapperService().merge(type, mappingSource.string(), false); + DocumentMapper updatedMapper = indexService.mapperService().merge(type, mappingSource, false); processedRefreshes.add(type); // if we end up with the same mapping as the original once, ignore @@ -401,11 +401,11 @@ public class MetaDataMappingService extends AbstractComponent { indicesToClose.add(indexMetaData.index()); // make sure to add custom default mapping if exists if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) { - indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source().string(), false); + indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false); } // only add the current relevant mapping (if exists) if (indexMetaData.mappings().containsKey(request.mappingType)) { - indexService.mapperService().merge(request.mappingType, indexMetaData.mappings().get(request.mappingType).source().string(), false); + indexService.mapperService().merge(request.mappingType, indexMetaData.mappings().get(request.mappingType).source(), false); } } @@ -419,9 +419,9 @@ public class MetaDataMappingService extends AbstractComponent { DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.mappingType); if (MapperService.DEFAULT_MAPPING.equals(request.mappingType)) { // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default - newMapper = indexService.mapperService().parse(request.mappingType, request.mappingSource, false); + newMapper = indexService.mapperService().parse(request.mappingType, new CompressedString(request.mappingSource), false); } else { - newMapper = indexService.mapperService().parse(request.mappingType, request.mappingSource); + newMapper = indexService.mapperService().parse(request.mappingType, new CompressedString(request.mappingSource)); if (existingMapper != null) { // first, simulate DocumentMapper.MergeResult mergeResult = existingMapper.merge(newMapper, mergeFlags().simulate(true)); @@ -462,7 +462,7 @@ public class MetaDataMappingService extends AbstractComponent { if (existingMappers.containsKey(entry.getKey())) { existingSource = existingMappers.get(entry.getKey()).mappingSource(); } - DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource().string(), false); + DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false); CompressedString updatedSource = mergedMapper.mappingSource(); if (existingSource != null) { diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 920e8d3567c..6ddbea639b2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; @@ -163,7 +164,30 @@ public class DocumentMapperParser extends AbstractIndexComponent { if (mapping == null) { mapping = Maps.newHashMap(); } + return parse(type, mapping, defaultSource); + } + public DocumentMapper parseCompressed(@Nullable String type, CompressedString source) throws MapperParsingException { + return parseCompressed(type, source, null); + } + + @SuppressWarnings({"unchecked"}) + public DocumentMapper parseCompressed(@Nullable String type, CompressedString source, String defaultSource) throws MapperParsingException { + Map mapping = null; + if (source != null) { + Map root = XContentHelper.convertToMap(source.compressed(), true).v2(); + Tuple> t = extractMapping(type, root); + type = t.v1(); + mapping = t.v2(); + } + if (mapping == null) { + mapping = Maps.newHashMap(); + } + return parse(type, mapping, defaultSource); + } + + @SuppressWarnings({"unchecked"}) + private DocumentMapper parse(String type, Map mapping, String defaultSource) throws MapperParsingException { if (type == null) { throw new MapperParsingException("Failed to derive type"); } @@ -246,6 +270,11 @@ public class DocumentMapperParser extends AbstractIndexComponent { } catch (Exception e) { throw new MapperParsingException("failed to parse mapping definition", e); } + return extractMapping(type, root); + } + + @SuppressWarnings({"unchecked"}) + private Tuple> extractMapping(String type, Map root) throws MapperParsingException { int size = root.size(); switch (size) { case 0: @@ -256,7 +285,6 @@ public class DocumentMapperParser extends AbstractIndexComponent { default: // we always assume the first and single key is the mapping type root throw new MapperParsingException("mapping must have the `type` as the root object"); - } String rootName = root.keySet().iterator().next(); diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index e10b72d8ffb..5252d85bea2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -30,8 +30,10 @@ import org.apache.lucene.queries.TermsFilter; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Filter; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.search.AndFilter; @@ -224,16 +226,20 @@ public class MapperService extends AbstractIndexComponent implements Iterable