honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

allow to parse directly the compressed mapping

This commit is contained in:
Shay Banon 2013-11-26 09:48:33 +01:00
parent 0610fc3ad2
commit 6f90a3e39a
10 changed files with 79 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -294,7 +294,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
// first, add the default mapping
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
try {
mapperService.merge(MapperService.DEFAULT_MAPPING, XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string(), false);
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), false);
} catch (Exception e) {
failureReason = "failed on parsing default mapping on index creation";
throw new MapperParsingException("mapping [" + MapperService.DEFAULT_MAPPING + "]", e);
@ -306,7 +306,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
}
try {
// apply the default here, its the first time we parse it
mapperService.merge(entry.getKey(), XContentFactory.jsonBuilder().map(entry.getValue()).string(), true);
mapperService.merge(entry.getKey(), new CompressedString(XContentFactory.jsonBuilder().map(entry.getValue()).string()), true);
} catch (Exception e) {
failureReason = "failed on parsing mappings on index creation";
throw new MapperParsingException("mapping [" + entry.getKey() + "]", e);

4
src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
View File

@ -140,11 +140,11 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
try {
indexService = indicesService.createIndex(indexMetaData.index(), indexMetaData.settings(), clusterService.localNode().id());
if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) {
indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source().string(), false);
indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false);
}
for (ObjectCursor<MappingMetaData> cursor : indexMetaData.mappings().values()) {
MappingMetaData mappingMetaData = cursor.value;
indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source().string(), false);
indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source(), false);
}
} catch (Exception e) {
logger.warn("[{}] failed to temporary create in order to apply alias action", e, indexMetaData.index());

16
src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
View File

@ -172,7 +172,7 @@ public class MetaDataMappingService extends AbstractComponent {
// only add the current relevant mapping (if exists)
if (indexMetaData.mappings().containsKey(type)) {
// don't apply the default mapping, it has been applied when the mapping was created
indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source().string(), false);
indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source(), false);
}
}
}
@ -215,11 +215,11 @@ public class MetaDataMappingService extends AbstractComponent {
removeIndex = true;
// only add the current relevant mapping (if exists)
if (indexMetaData.mappings().containsKey(type)) {
indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source().string(), false);
indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source(), false);
}
}
DocumentMapper updatedMapper = indexService.mapperService().merge(type, mappingSource.string(), false);
DocumentMapper updatedMapper = indexService.mapperService().merge(type, mappingSource, false);
processedRefreshes.add(type);
// if we end up with the same mapping as the original once, ignore
@ -401,11 +401,11 @@ public class MetaDataMappingService extends AbstractComponent {
indicesToClose.add(indexMetaData.index());
// make sure to add custom default mapping if exists
if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) {
indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source().string(), false);
indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false);
}
// only add the current relevant mapping (if exists)
if (indexMetaData.mappings().containsKey(request.mappingType)) {
indexService.mapperService().merge(request.mappingType, indexMetaData.mappings().get(request.mappingType).source().string(), false);
indexService.mapperService().merge(request.mappingType, indexMetaData.mappings().get(request.mappingType).source(), false);
}
}
@ -419,9 +419,9 @@ public class MetaDataMappingService extends AbstractComponent {
DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.mappingType);
if (MapperService.DEFAULT_MAPPING.equals(request.mappingType)) {
// _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default
newMapper = indexService.mapperService().parse(request.mappingType, request.mappingSource, false);
newMapper = indexService.mapperService().parse(request.mappingType, new CompressedString(request.mappingSource), false);
} else {
newMapper = indexService.mapperService().parse(request.mappingType, request.mappingSource);
newMapper = indexService.mapperService().parse(request.mappingType, new CompressedString(request.mappingSource));
if (existingMapper != null) {
// first, simulate
DocumentMapper.MergeResult mergeResult = existingMapper.merge(newMapper, mergeFlags().simulate(true));
@ -462,7 +462,7 @@ public class MetaDataMappingService extends AbstractComponent {
if (existingMappers.containsKey(entry.getKey())) {
existingSource = existingMappers.get(entry.getKey()).mappingSource();
}
DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource().string(), false);
DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false);
CompressedString updatedSource = mergedMapper.mappingSource();
if (existingSource != null) {

30
src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.geo.ShapesAvailability;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
@ -163,7 +164,30 @@ public class DocumentMapperParser extends AbstractIndexComponent {
if (mapping == null) {
mapping = Maps.newHashMap();
}
return parse(type, mapping, defaultSource);
}
public DocumentMapper parseCompressed(@Nullable String type, CompressedString source) throws MapperParsingException {
return parseCompressed(type, source, null);
}
@SuppressWarnings({"unchecked"})
public DocumentMapper parseCompressed(@Nullable String type, CompressedString source, String defaultSource) throws MapperParsingException {
Map<String, Object> mapping = null;
if (source != null) {
Map<String, Object> root = XContentHelper.convertToMap(source.compressed(), true).v2();
Tuple<String, Map<String, Object>> t = extractMapping(type, root);
type = t.v1();
mapping = t.v2();
}
if (mapping == null) {
mapping = Maps.newHashMap();
}
return parse(type, mapping, defaultSource);
}
@SuppressWarnings({"unchecked"})
private DocumentMapper parse(String type, Map<String, Object> mapping, String defaultSource) throws MapperParsingException {
if (type == null) {
throw new MapperParsingException("Failed to derive type");
}
@ -246,6 +270,11 @@ public class DocumentMapperParser extends AbstractIndexComponent {
} catch (Exception e) {
throw new MapperParsingException("failed to parse mapping definition", e);
}
return extractMapping(type, root);
}
@SuppressWarnings({"unchecked"})
private Tuple<String, Map<String, Object>> extractMapping(String type, Map<String, Object> root) throws MapperParsingException {
int size = root.size();
switch (size) {
case 0:
@ -256,7 +285,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
default:
// we always assume the first and single key is the mapping type root
throw new MapperParsingException("mapping must have the `type` as the root object");
}
String rootName = root.keySet().iterator().next();

18
src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -30,8 +30,10 @@ import org.apache.lucene.queries.TermsFilter;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchGenerationException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.lucene.search.AndFilter;
@ -224,16 +226,20 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
typeListeners.remove(listener);
}
public DocumentMapper merge(String type, String mappingSource, boolean applyDefault) {
public DocumentMapper merge(String type, CompressedString mappingSource, boolean applyDefault) {
if (DEFAULT_MAPPING.equals(type)) {
// verify we can parse it
DocumentMapper mapper = documentParser.parse(type, mappingSource);
DocumentMapper mapper = documentParser.parseCompressed(type, mappingSource);
// still add it as a document mapper so we have it registered and, for example, persisted back into
// the cluster meta data if needed, or checked for existence
synchronized (typeMutex) {
mappers = newMapBuilder(mappers).put(type, mapper).map();
}
defaultMappingSource = mappingSource;
try {
defaultMappingSource = mappingSource.string();
} catch (IOException e) {
throw new ElasticSearchGenerationException("failed to un-compress", e);
}
return mapper;
} else {
return merge(parse(type, mappingSource, applyDefault));
@ -358,18 +364,18 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
/**
* Just parses and returns the mapper without adding it, while still applying default mapping.
*/
public DocumentMapper parse(String mappingType, String mappingSource) throws MapperParsingException {
public DocumentMapper parse(String mappingType, CompressedString mappingSource) throws MapperParsingException {
return parse(mappingType, mappingSource, true);
}
public DocumentMapper parse(String mappingType, String mappingSource, boolean applyDefault) throws MapperParsingException {
public DocumentMapper parse(String mappingType, CompressedString mappingSource, boolean applyDefault) throws MapperParsingException {
String defaultMappingSource;
if (PercolatorService.TYPE_NAME.equals(mappingType)) {
defaultMappingSource = this.defaultPercolatorMappingSource;
} else {
defaultMappingSource = this.defaultMappingSource;
}
return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
return documentParser.parseCompressed(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
}
public boolean hasMapping(String mappingType) {

14
src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -28,7 +28,10 @@ import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.action.index.*;
import org.elasticsearch.cluster.action.index.NodeIndexCreatedAction;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
import org.elasticsearch.cluster.action.index.NodeMappingCreatedAction;
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -65,7 +68,10 @@ import org.elasticsearch.indices.recovery.RecoveryTarget;
import org.elasticsearch.indices.recovery.StartRecoveryRequest;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import static com.google.common.collect.Maps.newHashMap;
@ -398,7 +404,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
logger.debug("[{}] adding mapping [{}], source [{}]", index, mappingType, mappingSource.string());
}
// we don't apply default, since it has been applied when the mappings were parsed initially
mapperService.merge(mappingType, mappingSource.string(), false);
mapperService.merge(mappingType, mappingSource, false);
if (!mapperService.documentMapper(mappingType).mappingSource().equals(mappingSource)) {
// this might happen when upgrading from 0.15 to 0.16
logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index, mappingType, mappingSource, mapperService.documentMapper(mappingType).mappingSource());
@ -414,7 +420,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
logger.debug("[{}] updating mapping [{}], source [{}]", index, mappingType, mappingSource.string());
}
// we don't apply default, since it has been applied when the mappings were parsed initially
mapperService.merge(mappingType, mappingSource.string(), false);
mapperService.merge(mappingType, mappingSource, false);
if (!mapperService.documentMapper(mappingType).mappingSource().equals(mappingSource)) {
requiresRefresh = true;
// this might happen when upgrading from 0.15 to 0.16

3
src/test/java/org/elasticsearch/index/cache/id/SimpleIdCacheTests.java vendored
View File

@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.ImmutableSettings;
@ -266,7 +267,7 @@ public class SimpleIdCacheTests extends ElasticsearchTestCase {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(documentType.v1())
.startObject("_parent").field("type", documentType.v2()).endObject()
.endObject().endObject().string();
mapperService.merge(documentType.v1(), defaultMapping, true);
mapperService.merge(documentType.v1(), new CompressedString(defaultMapping), true);
}
idCache.setIndexService(new StubIndexService(mapperService));

7
src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.source;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
@ -173,7 +174,7 @@ public class DefaultSourceMappingTests extends ElasticsearchTestCase {
.endObject().endObject().string();
MapperService mapperService = MapperTestUtils.newMapperService();
mapperService.merge(MapperService.DEFAULT_MAPPING, defaultMapping, true);
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type");
assertThat(mapper.type(), equalTo("my_type"));
@ -187,12 +188,12 @@ public class DefaultSourceMappingTests extends ElasticsearchTestCase {
.endObject().endObject().string();
MapperService mapperService = MapperTestUtils.newMapperService();
mapperService.merge(MapperService.DEFAULT_MAPPING, defaultMapping, true);
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
.startObject("_source").field("enabled", true).endObject()
.endObject().endObject().string();
mapperService.merge("my_type", mapping, true);
mapperService.merge("my_type", new CompressedString(mapping), true);
DocumentMapper mapper = mapperService.documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));

7
src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.cache.recycler.CacheRecyclerModule;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
@ -122,7 +123,7 @@ public class SimpleIndexQueryParserTests extends ElasticsearchTestCase {
).createInjector();
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/query/mapping.json");
injector.getInstance(MapperService.class).merge("person", mapping, true);
injector.getInstance(MapperService.class).merge("person", new CompressedString(mapping), true);
injector.getInstance(MapperService.class).documentMapper("person").parse(new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/query/data.json")));
queryParser = injector.getInstance(IndexQueryParserService.class);
}
@ -1490,7 +1491,7 @@ public class SimpleIndexQueryParserTests extends ElasticsearchTestCase {
assertThat(((TermQuery) functionScoreQuery.getSubQuery()).getTerm(), equalTo(new Term("name.last", "banon")));
assertThat((double) ((BoostScoreFunction) functionScoreQuery.getFunction()).getBoost(), closeTo(1.3, 0.001));
}
@Test
public void testCustomBoostFactorQueryBuilder_withFunctionScoreWithoutQueryGiven() throws IOException {
IndexQueryParserService queryParser = queryParser();
@ -1498,7 +1499,7 @@ public class SimpleIndexQueryParserTests extends ElasticsearchTestCase {
assertThat(parsedQuery, instanceOf(FunctionScoreQuery.class));
FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) parsedQuery;
assertThat(functionScoreQuery.getSubQuery() instanceof XConstantScoreQuery, equalTo(true));
assertThat(((XConstantScoreQuery)functionScoreQuery.getSubQuery()).getFilter() instanceof MatchAllDocsFilter, equalTo(true));
assertThat(((XConstantScoreQuery) functionScoreQuery.getSubQuery()).getFilter() instanceof MatchAllDocsFilter, equalTo(true));
assertThat((double) ((BoostScoreFunction) functionScoreQuery.getFunction()).getBoost(), closeTo(1.3, 0.001));
}

11
src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.env.Environment;
@ -198,9 +199,9 @@ public class ChildrenConstantScoreQueryTests extends ElasticsearchLuceneTestCase
} else {
// Usage in HasChildFilterParser
query = new XConstantScoreQuery(
new CustomQueryWrappingFilter(
new ChildrenConstantScoreQuery(childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, applyAcceptedDocs)
)
new CustomQueryWrappingFilter(
new ChildrenConstantScoreQuery(childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, applyAcceptedDocs)
)
);
}
BitSetCollector collector = new BitSetCollector(indexReader.maxDoc());
@ -236,7 +237,7 @@ public class ChildrenConstantScoreQueryTests extends ElasticsearchLuceneTestCase
static void assertBitSet(FixedBitSet actual, FixedBitSet expected, IndexSearcher searcher) throws IOException {
if (!actual.equals(expected)) {
Description description= new StringDescription();
Description description = new StringDescription();
description.appendText(reason(actual, expected, searcher));
description.appendText("\nExpected: ");
description.appendValue(expected);
@ -270,7 +271,7 @@ public class ChildrenConstantScoreQueryTests extends ElasticsearchLuceneTestCase
index, ImmutableSettings.EMPTY, new Environment(), new AnalysisService(index), null, null, null
);
mapperService.merge(
childType, PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string(), true
childType, new CompressedString(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), true
);
final IndexService indexService = new SimpleIdCacheTests.StubIndexService(mapperService);
idCache.setIndexService(indexService);