honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MapperService to wrap a single DocumentMapper. (#29511) 

This refactors MapperService so that it wraps a single `DocumentMapper` rather
than a `Map<String, DocumentMapper>`. We will need follow-ups since I haven't
fixed most APIs that still expose collections of types of mappers, but this is
a start...
This commit is contained in:
Adrien Grand 2018-04-17 17:11:27 +02:00 committed by GitHub
parent a8c2cc6ce7
commit d7be9185c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 86 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -19,11 +19,9 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
@ -57,7 +55,6 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -78,12 +75,11 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@ -475,9 +471,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
// now, update the mappings with the actual source
Map<String, MappingMetaData> mappingsMetaData = new HashMap<>();
for (DocumentMapper mapper : mapperService.docMappers(true)) {
MappingMetaData mappingMd = new MappingMetaData(mapper);
mappingsMetaData.put(mapper.type(), mappingMd);
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(), mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
MappingMetaData mappingMd = new MappingMetaData(mapper);
mappingsMetaData.put(mapper.type(), mappingMd);
}
}
final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(request.index())

26
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
View File

@ -19,9 +19,7 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest;
@ -49,6 +47,7 @@ import org.elasticsearch.indices.InvalidTypeNameException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -175,10 +174,13 @@ public class MetaDataMappingService extends AbstractComponent {
String index = indexService.index().getName();
try {
List<String> updatedTypes = new ArrayList<>();
for (DocumentMapper mapper : indexService.mapperService().docMappers(true)) {
final String type = mapper.type();
if (!mapper.mappingSource().equals(builder.mapping(type).source())) {
updatedTypes.add(type);
MapperService mapperService = indexService.mapperService();
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(), mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
final String type = mapper.type();
if (!mapper.mappingSource().equals(builder.mapping(type).source())) {
updatedTypes.add(type);
}
}
}
@ -186,8 +188,10 @@ public class MetaDataMappingService extends AbstractComponent {
if (updatedTypes.isEmpty() == false) {
logger.warn("[{}] re-syncing mappings with cluster state because of types [{}]", index, updatedTypes);
dirty = true;
for (DocumentMapper mapper : indexService.mapperService().docMappers(true)) {
builder.putMapping(new MappingMetaData(mapper));
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(), mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
builder.putMapping(new MappingMetaData(mapper));
}
}
}
} catch (Exception e) {
@ -320,8 +324,10 @@ public class MetaDataMappingService extends AbstractComponent {
IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData);
// Mapping updates on a single type may have side-effects on other types so we need to
// update mapping metadata on all types
for (DocumentMapper mapper : mapperService.docMappers(true)) {
indexMetaDataBuilder.putMapping(new MappingMetaData(mapper.mappingSource()));
for (DocumentMapper mapper : Arrays.asList(mapperService.documentMapper(), mapperService.documentMapper(MapperService.DEFAULT_MAPPING))) {
if (mapper != null) {
indexMetaDataBuilder.putMapping(new MappingMetaData(mapper.mappingSource()));
}
}
builder.put(indexMetaDataBuilder);
}

3
server/src/main/java/org/elasticsearch/index/IndexWarmer.java
View File

@ -121,7 +121,8 @@ public final class IndexWarmer extends AbstractComponent {
public TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
DocumentMapper docMapper = mapperService.documentMapper();
if (docMapper != null) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final MappedFieldType fieldType = fieldMapper.fieldType();
final String indexName = fieldType.name();

3
server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java vendored
View File

@ -233,7 +233,8 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements I
boolean hasNested = false;
final Set<Query> warmUp = new HashSet<>();
final MapperService mapperService = indexShard.mapperService();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
DocumentMapper docMapper = mapperService.documentMapper();
if (docMapper != null) {
if (docMapper.hasNestedObjects()) {
hasNested = true;
for (ObjectMapper objectMapper : docMapper.objectMappers().values()) {

121
server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -57,6 +57,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@ -64,13 +65,12 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableMap;
public class MapperService extends AbstractIndexComponent implements Closeable {
@ -121,7 +121,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
private volatile String defaultMappingSource;
private volatile Map<String, DocumentMapper> mappers = emptyMap();
private volatile DocumentMapper mapper;
private volatile DocumentMapper defaultMapper;
private volatile FieldTypeLookup fieldTypes;
private volatile Map<String, ObjectMapper> fullPathObjectMappers = emptyMap();
@ -166,24 +167,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
return this.hasNested;
}
/**
* returns an immutable iterator over current document mappers.
*
* @param includingDefaultMapping indicates whether the iterator should contain the {@link #DEFAULT_MAPPING} document mapper.
* As is this not really an active type, you would typically set this to false
*/
public Iterable<DocumentMapper> docMappers(final boolean includingDefaultMapping) {
return () -> {
final Collection<DocumentMapper> documentMappers;
if (includingDefaultMapping) {
documentMappers = mappers.values();
} else {
documentMappers = mappers.values().stream().filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type())).collect(Collectors.toList());
}
return Collections.unmodifiableCollection(documentMappers).iterator();
};
}
public IndexAnalyzers getIndexAnalyzers() {
return this.indexAnalyzers;
}
@ -212,7 +195,13 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
assert indexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index() + " but was " + indexMetaData.getIndex();
// go over and add the relevant mappings (or update them)
final Set<String> existingMappers = new HashSet<>(mappers.keySet());
Set<String> existingMappers = new HashSet<>();
if (mapper != null) {
existingMappers.add(mapper.type());
}
if (defaultMapper != null) {
existingMappers.add(DEFAULT_MAPPING);
}
final Map<String, DocumentMapper> updatedEntries;
try {
// only update entries if needed
@ -314,29 +303,32 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
defaultMappingSourceOrLastStored = this.defaultMappingSource;
}
List<DocumentMapper> documentMappers = new ArrayList<>();
DocumentMapper documentMapper = null;
for (Map.Entry<String, CompressedXContent> entry : mappings.entrySet()) {
String type = entry.getKey();
if (type.equals(DEFAULT_MAPPING)) {
continue;
}
if (documentMapper != null) {
throw new IllegalArgumentException("Cannot put multiple mappings: " + mappings.keySet());
}
final boolean applyDefault =
// the default was already applied if we are recovering
reason != MergeReason.MAPPING_RECOVERY
// only apply the default mapping if we don't have the type yet
&& mappers.containsKey(type) == false;
&& this.mapper == null;
try {
DocumentMapper documentMapper =
documentMapper =
documentParser.parse(type, entry.getValue(), applyDefault ? defaultMappingSourceOrLastStored : null);
documentMappers.add(documentMapper);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
}
return internalMerge(defaultMapper, defaultMappingSource, documentMappers, reason);
return internalMerge(defaultMapper, defaultMappingSource, documentMapper, reason);
}
static void validateTypeName(String type) {
@ -361,13 +353,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
private synchronized Map<String, DocumentMapper> internalMerge(@Nullable DocumentMapper defaultMapper, @Nullable String defaultMappingSource,
List<DocumentMapper> documentMappers, MergeReason reason) {
DocumentMapper mapper, MergeReason reason) {
boolean hasNested = this.hasNested;
Map<String, ObjectMapper> fullPathObjectMappers = this.fullPathObjectMappers;
FieldTypeLookup fieldTypes = this.fieldTypes;
Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
Map<String, DocumentMapper> results = new LinkedHashMap<>(documentMappers.size() + 1);
Map<String, DocumentMapper> results = new LinkedHashMap<>(2);
if (defaultMapper != null) {
if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) {
@ -378,27 +369,23 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
"cannot have more than one type");
}
assert defaultMapper.type().equals(DEFAULT_MAPPING);
mappers.put(DEFAULT_MAPPING, defaultMapper);
results.put(DEFAULT_MAPPING, defaultMapper);
}
{
Set<String> actualTypes = new HashSet<>(mappers.keySet());
documentMappers.forEach(mapper -> actualTypes.add(mapper.type()));
actualTypes.remove(DEFAULT_MAPPING);
if (actualTypes.size() > 1) {
if (mapper != null && this.mapper != null && Objects.equals(this.mapper.type(), mapper.type()) == false) {
throw new IllegalArgumentException(
"Rejecting mapping update to [" + index().getName() + "] as the final mapping would have more than 1 type: " + actualTypes);
"Rejecting mapping update to [" + index().getName() + "] as the final mapping would have more than 1 type: " + Arrays.asList(this.mapper.type(), mapper.type()));
}
}
for (DocumentMapper mapper : documentMappers) {
DocumentMapper newMapper = null;
if (mapper != null) {
// check naming
validateTypeName(mapper.type());
// compute the merged DocumentMapper
DocumentMapper oldMapper = mappers.get(mapper.type());
DocumentMapper newMapper;
DocumentMapper oldMapper = this.mapper;
if (oldMapper != null) {
newMapper = oldMapper.merge(mapper.mapping());
} else {
@ -442,7 +429,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
results.put(newMapper.type(), newMapper);
mappers.put(newMapper.type(), newMapper);
}
if (reason == MergeReason.MAPPING_UPDATE) {
@ -456,24 +442,16 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
checkIndexSortCompatibility(indexSettings.getIndexSortConfig(), hasNested);
for (Map.Entry<String, DocumentMapper> entry : mappers.entrySet()) {
if (entry.getKey().equals(DEFAULT_MAPPING)) {
continue;
}
DocumentMapper documentMapper = entry.getValue();
// apply changes to the field types back
DocumentMapper updatedDocumentMapper = documentMapper.updateFieldType(fieldTypes.fullNameToFieldType);
if (updatedDocumentMapper != documentMapper) {
if (newMapper != null) {
DocumentMapper updatedDocumentMapper = newMapper.updateFieldType(fieldTypes.fullNameToFieldType);
if (updatedDocumentMapper != newMapper) {
// update both mappers and result
entry.setValue(updatedDocumentMapper);
if (results.containsKey(updatedDocumentMapper.type())) {
results.put(updatedDocumentMapper.type(), updatedDocumentMapper);
}
newMapper = updatedDocumentMapper;
results.put(updatedDocumentMapper.type(), updatedDocumentMapper);
}
}
// make structures immutable
mappers = Collections.unmodifiableMap(mappers);
results = Collections.unmodifiableMap(results);
// only need to immutably rewrap these if the previous reference was changed.
@ -486,7 +464,10 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
if (defaultMappingSource != null) {
this.defaultMappingSource = defaultMappingSource;
}
this.mappers = mappers;
if (newMapper != null) {
this.mapper = newMapper;
}
this.defaultMapper = defaultMapper;
this.fieldTypes = fieldTypes;
this.hasNested = hasNested;
this.fullPathObjectMappers = fullPathObjectMappers;
@ -498,7 +479,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
private boolean assertMappersShareSameFieldType() {
for (DocumentMapper mapper : docMappers(false)) {
if (mapper != null) {
List<FieldMapper> fieldMappers = new ArrayList<>();
Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers);
@ -692,18 +673,20 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
}
public boolean hasMapping(String mappingType) {
return mappers.containsKey(mappingType);
/**
* Get the set of types.
* @deprecated Indices may have one type at most, use {@link #documentMapper()} instead.
*/
@Deprecated
public Set<String> types() {
return mapper == null ? Collections.emptySet() : Collections.singleton(mapper.type());
}
/**
* Return the set of concrete types that have a mapping.
* NOTE: this does not return the default mapping.
* Return the document mapper, or {@code null} if no mapping has been put yet.
*/
public Collection<String> types() {
final Set<String> types = new HashSet<>(mappers.keySet());
types.remove(DEFAULT_MAPPING);
return Collections.unmodifiableSet(types);
public DocumentMapper documentMapper() {
return mapper;
}
/**
@ -712,7 +695,13 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
* the default mapping.
*/
public DocumentMapper documentMapper(String type) {
return mappers.get(type);
if (mapper != null && type.equals(mapper.type())) {
return mapper;
}
if (DEFAULT_MAPPING.equals(type)) {
return defaultMapper;
}
return null;
}
/**
@ -720,7 +709,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
* type has been dynamically created.
*/
public DocumentMapperForType documentMapperWithAutoCreate(String type) {
DocumentMapper mapper = mappers.get(type);
DocumentMapper mapper = documentMapper(type);
if (mapper != null) {
return new DocumentMapperForType(mapper, null);
}
@ -836,7 +825,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
/** Return a term that uniquely identifies the document, or {@code null} if the type is not allowed. */
public Term createUidTerm(String type, String id) {
if (hasMapping(type) == false) {
if (mapper == null || mapper.type().equals(type) == false) {
return null;
}
return new Term(IdFieldMapper.NAME, Uid.encodeId(id));

3
server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
View File

@ -89,7 +89,8 @@ public final class QueryParserHelper {
* @param field The field name to search.
*/
public static FieldMapper getFieldMapper(MapperService mapperService, String field) {
for (DocumentMapper mapper : mapperService.docMappers(true)) {
DocumentMapper mapper = mapperService.documentMapper();
if (mapper != null) {
FieldMapper fieldMapper = mapper.mappers().smartNameFieldMapper(field);
if (fieldMapper != null) {
return fieldMapper;

2
server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java
View File

@ -433,7 +433,7 @@ public class IndexCreationTaskTests extends ESTestCase {
when(docMapper.routingFieldMapper()).thenReturn(routingMapper);
when(mapper.docMappers(anyBoolean())).thenReturn(Collections.singletonList(docMapper));
when(mapper.documentMapper()).thenReturn(docMapper);
final Index index = new Index("target", "tgt1234");
final Supplier<Sort> supplier = mock(Supplier.class);

2
server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -119,7 +119,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
} else {
throw e;
}
assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING));
assertNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING));
}
public void testTotalFieldsExceedsLimit() throws Throwable {

2
server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
View File

@ -148,7 +148,7 @@ public class ClusterStateChanges extends AbstractComponent {
when(indexService.index()).thenReturn(indexMetaData.getIndex());
MapperService mapperService = mock(MapperService.class);
when(indexService.mapperService()).thenReturn(mapperService);
when(mapperService.docMappers(anyBoolean())).thenReturn(Collections.emptyList());
when(mapperService.documentMapper()).thenReturn(null);
when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() {});
when(indexService.getIndexSortSupplier()).thenReturn(() -> null);
return indexService;