diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 4536670872d..d1141aeb9f4 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -34,8 +34,6 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import java.util.Collections; -import static org.elasticsearch.common.util.set.Sets.newHashSet; - /** * This service is responsible for upgrading legacy index metadata to the current version *

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index 1817a71073c..b7d4e321de8 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -64,8 +64,11 @@ import org.elasticsearch.indices.recovery.RecoveryTargetService; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.indices.ttl.IndicesTTLService; +import org.elasticsearch.plugins.MapperPlugin; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -73,17 +76,16 @@ import java.util.Map; */ public class IndicesModule extends AbstractModule { - private final Map mapperParsers - = new LinkedHashMap<>(); - // Use a LinkedHashMap for metadataMappers because iteration order matters - private final Map metadataMapperParsers - = new LinkedHashMap<>(); + private final Map mapperParsers; + private final Map metadataMapperParsers; + private final MapperRegistry mapperRegistry; private final NamedWriteableRegistry namedWritableRegistry; - public IndicesModule(NamedWriteableRegistry namedWriteableRegistry) { + public IndicesModule(NamedWriteableRegistry namedWriteableRegistry, List mapperPlugins) { this.namedWritableRegistry = namedWriteableRegistry; - registerBuiltInMappers(); - registerBuiltInMetadataMappers(); + this.mapperParsers = getMappers(mapperPlugins); + this.metadataMapperParsers = getMetadataMappers(mapperPlugins); + this.mapperRegistry = new MapperRegistry(mapperParsers, metadataMapperParsers); registerBuildInWritables(); } @@ -92,65 +94,72 @@ public class IndicesModule extends AbstractModule { namedWritableRegistry.register(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new); } - private void registerBuiltInMappers() { + private Map getMappers(List mapperPlugins) { + Map mappers = new LinkedHashMap<>(); + + // builtin mappers for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) { - registerMapper(type.typeName(), new NumberFieldMapper.TypeParser(type)); + mappers.put(type.typeName(), new NumberFieldMapper.TypeParser(type)); } - registerMapper(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); - registerMapper(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); - registerMapper(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); - registerMapper(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); - registerMapper(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); - registerMapper(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser()); - registerMapper(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser()); - registerMapper(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); - registerMapper(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); - registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); - registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); - registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - + mappers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); + mappers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); + mappers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); + mappers.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); + mappers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + mappers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser()); + mappers.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser()); + mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); + mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); + mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); + mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); + mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { - registerMapper(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); + mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); } + + for (MapperPlugin mapperPlugin : mapperPlugins) { + for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { + if (mappers.put(entry.getKey(), entry.getValue()) != null) { + throw new IllegalArgumentException("Mapper [" + entry.getKey() + "] is already registered"); + } + } + } + return Collections.unmodifiableMap(mappers); } - private void registerBuiltInMetadataMappers() { - // NOTE: the order is important + private Map getMetadataMappers(List mapperPlugins) { + // Use a LinkedHashMap for metadataMappers because iteration order matters + Map metadataMappers = new LinkedHashMap<>(); + // builtin metadata mappers // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination - registerMetadataMapper(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); - registerMetadataMapper(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); - registerMetadataMapper(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); - registerMetadataMapper(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); - registerMetadataMapper(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); - registerMetadataMapper(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); - registerMetadataMapper(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); - registerMetadataMapper(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); - registerMetadataMapper(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); - registerMetadataMapper(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); - registerMetadataMapper(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); - // _field_names is not registered here, see #getMapperRegistry: we need to register it - // last so that it can see all other mappers, including those coming from plugins - } + metadataMappers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); + metadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); + metadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); + metadataMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); + metadataMappers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); + metadataMappers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); + metadataMappers.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); + metadataMappers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); + metadataMappers.put(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); + metadataMappers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); + metadataMappers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); + // _field_names is not registered here, see below - /** - * Register a mapper for the given type. - */ - public synchronized void registerMapper(String type, Mapper.TypeParser parser) { - if (mapperParsers.containsKey(type)) { - throw new IllegalArgumentException("A mapper is already registered for type [" + type + "]"); + for (MapperPlugin mapperPlugin : mapperPlugins) { + for (Map.Entry entry : mapperPlugin.getMetadataMappers().entrySet()) { + if (entry.getKey().equals(FieldNamesFieldMapper.NAME)) { + throw new IllegalArgumentException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]"); + } + if (metadataMappers.put(entry.getKey(), entry.getValue()) != null) { + throw new IllegalArgumentException("MetadataFieldMapper [" + entry.getKey() + "] is already registered"); + } + } } - mapperParsers.put(type, parser); - } - /** - * Register a root mapper under the given name. - */ - public synchronized void registerMetadataMapper(String name, MetadataFieldMapper.TypeParser parser) { - if (metadataMapperParsers.containsKey(name)) { - throw new IllegalArgumentException("A mapper is already registered for metadata mapper [" + name + "]"); - } - metadataMapperParsers.put(name, parser); + // we register _field_names here so that it has a chance to see all other mappers, including from plugins + metadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); + return Collections.unmodifiableMap(metadataMappers); } @Override @@ -172,16 +181,8 @@ public class IndicesModule extends AbstractModule { } // public for testing - public synchronized MapperRegistry getMapperRegistry() { - // NOTE: we register _field_names here so that it has a chance to see all other - // mappers, including from plugins - if (metadataMapperParsers.containsKey(FieldNamesFieldMapper.NAME)) { - throw new IllegalStateException("Metadata mapper [" + FieldNamesFieldMapper.NAME + "] is already registered"); - } - final Map metadataMapperParsers - = new LinkedHashMap<>(this.metadataMapperParsers); - metadataMapperParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); - return new MapperRegistry(mapperParsers, metadataMapperParsers); + public MapperRegistry getMapperRegistry() { + return mapperRegistry; } protected void bindMapperExtension() { diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index afc0e5b4cd8..f30fc9005b7 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -86,6 +86,7 @@ import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.node.service.NodeService; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; @@ -258,7 +259,7 @@ public class Node implements Closeable { modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry)); modules.add(new DiscoveryModule(this.settings)); modules.add(new ClusterModule(this.settings, clusterService)); - modules.add(new IndicesModule(namedWriteableRegistry)); + modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class))); modules.add(new SearchModule(settings, namedWriteableRegistry)); modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false)); modules.add(new GatewayModule()); diff --git a/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java new file mode 100644 index 00000000000..5dfcdc6bda4 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import java.util.Collections; +import java.util.Map; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; + +/** + * An extension point for {@link Plugin} implementations to add custom mappers + */ +public interface MapperPlugin { + + /** + * Returns additional mapper implementations added by this plugin. + * + * The key of the returned {@link Map} is the unique name for the mapper which will be used + * as the mapping {@code type}, and the value is a {@link Mapper.TypeParser} to parse the + * mapper settings into a {@link Mapper}. + */ + default Map getMappers() { + return Collections.emptyMap(); + } + + /** + * Returns additional metadata mapper implementations added by this plugin. + * + * The key of the returned {@link Map} is the unique name for the metadata mapper, which + * is used in the mapping json to configure the metadata mapper, and the value is a + * {@link MetadataFieldMapper.TypeParser} to parse the mapper settings into a + * {@link MetadataFieldMapper}. + */ + default Map getMetadataMappers() { + return Collections.emptyMap(); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 4467ea63082..2769534aee0 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -135,7 +135,7 @@ public class IndexModuleTests extends ESTestCase { environment = new Environment(settings); nodeServicesProvider = newNodeServiceProvider(settings, environment, null); nodeEnvironment = new NodeEnvironment(settings, environment); - mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry(); + mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry(); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 45c8fbdc39a..a0396b7abc6 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -2023,7 +2023,7 @@ public class InternalEngineTests extends ESTestCase { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); - MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry(); + MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry(); MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null); DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService); this.docMapper = b.build(mapperService); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java index 115faf2c6a1..81b7375ab2e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java @@ -19,21 +19,35 @@ package org.elasticsearch.index.mapper.externalvalues; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -public class ExternalMapperPlugin extends Plugin { +public class ExternalMapperPlugin extends Plugin implements MapperPlugin { public static final String EXTERNAL = "external"; public static final String EXTERNAL_BIS = "external_bis"; public static final String EXTERNAL_UPPER = "external_upper"; - public void onModule(IndicesModule indicesModule) { - indicesModule.registerMetadataMapper(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); - indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); - indicesModule.registerMapper(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); - indicesModule.registerMapper(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); - indicesModule.registerMapper(FakeStringFieldMapper.CONTENT_TYPE, new FakeStringFieldMapper.TypeParser()); + @Override + public Map getMappers() { + Map mappers = new HashMap<>(); + mappers.put(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); + mappers.put(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); + mappers.put(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); + mappers.put(FakeStringFieldMapper.CONTENT_TYPE, new FakeStringFieldMapper.TypeParser()); + return Collections.unmodifiableMap(mappers); + } + + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java index 2ec5aee9d15..11804ab1d2c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -39,10 +40,12 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.TermBasedFieldType; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.SortedSet; @@ -235,8 +238,10 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { public void testSeesFieldsFromPlugins() throws IOException { IndexService indexService = createIndex("test"); - IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry()); - indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser()); + IndicesModule indicesModule = newTestIndicesModule( + Collections.emptyMap(), + Collections.singletonMap("_dummy", new DummyMetadataFieldMapper.TypeParser()) + ); final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java new file mode 100644 index 00000000000..73e78650128 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.core.TextFieldMapper; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.internal.IdFieldMapper; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +public class IndicesModuleTests extends ESTestCase { + + private static class FakeMapperParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + return null; + } + } + + private static class FakeMetadataMapperParser implements MetadataFieldMapper.TypeParser { + @Override + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + return null; + } + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return null; + } + } + + List fakePlugins = Arrays.asList(new MapperPlugin() { + @Override + public Map getMappers() { + return Collections.singletonMap("fake-mapper", new FakeMapperParser()); + } + @Override + public Map getMetadataMappers() { + return Collections.singletonMap("fake-metadata-mapper", new FakeMetadataMapperParser()); + } + }); + + public void testBuiltinMappers() { + IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()); + assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); + assertFalse(module.getMapperRegistry().getMetadataMapperParsers().isEmpty()); + } + + public void testBuiltinWithPlugins() { + IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), fakePlugins); + MapperRegistry registry = module.getMapperRegistry(); + assertThat(registry.getMapperParsers().size(), Matchers.greaterThan(1)); + assertThat(registry.getMetadataMapperParsers().size(), Matchers.greaterThan(1)); + } + + public void testDuplicateBuiltinMapper() { + List plugins = Arrays.asList(new MapperPlugin() { + @Override + public Map getMappers() { + return Collections.singletonMap(TextFieldMapper.CONTENT_TYPE, new FakeMapperParser()); + } + }); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(new NamedWriteableRegistry(), plugins)); + assertThat(e.getMessage(), Matchers.containsString("already registered")); + } + + public void testDuplicateOtherPluginMapper() { + MapperPlugin plugin = new MapperPlugin() { + @Override + public Map getMappers() { + return Collections.singletonMap("foo", new FakeMapperParser()); + } + }; + List plugins = Arrays.asList(plugin, plugin); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(new NamedWriteableRegistry(), plugins)); + assertThat(e.getMessage(), Matchers.containsString("already registered")); + } + + public void testDuplicateBuiltinMetadataMapper() { + List plugins = Arrays.asList(new MapperPlugin() { + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(IdFieldMapper.NAME, new FakeMetadataMapperParser()); + } + }); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(new NamedWriteableRegistry(), plugins)); + assertThat(e.getMessage(), Matchers.containsString("already registered")); + } + + public void testDuplicateOtherPluginMetadataMapper() { + MapperPlugin plugin = new MapperPlugin() { + @Override + public Map getMetadataMappers() { + return Collections.singletonMap("foo", new FakeMetadataMapperParser()); + } + }; + List plugins = Arrays.asList(plugin, plugin); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(new NamedWriteableRegistry(), plugins)); + assertThat(e.getMessage(), Matchers.containsString("already registered")); + } + + public void testDuplicateFieldNamesMapper() { + List plugins = Arrays.asList(new MapperPlugin() { + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(FieldNamesFieldMapper.NAME, new FakeMetadataMapperParser()); + } + }); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(new NamedWriteableRegistry(), plugins)); + assertThat(e.getMessage(), Matchers.containsString("cannot contain metadata mapper [_field_names]")); + } + + public void testFieldNamesIsLast() { + IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()); + List fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet() + .stream().collect(Collectors.toList()); + assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); + } + + public void testFieldNamesIsLastWithPlugins() { + IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), fakePlugins); + List fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet() + .stream().collect(Collectors.toList()); + assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java index 08109a437cc..1a21069623d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java @@ -114,7 +114,7 @@ public class AggregatorParsingTests extends ESTestCase { b.bind(ScriptService.class).toInstance(scriptModule.getScriptService()); }, settingsModule, - new IndicesModule(namedWriteableRegistry) { + new IndicesModule(namedWriteableRegistry, Collections.emptyList()) { @Override protected void configure() { bindMapperExtension(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java index d953eb02174..4e0429fca87 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java @@ -145,8 +145,7 @@ public abstract class BaseAggregationTestCase contextFactory.get()); IndicesFieldDataCache cache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); - IndexFieldDataService indexFieldDataService =new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService); + IndexFieldDataService indexFieldDataService = new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() { @Override - public void onCache(ShardId shardId, Accountable accountable) { - - } - + public void onCache(ShardId shardId, Accountable accountable) {} @Override - public void onRemoval(ShardId shardId, Accountable accountable) { - - } + public void onRemoval(ShardId shardId, Accountable accountable) {} }); IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); contextFactory = () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java index 23251ca20d0..7cbf07e1981 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java @@ -19,20 +19,23 @@ package org.elasticsearch.percolator; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + import org.elasticsearch.action.ActionModule; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchModule; -import java.util.Arrays; -import java.util.List; - -public class PercolatorPlugin extends Plugin { +public class PercolatorPlugin extends Plugin implements MapperPlugin { public static final String NAME = "percolator"; @@ -56,10 +59,6 @@ public class PercolatorPlugin extends Plugin { } } - public void onModule(IndicesModule module) { - module.registerMapper(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser()); - } - public void onModule(SearchModule module) { module.registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD); module.registerFetchSubPhase(new PercolatorHighlightSubFetchPhase(settings, module.getHighlighters())); @@ -70,6 +69,11 @@ public class PercolatorPlugin extends Plugin { return Arrays.asList(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING); } + @Override + public Map getMappers() { + return Collections.singletonMap(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser()); + } + static boolean transportClientMode(Settings settings) { return TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())); } diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java index d523e0f0184..6cf957f05c0 100644 --- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java @@ -19,18 +19,20 @@ package org.elasticsearch.mapper.attachments; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.SettingsModule; -import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import java.util.Arrays; -import java.util.List; - -public class MapperAttachmentsPlugin extends Plugin { +public class MapperAttachmentsPlugin extends Plugin implements MapperPlugin { private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment"); private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger); @@ -44,7 +46,8 @@ public class MapperAttachmentsPlugin extends Plugin { AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING); } - public void onModule(IndicesModule indicesModule) { - indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser()); + @Override + public Map getMappers() { + return Collections.singletonMap("attachment", new AttachmentMapper.TypeParser()); } } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java index e0d5d7a2ec6..29dfff66d96 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java @@ -19,12 +19,18 @@ package org.elasticsearch.mapper.attachments; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -33,9 +39,10 @@ public abstract class AttachmentUnitTestCase extends ESTestCase { protected Settings testSettings; protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() { - IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry()); - indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); - return indicesModule; + return newTestIndicesModule( + Collections.singletonMap(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()), + Collections.emptyMap() + ); } @Before diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java index 384fc4272f6..987a4cf9bc0 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java @@ -19,14 +19,18 @@ package org.elasticsearch.plugin.mapper; +import java.util.Collections; +import java.util.Map; + +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.murmur3.Murmur3FieldMapper; -import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -public class MapperMurmur3Plugin extends Plugin { +public class MapperMurmur3Plugin extends Plugin implements MapperPlugin { - public void onModule(IndicesModule indicesModule) { - indicesModule.registerMapper(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + @Override + public Map getMappers() { + return Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); } - } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java index 1aecfdf444d..1df48d5695a 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java @@ -19,13 +19,18 @@ package org.elasticsearch.plugin.mapper; +import java.util.Collections; +import java.util.Map; + +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.size.SizeFieldMapper; -import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -public class MapperSizePlugin extends Plugin { +public class MapperSizePlugin extends Plugin implements MapperPlugin { - public void onModule(IndicesModule indicesModule) { - indicesModule.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); } } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index ce44fef59a5..b0802a955df 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.mapper.size; +import java.util.Collections; +import java.util.Map; + import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -26,10 +29,12 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; @@ -48,8 +53,9 @@ public class SizeMappingTests extends ESSingleNodeTestCase { @Before public void before() { indexService = createIndex("test"); - IndicesModule indices = new IndicesModule(new NamedWriteableRegistry()); - indices.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); + IndicesModule indices = newTestIndicesModule(Collections.emptyMap(), + Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()) + ); mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::newQueryShardContext); parser = mapperService.documentMapperParser(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 1ac0b53cbab..8889dc5aac5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -42,7 +42,7 @@ import static org.elasticsearch.test.ESTestCase.createAnalysisService; public class MapperTestUtils { public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException { - IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry()); + IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()); return newMapperService(tempDir, indexSettings, indicesModule); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 95fadc99e5c..6a5ea4f7afe 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -895,7 +895,7 @@ public abstract class AbstractQueryTestCase> b.bind(Environment.class).toInstance(new Environment(settings)); b.bind(ThreadPool.class).toInstance(threadPool); }, - settingsModule, new IndicesModule(namedWriteableRegistry) { + settingsModule, new IndicesModule(namedWriteableRegistry, Collections.emptyList()) { @Override public void configure() { // skip services diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 50d6dc88aaf..860f133a453 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; @@ -56,8 +57,12 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.ScriptModule; @@ -810,4 +815,21 @@ public abstract class ESTestCase extends LuceneTestCase { Environment environment = new Environment(settings); return new ScriptModule(settings, environment, null, singletonList(new MockScriptEngine()), emptyList()); } + + /** Creates an IndicesModule for testing with the given mappers and metadata mappers. */ + public static IndicesModule newTestIndicesModule(Map extraMappers, + Map extraMetadataMappers) { + return new IndicesModule(new NamedWriteableRegistry(), Collections.singletonList( + new MapperPlugin() { + @Override + public Map getMappers() { + return extraMappers; + } + @Override + public Map getMetadataMappers() { + return extraMetadataMappers; + } + } + )); + } }