From e7618b8528736b96e84977795633166240f2127a Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 7 May 2015 23:05:12 -0700 Subject: [PATCH] Settings: Remove file based index templates As a follow up to #10870, this removes support for index templates on disk. It also removes a missed place still allowing disk based mappings. closes #11052 --- docs/reference/indices/templates.asciidoc | 38 +----- docs/reference/migration/migrate_2_0.asciidoc | 5 +- .../metadata/MetaDataCreateIndexService.java | 40 ------- .../index/mapper/FileBasedMappingsTests.java | 111 ------------------ .../IndexTemplateFileLoadingTests.java | 104 ---------------- 5 files changed, 5 insertions(+), 293 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java delete mode 100644 src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc index 36d405c1ac5..3a5a3c5731b 100644 --- a/docs/reference/indices/templates.asciidoc +++ b/docs/reference/indices/templates.asciidoc @@ -68,7 +68,7 @@ curl -XDELETE localhost:9200/_template/template_1 [float] [[getting]] -=== GETting templates +=== Getting templates Index templates are identified by a name (in the above case `template_1`) and can be retrieved using the following: @@ -157,39 +157,3 @@ for indices of that start with `te*`, source will still be enabled. Note, for mappings, the merging is "deep", meaning that specific object/property based mappings can easily be added/overridden on higher order templates, with lower order templates providing the basis. - -[float] -[[config]] -=== Config - -Index templates can also be placed within the config location -(`path.conf`) under the `templates` directory (note, make sure to place -them on all master eligible nodes). For example, a file called -`template_1.json` can be placed under `config/templates` and it will be -added if it matches an index. Here is a sample of the mentioned file: - -[source,js] --------------------------------------------------- -{ - "template_1" : { - "template" : "*", - "settings" : { - "index.number_of_shards" : 2 - }, - "mappings" : { - "_default_" : { - "_source" : { - "enabled" : false - } - }, - "type1" : { - "_all" : { - "enabled" : false - } - } - } - } -} --------------------------------------------------- - -*Please note that templates added this way will not appear in the `/_template/*` API request.* diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 06efc7a59af..0279d9d4761 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -519,4 +519,7 @@ Log messages are now truncated at 10,000 characters. This can be changed in the The `top_children` query has been removed in favour of the `has_child` query. The `top_children` query wasn't always faster than the `has_child` query and the `top_children` query was often inaccurate. The total hits and any aggregations in the -same search request will likely be off if `top_children` was used. \ No newline at end of file +same search request will likely be off if `top_children` was used. + +=== Removed file based index templates +Index templates can no longer be configured on disk. Use the `_template` API instead. diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 732561f66f1..1d0f584f618 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -300,22 +300,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - // now add config level mappings - Path mappingsDir = environment.configFile().resolve("mappings"); - if (Files.isDirectory(mappingsDir)) { - // first index level - Path indexMappingsDir = mappingsDir.resolve(request.index()); - if (Files.isDirectory(indexMappingsDir)) { - addMappings(mappings, indexMappingsDir); - } - - // second is the _default mapping - Path defaultMappingsDir = mappingsDir.resolve("_default"); - if (Files.isDirectory(defaultMappingsDir)) { - addMappings(mappings, defaultMappingsDir); - } - } - ImmutableSettings.Builder indexSettingsBuilder = settingsBuilder(); // apply templates, here, in reverse order, since first ones are better matching for (int i = templates.size() - 1; i >= 0; i--) { @@ -517,30 +501,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - // see if we have templates defined under config - final Path templatesDir = environment.configFile().resolve("templates"); - if (Files.isDirectory(templatesDir)) { - try (DirectoryStream stream = Files.newDirectoryStream(templatesDir)) { - for (Path templatesFile : stream) { - if (Files.isRegularFile(templatesFile)) { - XContentParser parser = null; - try { - final byte[] templatesData = Files.readAllBytes(templatesFile); - parser = XContentHelper.createParser(templatesData, 0, templatesData.length); - IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, templatesFile.getFileName().toString()); - if (indexTemplateFilter.apply(request, template)) { - templates.add(template); - } - } catch (Exception e) { - logger.warn("[{}] failed to read template [{}] from config", e, request.index(), templatesFile.toAbsolutePath()); - } finally { - Releasables.closeWhileHandlingException(parser); - } - } - } - } - } - CollectionUtil.timSort(templates, new Comparator() { @Override public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) { diff --git a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java b/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java deleted file mode 100644 index 6772a5dfcd4..00000000000 --- a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import com.google.common.collect.ImmutableSet; - -import org.apache.lucene.util.IOUtils; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.elasticsearch.test.ElasticsearchTestCase; - -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; - -public class FileBasedMappingsTests extends ElasticsearchTestCase { - - private static final String NAME = FileBasedMappingsTests.class.getSimpleName(); - - public void testFileBasedMappings() throws Exception { - Path configDir = createTempDir(); - Path mappingsDir = configDir.resolve("mappings"); - Path indexMappings = mappingsDir.resolve("index").resolve("type.json"); - Path defaultMappings = mappingsDir.resolve("_default").resolve("type.json"); - try { - Files.createDirectories(indexMappings.getParent()); - Files.createDirectories(defaultMappings.getParent()); - - try (OutputStream stream = Files.newOutputStream(indexMappings); - XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) { - builder.startObject() - .startObject("type") - .startObject("properties") - .startObject("f") - .field("type", "string") - .endObject() - .endObject() - .endObject() - .endObject(); - } - - try (OutputStream stream = Files.newOutputStream(defaultMappings); - XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) { - builder.startObject() - .startObject("type") - .startObject("properties") - .startObject("g") - .field("type", "string") - .endObject() - .endObject() - .endObject() - .endObject(); - } - - Settings settings = ImmutableSettings.builder() - .put(ClusterName.SETTING, NAME) - .put("node.name", NAME) - .put("path.home", createTempDir()) - .put("path.conf", configDir.toAbsolutePath()) - .put("http.enabled", false) - .build(); - - try (Node node = NodeBuilder.nodeBuilder().local(true).data(true).settings(settings).node()) { - - assertAcked(node.client().admin().indices().prepareCreate("index").addMapping("type", "h", "type=string").get()); - try { - final GetMappingsResponse response = node.client().admin().indices().prepareGetMappings("index").get(); - assertTrue(response.mappings().toString(), response.mappings().containsKey("index")); - MappingMetaData mappings = response.mappings().get("index").get("type"); - assertNotNull(mappings); - Map properties = (Map) (mappings.getSourceAsMap().get("properties")); - assertNotNull(properties); - assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet()); - } finally { - // remove the index... - assertAcked(node.client().admin().indices().prepareDelete("index")); - } - } - } finally { - IOUtils.rm(configDir); - } - } - -} diff --git a/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java b/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java deleted file mode 100644 index 86b8c3e8d1e..00000000000 --- a/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.indices.template; - -import com.carrotsearch.randomizedtesting.LifecycleScope; -import com.google.common.base.Charsets; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import static org.elasticsearch.test.ElasticsearchIntegrationTest.*; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -@ClusterScope(scope= Scope.TEST, numDataNodes =1) -public class IndexTemplateFileLoadingTests extends ElasticsearchIntegrationTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); - settingsBuilder.put(super.nodeSettings(nodeOrdinal)); - - try { - Path directory = createTempDir(); - settingsBuilder.put("path.conf", directory.toAbsolutePath()); - - Path templatesDir = directory.resolve("templates"); - Files.createDirectory(templatesDir); - - Path dst = templatesDir.resolve("template.json"); - String templatePath = "/org/elasticsearch/indices/template/template" + randomInt(5) + ".json"; - logger.info("Picking template path [{}]", templatePath); - // random template, one uses the 'setting.index.number_of_shards', the other 'settings.number_of_shards' - String template = Streams.copyToStringFromClasspath(templatePath); - Files.write(dst, template.getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - throw new RuntimeException(e); - } - - return settingsBuilder.build(); - } - - @Override - protected int numberOfShards() { - //number of shards won't be set through index settings, the one from the index templates needs to be used - return -1; - } - - @Override - protected int numberOfReplicas() { - //number of replicas won't be set through index settings, the one from the index templates needs to be used - return -1; - } - - @Test - public void testThatLoadingTemplateFromFileWorks() throws Exception { - final int iters = scaledRandomIntBetween(1, 5); - Set indices = new HashSet<>(); - for (int i = 0; i < iters; i++) { - String indexName = "foo" + randomAsciiOfLengthBetween(0, 5).toLowerCase(Locale.ROOT); - if (indices.contains(indexName)) { - continue; - } - indices.add(indexName); - createIndex(indexName); - ensureYellow(); // ensuring yellow so the test fails faster if the template cannot be loaded - - ClusterStateResponse stateResponse = client().admin().cluster().prepareState().setIndices(indexName).get(); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfShards(), is(10)); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfReplicas(), is(0)); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().size(), equalTo(1)); - String aliasName = indexName + "-alias"; - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().get(aliasName).alias(), equalTo(aliasName)); - } - } -}