honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Settings: Remove file based index templates 

As a follow up to #10870, this removes support for
index templates on disk. It also removes a missed
place still allowing disk based mappings.

closes #11052
This commit is contained in:
Ryan Ernst 2015-05-07 23:05:12 -07:00
parent 6dcd5bf618
commit e7618b8528
5 changed files with 5 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
docs/reference/indices/templates.asciidoc
View File

@ -68,7 +68,7 @@ curl -XDELETE localhost:9200/_template/template_1
[float] [float]
[[getting]] [[getting]]
=== GETting templates === Getting templates
Index templates are identified by a name (in the above case Index templates are identified by a name (in the above case
`template_1`) and can be retrieved using the following: `template_1`) and can be retrieved using the following:
@ -157,39 +157,3 @@ for indices of that start with `te*`, source will still be enabled.
Note, for mappings, the merging is "deep", meaning that specific Note, for mappings, the merging is "deep", meaning that specific
object/property based mappings can easily be added/overridden on higher object/property based mappings can easily be added/overridden on higher
order templates, with lower order templates providing the basis. order templates, with lower order templates providing the basis.
[float]
[[config]]
=== Config
Index templates can also be placed within the config location
(`path.conf`) under the `templates` directory (note, make sure to place
them on all master eligible nodes). For example, a file called
`template_1.json` can be placed under `config/templates` and it will be
added if it matches an index. Here is a sample of the mentioned file:
[source,js]
--------------------------------------------------
{
"template_1" : {
"template" : "*",
"settings" : {
"index.number_of_shards" : 2
},
"mappings" : {
"_default_" : {
"_source" : {
"enabled" : false
}
},
"type1" : {
"_all" : {
"enabled" : false
}
}
}
}
}
--------------------------------------------------
*Please note that templates added this way will not appear in the `/_template/*` API request.*

5
docs/reference/migration/migrate_2_0.asciidoc
View File

@ -519,4 +519,7 @@ Log messages are now truncated at 10,000 characters. This can be changed in the
The `top_children` query has been removed in favour of the `has_child` query. The `top_children` query wasn't always faster The `top_children` query has been removed in favour of the `has_child` query. The `top_children` query wasn't always faster
than the `has_child` query and the `top_children` query was often inaccurate. The total hits and any aggregations in the than the `has_child` query and the `top_children` query was often inaccurate. The total hits and any aggregations in the
same search request will likely be off if `top_children` was used. same search request will likely be off if `top_children` was used.
=== Removed file based index templates
Index templates can no longer be configured on disk. Use the `_template` API instead.

40
src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -300,22 +300,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
} }
} }
// now add config level mappings
Path mappingsDir = environment.configFile().resolve("mappings");
if (Files.isDirectory(mappingsDir)) {
// first index level
Path indexMappingsDir = mappingsDir.resolve(request.index());
if (Files.isDirectory(indexMappingsDir)) {
addMappings(mappings, indexMappingsDir);
}
// second is the _default mapping
Path defaultMappingsDir = mappingsDir.resolve("_default");
if (Files.isDirectory(defaultMappingsDir)) {
addMappings(mappings, defaultMappingsDir);
}
}
ImmutableSettings.Builder indexSettingsBuilder = settingsBuilder(); ImmutableSettings.Builder indexSettingsBuilder = settingsBuilder();
// apply templates, here, in reverse order, since first ones are better matching // apply templates, here, in reverse order, since first ones are better matching
for (int i = templates.size() - 1; i >= 0; i--) { for (int i = templates.size() - 1; i >= 0; i--) {
@ -517,30 +501,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
} }
} }
// see if we have templates defined under config
final Path templatesDir = environment.configFile().resolve("templates");
if (Files.isDirectory(templatesDir)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(templatesDir)) {
for (Path templatesFile : stream) {
if (Files.isRegularFile(templatesFile)) {
XContentParser parser = null;
try {
final byte[] templatesData = Files.readAllBytes(templatesFile);
parser = XContentHelper.createParser(templatesData, 0, templatesData.length);
IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, templatesFile.getFileName().toString());
if (indexTemplateFilter.apply(request, template)) {
templates.add(template);
}
} catch (Exception e) {
logger.warn("[{}] failed to read template [{}] from config", e, request.index(), templatesFile.toAbsolutePath());
} finally {
Releasables.closeWhileHandlingException(parser);
}
}
}
}
}
CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() { CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
@Override @Override
public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) { public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {

111
src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java
View File

@ -1,111 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.test.ElasticsearchTestCase;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
public class FileBasedMappingsTests extends ElasticsearchTestCase {
private static final String NAME = FileBasedMappingsTests.class.getSimpleName();
public void testFileBasedMappings() throws Exception {
Path configDir = createTempDir();
Path mappingsDir = configDir.resolve("mappings");
Path indexMappings = mappingsDir.resolve("index").resolve("type.json");
Path defaultMappings = mappingsDir.resolve("_default").resolve("type.json");
try {
Files.createDirectories(indexMappings.getParent());
Files.createDirectories(defaultMappings.getParent());
try (OutputStream stream = Files.newOutputStream(indexMappings);
XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) {
builder.startObject()
.startObject("type")
.startObject("properties")
.startObject("f")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject();
}
try (OutputStream stream = Files.newOutputStream(defaultMappings);
XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) {
builder.startObject()
.startObject("type")
.startObject("properties")
.startObject("g")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject();
}
Settings settings = ImmutableSettings.builder()
.put(ClusterName.SETTING, NAME)
.put("node.name", NAME)
.put("path.home", createTempDir())
.put("path.conf", configDir.toAbsolutePath())
.put("http.enabled", false)
.build();
try (Node node = NodeBuilder.nodeBuilder().local(true).data(true).settings(settings).node()) {
assertAcked(node.client().admin().indices().prepareCreate("index").addMapping("type", "h", "type=string").get());
try {
final GetMappingsResponse response = node.client().admin().indices().prepareGetMappings("index").get();
assertTrue(response.mappings().toString(), response.mappings().containsKey("index"));
MappingMetaData mappings = response.mappings().get("index").get("type");
assertNotNull(mappings);
Map<?, ?> properties = (Map<?, ?>) (mappings.getSourceAsMap().get("properties"));
assertNotNull(properties);
assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet());
} finally {
// remove the index...
assertAcked(node.client().admin().indices().prepareDelete("index"));
}
}
} finally {
IOUtils.rm(configDir);
}
}
}

104
src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java
View File

@ -1,104 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices.template;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.base.Charsets;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.*;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ClusterScope(scope= Scope.TEST, numDataNodes =1)
public class IndexTemplateFileLoadingTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder();
settingsBuilder.put(super.nodeSettings(nodeOrdinal));
try {
Path directory = createTempDir();
settingsBuilder.put("path.conf", directory.toAbsolutePath());
Path templatesDir = directory.resolve("templates");
Files.createDirectory(templatesDir);
Path dst = templatesDir.resolve("template.json");
String templatePath = "/org/elasticsearch/indices/template/template" + randomInt(5) + ".json";
logger.info("Picking template path [{}]", templatePath);
// random template, one uses the 'setting.index.number_of_shards', the other 'settings.number_of_shards'
String template = Streams.copyToStringFromClasspath(templatePath);
Files.write(dst, template.getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
throw new RuntimeException(e);
}
return settingsBuilder.build();
}
@Override
protected int numberOfShards() {
//number of shards won't be set through index settings, the one from the index templates needs to be used
return -1;
}
@Override
protected int numberOfReplicas() {
//number of replicas won't be set through index settings, the one from the index templates needs to be used
return -1;
}
@Test
public void testThatLoadingTemplateFromFileWorks() throws Exception {
final int iters = scaledRandomIntBetween(1, 5);
Set<String> indices = new HashSet<>();
for (int i = 0; i < iters; i++) {
String indexName = "foo" + randomAsciiOfLengthBetween(0, 5).toLowerCase(Locale.ROOT);
if (indices.contains(indexName)) {
continue;
}
indices.add(indexName);
createIndex(indexName);
ensureYellow(); // ensuring yellow so the test fails faster if the template cannot be loaded
ClusterStateResponse stateResponse = client().admin().cluster().prepareState().setIndices(indexName).get();
assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfShards(), is(10));
assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfReplicas(), is(0));
assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().size(), equalTo(1));
String aliasName = indexName + "-alias";
assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().get(aliasName).alias(), equalTo(aliasName));
}
}
}