From ca2dc1801cb4de6c041971639f9ccc5f5108b32a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 7 May 2012 14:00:37 +0300 Subject: [PATCH] Index Template: Allow to register index warmers in an index template, closes #1916. --- .../template/put/PutIndexTemplateRequest.java | 68 ++++++++++++++----- .../put/TransportPutIndexTemplateAction.java | 1 + .../cluster/metadata/IndexMetaData.java | 10 +++ .../metadata/IndexTemplateMetaData.java | 62 ++++++++++++++++- .../metadata/MetaDataCreateIndexService.java | 19 ++++++ .../MetaDataIndexTemplateService.java | 9 +++ .../search/warmer/IndexWarmersMetaData.java | 47 +++++++++++-- .../wamer/LocalGatewayIndicesWarmerTests.java | 31 +++++++++ .../wamer/SimpleIndicesWarmerTests.java | 41 +++++++++++ 9 files changed, 265 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index b6975bf8366..62c8ed3e432 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -21,8 +21,10 @@ package org.elasticsearch.action.admin.indices.template.put; import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeOperationRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -64,6 +66,8 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { private Map mappings = newHashMap(); + private Map customs = newHashMap(); + private TimeValue timeout = new TimeValue(10, TimeUnit.SECONDS); PutIndexTemplateRequest() { @@ -254,25 +258,35 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { */ public PutIndexTemplateRequest source(Map templateSource) { Map source = templateSource; - if (source.containsKey("template")) { - template(source.get("template").toString()); - } - if (source.containsKey("order")) { - order(XContentMapValues.nodeIntegerValue(source.get("order"), order())); - } - if (source.containsKey("settings")) { - if (!(source.get("settings") instanceof Map)) { - throw new ElasticSearchIllegalArgumentException("Malformed settings section, should include an inner object"); - } - settings((Map) source.get("settings")); - } - if (source.containsKey("mappings")) { - Map mappings = (Map) source.get("mappings"); - for (Map.Entry entry : mappings.entrySet()) { + for (Map.Entry entry : source.entrySet()) { + String name = entry.getKey(); + if (name.equals("template")) { + template(entry.getValue().toString()); + } else if (name.equals("order")) { + order(XContentMapValues.nodeIntegerValue(entry.getValue(), order())); + } else if (name.equals("settings")) { if (!(entry.getValue() instanceof Map)) { - throw new ElasticSearchIllegalArgumentException("Malformed mappings section for type [" + entry.getKey() + "], should include an inner object describing the mapping"); + throw new ElasticSearchIllegalArgumentException("Malformed settings section, should include an inner object"); + } + settings((Map) entry.getValue()); + } else if (name.equals("mappings")) { + Map mappings = (Map) entry.getValue(); + for (Map.Entry entry1 : mappings.entrySet()) { + if (!(entry1.getValue() instanceof Map)) { + throw new ElasticSearchIllegalArgumentException("Malformed mappings section for type [" + entry1.getKey() + "], should include an inner object describing the mapping"); + } + mapping(entry1.getKey(), (Map) entry1.getValue()); + } + } else { + // maybe custom? + IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name); + if (factory != null) { + try { + customs.put(name, factory.fromMap((Map) entry.getValue())); + } catch (IOException e) { + throw new ElasticSearchParseException("failed to parse custom metadata for [" + name + "]"); + } } - mapping(entry.getKey(), (Map) entry.getValue()); } } return this; @@ -308,6 +322,15 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { } + public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { + customs.put(custom.type(), custom); + return this; + } + + Map customs() { + return this.customs; + } + /** * Timeout to wait till the put mapping gets acknowledged of all current cluster nodes. Defaults to * 10s. @@ -347,6 +370,12 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { for (int i = 0; i < size; i++) { mappings.put(in.readUTF(), in.readUTF()); } + int customSize = in.readVInt(); + for (int i = 0; i < customSize; i++) { + String type = in.readUTF(); + IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in); + customs.put(type, customIndexMetaData); + } } @Override @@ -364,5 +393,10 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { out.writeUTF(entry.getKey()); out.writeUTF(entry.getValue()); } + out.writeVInt(customs.size()); + for (Map.Entry entry : customs.entrySet()) { + out.writeUTF(entry.getKey()); + IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out); + } } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 9f2ae0088f2..ddc327b3a04 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -88,6 +88,7 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeOperatio .order(request.order()) .settings(request.settings()) .mappings(request.mappings()) + .customs(request.customs()) .create(request.create()), new MetaDataIndexTemplateService.PutListener() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 24e751f16f2..929aebf3602 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -57,6 +57,8 @@ public class IndexMetaData { public interface Custom { + String type(); + interface Factory { String type(); @@ -65,9 +67,17 @@ public class IndexMetaData { void writeTo(T customIndexMetaData, StreamOutput out) throws IOException; + T fromMap(Map map) throws IOException; + T fromXContent(XContentParser parser) throws IOException; void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException; + + /** + * Merges from first to second, with first being more important, i.e., if something exists in first and second, + * first will prevail. + */ + T merge(T first, T second); } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 6e583c1df4e..729152e85ff 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -50,12 +50,15 @@ public class IndexTemplateMetaData { // the mapping source should always include the type as top level private final ImmutableMap mappings; - public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableMap mappings) { + private final ImmutableMap customs; + + public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableMap mappings, ImmutableMap customs) { this.name = name; this.order = order; this.template = template; this.settings = settings; this.mappings = mappings; + this.customs = customs; } public String name() { @@ -98,6 +101,18 @@ public class IndexTemplateMetaData { return this.mappings; } + public ImmutableMap customs() { + return this.customs; + } + + public ImmutableMap getCustoms() { + return this.customs; + } + + public T custom(String type) { + return (T) customs.get(type); + } + public static Builder builder(String name) { return new Builder(name); } @@ -140,6 +155,8 @@ public class IndexTemplateMetaData { private MapBuilder mappings = MapBuilder.newMapBuilder(); + private MapBuilder customs = MapBuilder.newMapBuilder(); + public Builder(String name) { this.name = name; } @@ -150,6 +167,7 @@ public class IndexTemplateMetaData { template(indexTemplateMetaData.template()); settings(indexTemplateMetaData.settings()); mappings.putAll(indexTemplateMetaData.mappings()); + customs.putAll(indexTemplateMetaData.customs()); } public Builder order(int order) { @@ -191,8 +209,22 @@ public class IndexTemplateMetaData { return this; } + public Builder putCustom(String type, IndexMetaData.Custom customIndexMetaData) { + this.customs.put(type, customIndexMetaData); + return this; + } + + public Builder removeCustom(String type) { + this.customs.remove(type); + return this; + } + + public IndexMetaData.Custom getCustom(String type) { + return this.customs.get(type); + } + public IndexTemplateMetaData build() { - return new IndexTemplateMetaData(name, order, template, settings, mappings.immutableMap()); + return new IndexTemplateMetaData(name, order, template, settings, mappings.immutableMap(), customs.immutableMap()); } public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -216,6 +248,12 @@ public class IndexTemplateMetaData { } builder.endArray(); + for (Map.Entry entry : indexTemplateMetaData.customs().entrySet()) { + builder.startObject(entry.getKey(), XContentBuilder.FieldCaseConversion.NONE); + IndexMetaData.lookupFactorySafe(entry.getKey()).toXContent(entry.getValue(), builder, params); + builder.endObject(); + } + builder.endObject(); } @@ -262,6 +300,15 @@ public class IndexTemplateMetaData { builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string()); } } + } else { + // check if its a custom index metadata + IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn + parser.skipChildren(); + } else { + builder.putCustom(factory.type(), factory.fromXContent(parser)); + } } } else if (token == XContentParser.Token.START_ARRAY) { if ("mappings".equals(currentFieldName)) { @@ -299,6 +346,12 @@ public class IndexTemplateMetaData { for (int i = 0; i < mappingsSize; i++) { builder.putMapping(in.readUTF(), CompressedString.readCompressedString(in)); } + int customSize = in.readVInt(); + for (int i = 0; i < customSize; i++) { + String type = in.readUTF(); + IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in); + builder.putCustom(type, customIndexMetaData); + } return builder.build(); } @@ -312,6 +365,11 @@ public class IndexTemplateMetaData { out.writeUTF(entry.getKey()); entry.getValue().writeTo(out); } + out.writeVInt(indexTemplateMetaData.customs().size()); + for (Map.Entry entry : indexTemplateMetaData.customs().entrySet()) { + out.writeUTF(entry.getKey()); + IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out); + } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index f0927844c67..b6da90ed3f0 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -146,12 +146,16 @@ public class MetaDataCreateIndexService extends AbstractComponent { // find templates, highest order are better matching List templates = findTemplates(request, currentState); + Map customs = Maps.newHashMap(); + // add the request mapping Map> mappings = Maps.newHashMap(); for (Map.Entry entry : request.mappings.entrySet()) { mappings.put(entry.getKey(), parseMapping(entry.getValue())); } + // TODO: request should be able to add custom metadata + // apply templates, merging the mappings into the request mapping if exists for (IndexTemplateMetaData template : templates) { for (Map.Entry entry : template.mappings().entrySet()) { @@ -161,6 +165,18 @@ public class MetaDataCreateIndexService extends AbstractComponent { mappings.put(entry.getKey(), parseMapping(entry.getValue().string())); } } + // handle custom + for (Map.Entry customEntry : template.customs().entrySet()) { + String type = customEntry.getKey(); + IndexMetaData.Custom custom = customEntry.getValue(); + IndexMetaData.Custom existing = customs.get(type); + if (existing == null) { + customs.put(type, custom); + } else { + IndexMetaData.Custom merged = IndexMetaData.lookupFactorySafe(type).merge(existing, custom); + customs.put(type, merged); + } + } } // now add config level mappings @@ -259,6 +275,9 @@ public class MetaDataCreateIndexService extends AbstractComponent { for (MappingMetaData mappingMd : mappingsMetaData.values()) { indexMetaDataBuilder.putMapping(mappingMd); } + for (Map.Entry customEntry : customs.entrySet()) { + indexMetaDataBuilder.putCustom(customEntry.getKey(), customEntry.getValue()); + } indexMetaDataBuilder.state(request.state); final IndexMetaData indexMetaData = indexMetaDataBuilder.build(); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index 841b3f437bb..90fd8913e6d 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -106,6 +106,9 @@ public class MetaDataIndexTemplateService extends AbstractComponent { for (Map.Entry entry : request.mappings.entrySet()) { templateBuilder.putMapping(entry.getKey(), entry.getValue()); } + for (Map.Entry entry : request.customs.entrySet()) { + templateBuilder.putCustom(entry.getKey(), entry.getValue()); + } } catch (Exception e) { listener.onFailure(e); return; @@ -180,6 +183,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent { String template; Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS; Map mappings = Maps.newHashMap(); + Map customs = Maps.newHashMap(); public PutRequest(String cause, String name) { this.cause = cause; @@ -211,6 +215,11 @@ public class MetaDataIndexTemplateService extends AbstractComponent { return this; } + public PutRequest customs(Map customs) { + this.customs.putAll(customs); + return this; + } + public PutRequest putMapping(String mappingType, String mappingSource) { mappings.put(mappingType, mappingSource); return this; diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 75918153987..c4f2f0a79a8 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -20,16 +20,14 @@ package org.elasticsearch.search.warmer; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.*; import java.io.IOException; import java.util.ArrayList; @@ -84,6 +82,11 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { return this.entries; } + @Override + public String type() { + return TYPE; + } + public static class Factory implements IndexMetaData.Custom.Factory { @Override @@ -115,6 +118,23 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { } } + @Override + public IndexWarmersMetaData fromMap(Map map) throws IOException { + // if it starts with the type, remove it + if (map.size() == 1 && map.containsKey(TYPE)) { + map = (Map) map.values().iterator().next(); + } + XContentBuilder builder = XContentFactory.smileBuilder().map(map); + XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.underlyingBytes(), 0, builder.underlyingBytesLength()); + try { + // move to START_OBJECT + parser.nextToken(); + return fromXContent(parser); + } finally { + parser.close(); + } + } + @Override public IndexWarmersMetaData fromXContent(XContentParser parser) throws IOException { // we get here after we are at warmers token @@ -178,5 +198,24 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { } builder.endObject(); } + + @Override + public IndexWarmersMetaData merge(IndexWarmersMetaData first, IndexWarmersMetaData second) { + List entries = Lists.newArrayList(); + entries.addAll(first.entries()); + for (Entry secondEntry : second.entries()) { + boolean found = false; + for (Entry firstEntry : first.entries()) { + if (firstEntry.name().equals(secondEntry.name())) { + found = true; + break; + } + } + if (!found) { + entries.add(secondEntry); + } + } + return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); + } } } diff --git a/src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java b/src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java index 52a661ffedb..bf0742a952b 100644 --- a/src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java @@ -82,12 +82,34 @@ public class LocalGatewayIndicesWarmerTests extends AbstractNodesTests { .setSearchRequest(client("node1").prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "value2"))) .execute().actionGet(); + logger.info("--> put template with warmer"); + client("node1").admin().indices().preparePutTemplate("template_1") + .setSource("{\n" + + " \"template\" : \"xxx\",\n" + + " \"warmers\" : {\n" + + " \"warmer_1\" : {\n" + + " \"types\" : [],\n" + + " \"source\" : {\n" + + " \"query\" : {\n" + + " \"match_all\" : {}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}") + .execute().actionGet(); + + logger.info("--> verify warmers are registered in cluster state"); ClusterState clusterState = client("node1").admin().cluster().prepareState().execute().actionGet().state(); IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(2)); + IndexWarmersMetaData templateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE); + assertThat(templateWarmers, Matchers.notNullValue()); + assertThat(templateWarmers.entries().size(), equalTo(1)); + logger.info("--> close the node"); closeNode("node1"); @@ -106,6 +128,15 @@ public class LocalGatewayIndicesWarmerTests extends AbstractNodesTests { assertThat(recoveredWarmersMetaData.entries().get(i).source(), equalTo(warmersMetaData.entries().get(i).source())); } + logger.info("--> verify warmers in template are recovered"); + IndexWarmersMetaData recoveredTemplateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE); + assertThat(recoveredTemplateWarmers.entries().size(), equalTo(templateWarmers.entries().size())); + for (int i = 0; i < templateWarmers.entries().size(); i++) { + assertThat(recoveredTemplateWarmers.entries().get(i).name(), equalTo(templateWarmers.entries().get(i).name())); + assertThat(recoveredTemplateWarmers.entries().get(i).source(), equalTo(templateWarmers.entries().get(i).source())); + } + + logger.info("--> delete warmer warmer_1"); client("node1").admin().indices().prepareDeleteWarmer().setIndices("test").setName("warmer_1").execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/test/integration/indices/wamer/SimpleIndicesWarmerTests.java b/src/test/java/org/elasticsearch/test/integration/indices/wamer/SimpleIndicesWarmerTests.java index 99675f3072a..466e8774047 100644 --- a/src/test/java/org/elasticsearch/test/integration/indices/wamer/SimpleIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/test/integration/indices/wamer/SimpleIndicesWarmerTests.java @@ -20,13 +20,19 @@ package org.elasticsearch.test.integration.indices.wamer; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.integration.AbstractNodesTests; +import org.hamcrest.Matchers; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + /** */ public class SimpleIndicesWarmerTests extends AbstractNodesTests { @@ -70,4 +76,39 @@ public class SimpleIndicesWarmerTests extends AbstractNodesTests { client.prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); client.prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); } + + @Test + public void templateWarmer() { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().preparePutTemplate("template_1") + .setSource("{\n" + + " \"template\" : \"*\",\n" + + " \"warmers\" : {\n" + + " \"warmer_1\" : {\n" + + " \"types\" : [],\n" + + " \"source\" : {\n" + + " \"query\" : {\n" + + " \"match_all\" : {}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}") + .execute().actionGet(); + + client.admin().indices().prepareCreate("test") + .setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + ClusterState clusterState = client.admin().cluster().prepareState().execute().actionGet().state(); + IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + assertThat(warmersMetaData, Matchers.notNullValue()); + assertThat(warmersMetaData.entries().size(), equalTo(1)); + + client.prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client.prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); + } }