honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Index Template: Allow to register index warmers in an index template, closes #1916.

This commit is contained in:
Shay Banon 2012-05-07 14:00:37 +03:00
parent 9638c4700d
commit ca2dc1801c
9 changed files with 265 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
View File

@ -21,8 +21,10 @@ package org.elasticsearch.action.admin.indices.template.put;
import org.elasticsearch.ElasticSearchGenerationException;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.master.MasterNodeOperationRequest;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -64,6 +66,8 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest {
private Map<String, String> mappings = newHashMap();
private Map<String, IndexMetaData.Custom> customs = newHashMap();
private TimeValue timeout = new TimeValue(10, TimeUnit.SECONDS);
PutIndexTemplateRequest() {
@ -254,25 +258,35 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest {
*/
public PutIndexTemplateRequest source(Map templateSource) {
Map<String, Object> source = templateSource;
if (source.containsKey("template")) {
template(source.get("template").toString());
}
if (source.containsKey("order")) {
order(XContentMapValues.nodeIntegerValue(source.get("order"), order()));
}
if (source.containsKey("settings")) {
if (!(source.get("settings") instanceof Map)) {
for (Map.Entry<String, Object> entry : source.entrySet()) {
String name = entry.getKey();
if (name.equals("template")) {
template(entry.getValue().toString());
} else if (name.equals("order")) {
order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
} else if (name.equals("settings")) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException("Malformed settings section, should include an inner object");
}
settings((Map<String, Object>) source.get("settings"));
settings((Map<String, Object>) entry.getValue());
} else if (name.equals("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
if (!(entry1.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException("Malformed mappings section for type [" + entry1.getKey() + "], should include an inner object describing the mapping");
}
mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
}
} else {
// maybe custom?
IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name);
if (factory != null) {
try {
customs.put(name, factory.fromMap((Map<String, Object>) entry.getValue()));
} catch (IOException e) {
throw new ElasticSearchParseException("failed to parse custom metadata for [" + name + "]");
}
if (source.containsKey("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
for (Map.Entry<String, Object> entry : mappings.entrySet()) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException("Malformed mappings section for type [" + entry.getKey() + "], should include an inner object describing the mapping");
}
mapping(entry.getKey(), (Map<String, Object>) entry.getValue());
}
}
return this;
@ -308,6 +322,15 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest {
}
public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) {
customs.put(custom.type(), custom);
return this;
}
Map<String, IndexMetaData.Custom> customs() {
return this.customs;
}
/**
* Timeout to wait till the put mapping gets acknowledged of all current cluster nodes. Defaults to
* <tt>10s</tt>.
@ -347,6 +370,12 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest {
for (int i = 0; i < size; i++) {
mappings.put(in.readUTF(), in.readUTF());
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readUTF();
IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in);
customs.put(type, customIndexMetaData);
}
}
@Override
@ -364,5 +393,10 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest {
out.writeUTF(entry.getKey());
out.writeUTF(entry.getValue());
}
out.writeVInt(customs.size());
for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
out.writeUTF(entry.getKey());
IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out);
}
}
}

1
src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
View File

@ -88,6 +88,7 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeOperatio
.order(request.order())
.settings(request.settings())
.mappings(request.mappings())
.customs(request.customs())
.create(request.create()),
new MetaDataIndexTemplateService.PutListener() {

10
src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -57,6 +57,8 @@ public class IndexMetaData {
public interface Custom {
String type();
interface Factory<T extends Custom> {
String type();
@ -65,9 +67,17 @@ public class IndexMetaData {
void writeTo(T customIndexMetaData, StreamOutput out) throws IOException;
T fromMap(Map<String, Object> map) throws IOException;
T fromXContent(XContentParser parser) throws IOException;
void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException;
/**
* Merges from first to second, with first being more important, i.e., if something exists in first and second,
* first will prevail.
*/
T merge(T first, T second);
}
}

62
src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View File

@ -50,12 +50,15 @@ public class IndexTemplateMetaData {
// the mapping source should always include the type as top level
private final ImmutableMap<String, CompressedString> mappings;
public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableMap<String, CompressedString> mappings) {
private final ImmutableMap<String, IndexMetaData.Custom> customs;
public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableMap<String, CompressedString> mappings, ImmutableMap<String, IndexMetaData.Custom> customs) {
this.name = name;
this.order = order;
this.template = template;
this.settings = settings;
this.mappings = mappings;
this.customs = customs;
}
public String name() {
@ -98,6 +101,18 @@ public class IndexTemplateMetaData {
return this.mappings;
}
public ImmutableMap<String, IndexMetaData.Custom> customs() {
return this.customs;
}
public ImmutableMap<String, IndexMetaData.Custom> getCustoms() {
return this.customs;
}
public <T extends IndexMetaData.Custom> T custom(String type) {
return (T) customs.get(type);
}
public static Builder builder(String name) {
return new Builder(name);
}
@ -140,6 +155,8 @@ public class IndexTemplateMetaData {
private MapBuilder<String, CompressedString> mappings = MapBuilder.newMapBuilder();
private MapBuilder<String, IndexMetaData.Custom> customs = MapBuilder.newMapBuilder();
public Builder(String name) {
this.name = name;
}
@ -150,6 +167,7 @@ public class IndexTemplateMetaData {
template(indexTemplateMetaData.template());
settings(indexTemplateMetaData.settings());
mappings.putAll(indexTemplateMetaData.mappings());
customs.putAll(indexTemplateMetaData.customs());
}
public Builder order(int order) {
@ -191,8 +209,22 @@ public class IndexTemplateMetaData {
return this;
}
public Builder putCustom(String type, IndexMetaData.Custom customIndexMetaData) {
this.customs.put(type, customIndexMetaData);
return this;
}
public Builder removeCustom(String type) {
this.customs.remove(type);
return this;
}
public IndexMetaData.Custom getCustom(String type) {
return this.customs.get(type);
}
public IndexTemplateMetaData build() {
return new IndexTemplateMetaData(name, order, template, settings, mappings.immutableMap());
return new IndexTemplateMetaData(name, order, template, settings, mappings.immutableMap(), customs.immutableMap());
}
public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
@ -216,6 +248,12 @@ public class IndexTemplateMetaData {
}
builder.endArray();
for (Map.Entry<String, IndexMetaData.Custom> entry : indexTemplateMetaData.customs().entrySet()) {
builder.startObject(entry.getKey(), XContentBuilder.FieldCaseConversion.NONE);
IndexMetaData.lookupFactorySafe(entry.getKey()).toXContent(entry.getValue(), builder, params);
builder.endObject();
}
builder.endObject();
}
@ -262,6 +300,15 @@ public class IndexTemplateMetaData {
builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string());
}
}
} else {
// check if its a custom index metadata
IndexMetaData.Custom.Factory<IndexMetaData.Custom> factory = IndexMetaData.lookupFactory(currentFieldName);
if (factory == null) {
//TODO warn
parser.skipChildren();
} else {
builder.putCustom(factory.type(), factory.fromXContent(parser));
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
@ -299,6 +346,12 @@ public class IndexTemplateMetaData {
for (int i = 0; i < mappingsSize; i++) {
builder.putMapping(in.readUTF(), CompressedString.readCompressedString(in));
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readUTF();
IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in);
builder.putCustom(type, customIndexMetaData);
}
return builder.build();
}
@ -312,6 +365,11 @@ public class IndexTemplateMetaData {
out.writeUTF(entry.getKey());
entry.getValue().writeTo(out);
}
out.writeVInt(indexTemplateMetaData.customs().size());
for (Map.Entry<String, IndexMetaData.Custom> entry : indexTemplateMetaData.customs().entrySet()) {
out.writeUTF(entry.getKey());
IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out);
}
}
}

19
src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -146,12 +146,16 @@ public class MetaDataCreateIndexService extends AbstractComponent {
// find templates, highest order are better matching
List<IndexTemplateMetaData> templates = findTemplates(request, currentState);
Map<String, Custom> customs = Maps.newHashMap();
// add the request mapping
Map<String, Map<String, Object>> mappings = Maps.newHashMap();
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
}
// TODO: request should be able to add custom metadata
// apply templates, merging the mappings into the request mapping if exists
for (IndexTemplateMetaData template : templates) {
for (Map.Entry<String, CompressedString> entry : template.mappings().entrySet()) {
@ -161,6 +165,18 @@ public class MetaDataCreateIndexService extends AbstractComponent {
mappings.put(entry.getKey(), parseMapping(entry.getValue().string()));
}
}
// handle custom
for (Map.Entry<String, Custom> customEntry : template.customs().entrySet()) {
String type = customEntry.getKey();
IndexMetaData.Custom custom = customEntry.getValue();
IndexMetaData.Custom existing = customs.get(type);
if (existing == null) {
customs.put(type, custom);
} else {
IndexMetaData.Custom merged = IndexMetaData.lookupFactorySafe(type).merge(existing, custom);
customs.put(type, merged);
}
}
}
// now add config level mappings
@ -259,6 +275,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
for (MappingMetaData mappingMd : mappingsMetaData.values()) {
indexMetaDataBuilder.putMapping(mappingMd);
}
for (Map.Entry<String, Custom> customEntry : customs.entrySet()) {
indexMetaDataBuilder.putCustom(customEntry.getKey(), customEntry.getValue());
}
indexMetaDataBuilder.state(request.state);
final IndexMetaData indexMetaData = indexMetaDataBuilder.build();

9
src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View File

@ -106,6 +106,9 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
}
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
templateBuilder.putCustom(entry.getKey(), entry.getValue());
}
} catch (Exception e) {
listener.onFailure(e);
return;
@ -180,6 +183,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
String template;
Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS;
Map<String, String> mappings = Maps.newHashMap();
Map<String, IndexMetaData.Custom> customs = Maps.newHashMap();
public PutRequest(String cause, String name) {
this.cause = cause;
@ -211,6 +215,11 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
return this;
}
public PutRequest customs(Map<String, IndexMetaData.Custom> customs) {
this.customs.putAll(customs);
return this;
}
public PutRequest putMapping(String mappingType, String mappingSource) {
mappings.put(mappingType, mappingSource);
return this;

47
src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java
View File

@ -20,16 +20,14 @@
package org.elasticsearch.search.warmer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.BytesHolder;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.*;
import java.io.IOException;
import java.util.ArrayList;
@ -84,6 +82,11 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
return this.entries;
}
@Override
public String type() {
return TYPE;
}
public static class Factory implements IndexMetaData.Custom.Factory<IndexWarmersMetaData> {
@Override
@ -115,6 +118,23 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
}
}
@Override
public IndexWarmersMetaData fromMap(Map<String, Object> map) throws IOException {
// if it starts with the type, remove it
if (map.size() == 1 && map.containsKey(TYPE)) {
map = (Map<String, Object>) map.values().iterator().next();
}
XContentBuilder builder = XContentFactory.smileBuilder().map(map);
XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.underlyingBytes(), 0, builder.underlyingBytesLength());
try {
// move to START_OBJECT
parser.nextToken();
return fromXContent(parser);
} finally {
parser.close();
}
}
@Override
public IndexWarmersMetaData fromXContent(XContentParser parser) throws IOException {
// we get here after we are at warmers token
@ -178,5 +198,24 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
}
builder.endObject();
}
@Override
public IndexWarmersMetaData merge(IndexWarmersMetaData first, IndexWarmersMetaData second) {
List<Entry> entries = Lists.newArrayList();
entries.addAll(first.entries());
for (Entry secondEntry : second.entries()) {
boolean found = false;
for (Entry firstEntry : first.entries()) {
if (firstEntry.name().equals(secondEntry.name())) {
found = true;
break;
}
}
if (!found) {
entries.add(secondEntry);
}
}
return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()]));
}
}
}

31
src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java
View File

@ -82,12 +82,34 @@ public class LocalGatewayIndicesWarmerTests extends AbstractNodesTests {
.setSearchRequest(client("node1").prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "value2")))
.execute().actionGet();
logger.info("--> put template with warmer");
client("node1").admin().indices().preparePutTemplate("template_1")
.setSource("{\n" +
" \"template\" : \"xxx\",\n" +
" \"warmers\" : {\n" +
" \"warmer_1\" : {\n" +
" \"types\" : [],\n" +
" \"source\" : {\n" +
" \"query\" : {\n" +
" \"match_all\" : {}\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}")
.execute().actionGet();
logger.info("--> verify warmers are registered in cluster state");
ClusterState clusterState = client("node1").admin().cluster().prepareState().execute().actionGet().state();
IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE);
assertThat(warmersMetaData, Matchers.notNullValue());
assertThat(warmersMetaData.entries().size(), equalTo(2));
IndexWarmersMetaData templateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE);
assertThat(templateWarmers, Matchers.notNullValue());
assertThat(templateWarmers.entries().size(), equalTo(1));
logger.info("--> close the node");
closeNode("node1");
@ -106,6 +128,15 @@ public class LocalGatewayIndicesWarmerTests extends AbstractNodesTests {
assertThat(recoveredWarmersMetaData.entries().get(i).source(), equalTo(warmersMetaData.entries().get(i).source()));
}
logger.info("--> verify warmers in template are recovered");
IndexWarmersMetaData recoveredTemplateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE);
assertThat(recoveredTemplateWarmers.entries().size(), equalTo(templateWarmers.entries().size()));
for (int i = 0; i < templateWarmers.entries().size(); i++) {
assertThat(recoveredTemplateWarmers.entries().get(i).name(), equalTo(templateWarmers.entries().get(i).name()));
assertThat(recoveredTemplateWarmers.entries().get(i).source(), equalTo(templateWarmers.entries().get(i).source()));
}
logger.info("--> delete warmer warmer_1");
client("node1").admin().indices().prepareDeleteWarmer().setIndices("test").setName("warmer_1").execute().actionGet();

41
src/test/java/org/elasticsearch/test/integration/indices/wamer/SimpleIndicesWarmerTests.java
View File

@ -20,13 +20,19 @@
package org.elasticsearch.test.integration.indices.wamer;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.warmer.IndexWarmersMetaData;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.hamcrest.Matchers;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*/
public class SimpleIndicesWarmerTests extends AbstractNodesTests {
@ -70,4 +76,39 @@ public class SimpleIndicesWarmerTests extends AbstractNodesTests {
client.prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet();
}
@Test
public void templateWarmer() {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().preparePutTemplate("template_1")
.setSource("{\n" +
" \"template\" : \"*\",\n" +
" \"warmers\" : {\n" +
" \"warmer_1\" : {\n" +
" \"types\" : [],\n" +
" \"source\" : {\n" +
" \"query\" : {\n" +
" \"match_all\" : {}\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}")
.execute().actionGet();
client.admin().indices().prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
.execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
ClusterState clusterState = client.admin().cluster().prepareState().execute().actionGet().state();
IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE);
assertThat(warmersMetaData, Matchers.notNullValue());
assertThat(warmersMetaData.entries().size(), equalTo(1));
client.prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet();
}
}