diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 728e5a5d593..f0f890b6813 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -139,6 +139,11 @@ public class AliasMetaData { return alias; } + public Builder filter(CompressedString filter) { + this.filter = filter; + return this; + } + public Builder filter(String filter) { if (!Strings.hasLength(filter)) { this.filter = null; @@ -201,12 +206,18 @@ public class AliasMetaData { public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(aliasMetaData.alias(), XContentBuilder.FieldCaseConversion.NONE); + boolean binary = params.paramAsBoolean("binary", false); + if (aliasMetaData.filter() != null) { - byte[] data = aliasMetaData.filter().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map filter = parser.mapOrdered(); - parser.close(); - builder.field("filter", filter); + if (binary) { + builder.field("filter", aliasMetaData.filter.compressed()); + } else { + byte[] data = aliasMetaData.filter().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map filter = parser.mapOrdered(); + parser.close(); + builder.field("filter", filter); + } } if (aliasMetaData.indexRouting() != null) { builder.field("index_routing", aliasMetaData.indexRouting()); @@ -235,6 +246,10 @@ public class AliasMetaData { Map filter = parser.mapOrdered(); builder.filter(filter); } + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + if ("filter".equals(currentFieldName)) { + builder.filter(new CompressedString(parser.binaryValue())); + } } else if (token == XContentParser.Token.VALUE_STRING) { if ("routing".equals(currentFieldName)) { builder.routing(parser.text()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 05b096c1eca..bfb707a12be 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeFilters; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; @@ -346,7 +347,7 @@ public class IndexMetaData { public Builder putMapping(String type, String source) throws IOException { XContentParser parser = XContentFactory.xContent(source).createParser(source); try { - putMapping(new MappingMetaData(type, parser.map())); + putMapping(new MappingMetaData(type, parser.mapOrdered())); } finally { parser.close(); } @@ -413,6 +414,8 @@ public class IndexMetaData { builder.field("version", indexMetaData.version()); builder.field("state", indexMetaData.state().toString().toLowerCase()); + boolean binary = params.paramAsBoolean("binary", false); + builder.startObject("settings"); for (Map.Entry entry : indexMetaData.settings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); @@ -421,11 +424,15 @@ public class IndexMetaData { builder.startArray("mappings"); for (Map.Entry entry : indexMetaData.mappings().entrySet()) { - byte[] data = entry.getValue().source().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrdered(); - parser.close(); - builder.map(mapping); + if (binary) { + builder.value(entry.getValue().source().compressed()); + } else { + byte[] data = entry.getValue().source().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrdered(); + parser.close(); + builder.map(mapping); + } } builder.endArray(); @@ -462,10 +469,14 @@ public class IndexMetaData { builder.settings(settingsBuilder.build()); } else if ("mappings".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - Map mapping = parser.mapOrdered(); - if (mapping.size() == 1) { - String mappingType = mapping.keySet().iterator().next(); - builder.putMapping(new MappingMetaData(mappingType, mapping)); + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); + } else { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + builder.putMapping(new MappingMetaData(mappingType, mapping)); + } } } } else if ("aliases".equals(currentFieldName)) { @@ -473,6 +484,20 @@ public class IndexMetaData { builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); } } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("mappings".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); + } else { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + builder.putMapping(new MappingMetaData(mappingType, mapping)); + } + } + } + } } else if (token.isValue()) { if ("state".equals(currentFieldName)) { builder.state(State.fromString(parser.text())); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 7d6e38bf782..15f9aeeeeb4 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -250,9 +251,9 @@ public class MappingMetaData { private final CompressedString source; - private final Id id; - private final Routing routing; - private final Timestamp timestamp; + private Id id; + private Routing routing; + private Timestamp timestamp; public MappingMetaData(DocumentMapper docMapper) { this.type = docMapper.type(); @@ -262,6 +263,20 @@ public class MappingMetaData { this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format()); } + public MappingMetaData(CompressedString mapping) throws IOException { + this.source = mapping; + Map mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose(); + if (mappingMap.size() != 1) { + throw new ElasticSearchIllegalStateException("Can't derive type from mapping, no root type: " + mapping.string()); + } + this.type = mappingMap.keySet().iterator().next(); + initMappers((Map) mappingMap.get(this.type)); + } + + public MappingMetaData(Map mapping) throws IOException { + this(mapping.keySet().iterator().next(), mapping); + } + public MappingMetaData(String type, Map mapping) throws IOException { this.type = type; this.source = new CompressedString(XContentFactory.jsonBuilder().map(mapping).string()); @@ -269,6 +284,10 @@ public class MappingMetaData { if (mapping.size() == 1 && mapping.containsKey(type)) { withoutType = (Map) mapping.get(type); } + initMappers(withoutType); + } + + private void initMappers(Map withoutType) { if (withoutType.containsKey("_id")) { String path = null; Map routingNode = (Map) withoutType.get("_id"); diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedString.java b/src/main/java/org/elasticsearch/common/compress/CompressedString.java index c06dba82569..b6230ade2ba 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedString.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedString.java @@ -40,6 +40,10 @@ public class CompressedString implements Streamable { CompressedString() { } + public CompressedString(byte[] compressed) { + this.bytes = compressed; + } + public CompressedString(String str) throws IOException { UnicodeUtil.UTF8Result result = Unicode.unsafeFromStringAsUtf8(str); this.bytes = LZFEncoder.encode(result.result, result.length); diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index f94d914f6ec..daf32fdf1d5 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -19,6 +19,7 @@ package org.elasticsearch.gateway.local.state.meta; +import com.google.common.collect.Maps; import com.google.common.io.Closeables; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -42,6 +43,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Map; import java.util.Set; /** @@ -53,13 +55,24 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS private volatile MetaData currentMetaData; + private final XContentType format; + private final ToXContent.Params formatParams; + @Inject public LocalGatewayMetaState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayMetaState nodesListGatewayMetaState) throws Exception { super(settings); this.nodeEnv = nodeEnv; - + this.format = XContentType.fromRestContentType(settings.get("format", "smile")); nodesListGatewayMetaState.init(this); + if (this.format == XContentType.SMILE) { + Map params = Maps.newHashMap(); + params.put("binary", "true"); + formatParams = new ToXContent.MapParams(params); + } else { + formatParams = ToXContent.EMPTY_PARAMS; + } + try { pre019Upgrade(); long start = System.currentTimeMillis(); @@ -150,9 +163,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes()); builder.startObject(); - IndexMetaData.Builder.toXContent(indexMetaData, builder, ToXContent.EMPTY_PARAMS); + IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams); builder.endObject(); builder.flush(); @@ -201,9 +214,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes()); builder.startObject(); - MetaData.Builder.toXContent(globalMetaData, builder, ToXContent.EMPTY_PARAMS); + MetaData.Builder.toXContent(globalMetaData, builder, formatParams); builder.endObject(); builder.flush(); diff --git a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java index 343cabfbab3..d4b46b051bc 100644 --- a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java +++ b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.gateway.Gateway; +import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; import org.elasticsearch.node.internal.InternalNode; @@ -295,6 +296,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject() .endObject().endObject().endObject()) .execute().actionGet(); + node2.client().admin().indices().prepareAliases().addAlias("test", "test_alias", FilterBuilders.termFilter("field", "value")).execute().actionGet(); logger.info("--> closing the second node"); @@ -318,6 +320,8 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { ClusterState state = node1.client().admin().cluster().prepareState().execute().actionGet().state(); assertThat(state.metaData().index("test").mapping("type2"), notNullValue()); assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*")); + assertThat(state.metaData().index("test").aliases().get("test_alias"), notNullValue()); + assertThat(state.metaData().index("test").aliases().get("test_alias").filter(), notNullValue()); } @Test