by default, index metadata to be stored in smile format and store binary format mapping and alias filter to improve the cost it takes to persist them

src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java

@@ -139,6 +139,11 @@ public class AliasMetaData {
             return alias;
         }
 
+        public Builder filter(CompressedString filter) {
+            this.filter = filter;
+            return this;
+        }
+
         public Builder filter(String filter) {
             if (!Strings.hasLength(filter)) {
                 this.filter = null;
@@ -201,12 +206,18 @@ public class AliasMetaData {
         public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.startObject(aliasMetaData.alias(), XContentBuilder.FieldCaseConversion.NONE);
 
+            boolean binary = params.paramAsBoolean("binary", false);
+
             if (aliasMetaData.filter() != null) {
-                byte[] data = aliasMetaData.filter().uncompressed();
+                if (binary) {
-                XContentParser parser = XContentFactory.xContent(data).createParser(data);
+                    builder.field("filter", aliasMetaData.filter.compressed());
-                Map<String, Object> filter = parser.mapOrdered();
+                } else {
-                parser.close();
+                    byte[] data = aliasMetaData.filter().uncompressed();
-                builder.field("filter", filter);
+                    XContentParser parser = XContentFactory.xContent(data).createParser(data);
+                    Map<String, Object> filter = parser.mapOrdered();
+                    parser.close();
+                    builder.field("filter", filter);
+                }
             }
             if (aliasMetaData.indexRouting() != null) {
                 builder.field("index_routing", aliasMetaData.indexRouting());
@@ -235,6 +246,10 @@ public class AliasMetaData {
                         Map<String, Object> filter = parser.mapOrdered();
                         builder.filter(filter);
                     }
+                } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                    if ("filter".equals(currentFieldName)) {
+                        builder.filter(new CompressedString(parser.binaryValue()));
+                    }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
                     if ("routing".equals(currentFieldName)) {
                         builder.routing(parser.text());

src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Preconditions;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.compress.CompressedString;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
@@ -346,7 +347,7 @@ public class IndexMetaData {
         public Builder putMapping(String type, String source) throws IOException {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             try {
-                putMapping(new MappingMetaData(type, parser.map()));
+                putMapping(new MappingMetaData(type, parser.mapOrdered()));
             } finally {
                 parser.close();
             }
@@ -413,6 +414,8 @@ public class IndexMetaData {
             builder.field("version", indexMetaData.version());
             builder.field("state", indexMetaData.state().toString().toLowerCase());
 
+            boolean binary = params.paramAsBoolean("binary", false);
+
             builder.startObject("settings");
             for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
                 builder.field(entry.getKey(), entry.getValue());
@@ -421,11 +424,15 @@ public class IndexMetaData {
 
             builder.startArray("mappings");
             for (Map.Entry<String, MappingMetaData> entry : indexMetaData.mappings().entrySet()) {
-                byte[] data = entry.getValue().source().uncompressed();
+                if (binary) {
-                XContentParser parser = XContentFactory.xContent(data).createParser(data);
+                    builder.value(entry.getValue().source().compressed());
-                Map<String, Object> mapping = parser.mapOrdered();
+                } else {
-                parser.close();
+                    byte[] data = entry.getValue().source().uncompressed();
-                builder.map(mapping);
+                    XContentParser parser = XContentFactory.xContent(data).createParser(data);
+                    Map<String, Object> mapping = parser.mapOrdered();
+                    parser.close();
+                    builder.map(mapping);
+                }
             }
             builder.endArray();
 
@@ -462,10 +469,14 @@ public class IndexMetaData {
                         builder.settings(settingsBuilder.build());
                     } else if ("mappings".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            Map<String, Object> mapping = parser.mapOrdered();
+                            if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-                            if (mapping.size() == 1) {
+                                builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
-                                String mappingType = mapping.keySet().iterator().next();
+                            } else {
-                                builder.putMapping(new MappingMetaData(mappingType, mapping));
+                                Map<String, Object> mapping = parser.mapOrdered();
+                                if (mapping.size() == 1) {
+                                    String mappingType = mapping.keySet().iterator().next();
+                                    builder.putMapping(new MappingMetaData(mappingType, mapping));
+                                }
                             }
                         }
                     } else if ("aliases".equals(currentFieldName)) {
@@ -473,6 +484,20 @@ public class IndexMetaData {
                             builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
                         }
                     }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if ("mappings".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                                builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
+                            } else {
+                                Map<String, Object> mapping = parser.mapOrdered();
+                                if (mapping.size() == 1) {
+                                    String mappingType = mapping.keySet().iterator().next();
+                                    builder.putMapping(new MappingMetaData(mappingType, mapping));
+                                }
+                            }
+                        }
+                    }
                 } else if (token.isValue()) {
                     if ("state".equals(currentFieldName)) {
                         builder.state(State.fromString(parser.text()));

src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -250,9 +251,9 @@ public class MappingMetaData {
 
     private final CompressedString source;
 
-    private final Id id;
+    private Id id;
-    private final Routing routing;
+    private Routing routing;
-    private final Timestamp timestamp;
+    private Timestamp timestamp;
 
     public MappingMetaData(DocumentMapper docMapper) {
         this.type = docMapper.type();
@@ -262,6 +263,20 @@ public class MappingMetaData {
         this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
     }
 
+    public MappingMetaData(CompressedString mapping) throws IOException {
+        this.source = mapping;
+        Map<String, Object> mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose();
+        if (mappingMap.size() != 1) {
+            throw new ElasticSearchIllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
+        }
+        this.type = mappingMap.keySet().iterator().next();
+        initMappers((Map<String, Object>) mappingMap.get(this.type));
+    }
+
+    public MappingMetaData(Map<String, Object> mapping) throws IOException {
+        this(mapping.keySet().iterator().next(), mapping);
+    }
+
     public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
         this.type = type;
         this.source = new CompressedString(XContentFactory.jsonBuilder().map(mapping).string());
@@ -269,6 +284,10 @@ public class MappingMetaData {
         if (mapping.size() == 1 && mapping.containsKey(type)) {
             withoutType = (Map<String, Object>) mapping.get(type);
         }
+        initMappers(withoutType);
+    }
+
+    private void initMappers(Map<String, Object> withoutType) {
         if (withoutType.containsKey("_id")) {
             String path = null;
             Map<String, Object> routingNode = (Map<String, Object>) withoutType.get("_id");

src/main/java/org/elasticsearch/common/compress/CompressedString.java

@@ -40,6 +40,10 @@ public class CompressedString implements Streamable {
     CompressedString() {
     }
 
+    public CompressedString(byte[] compressed) {
+        this.bytes = compressed;
+    }
+
     public CompressedString(String str) throws IOException {
         UnicodeUtil.UTF8Result result = Unicode.unsafeFromStringAsUtf8(str);
         this.bytes = LZFEncoder.encode(result.result, result.length);

src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.gateway.local.state.meta;
 
+import com.google.common.collect.Maps;
 import com.google.common.io.Closeables;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -42,6 +43,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -53,13 +55,24 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
 
     private volatile MetaData currentMetaData;
 
+    private final XContentType format;
+    private final ToXContent.Params formatParams;
+
     @Inject
     public LocalGatewayMetaState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayMetaState nodesListGatewayMetaState) throws Exception {
         super(settings);
         this.nodeEnv = nodeEnv;
-
+        this.format = XContentType.fromRestContentType(settings.get("format", "smile"));
         nodesListGatewayMetaState.init(this);
 
+        if (this.format == XContentType.SMILE) {
+            Map<String, String> params = Maps.newHashMap();
+            params.put("binary", "true");
+            formatParams = new ToXContent.MapParams(params);
+        } else {
+            formatParams = ToXContent.EMPTY_PARAMS;
+        }
+
         try {
             pre019Upgrade();
             long start = System.currentTimeMillis();
@@ -150,9 +163,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
         logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason);
         CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry();
         try {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes());
+            XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes());
             builder.startObject();
-            IndexMetaData.Builder.toXContent(indexMetaData, builder, ToXContent.EMPTY_PARAMS);
+            IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams);
             builder.endObject();
             builder.flush();
 
@@ -201,9 +214,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
 
         CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry();
         try {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes());
+            XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes());
             builder.startObject();
-            MetaData.Builder.toXContent(globalMetaData, builder, ToXContent.EMPTY_PARAMS);
+            MetaData.Builder.toXContent(globalMetaData, builder, formatParams);
             builder.endObject();
             builder.flush();
 

src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.gateway.Gateway;
+import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalNode;
@@ -295,6 +296,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
                         .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
                         .endObject().endObject().endObject())
                 .execute().actionGet();
+        node2.client().admin().indices().prepareAliases().addAlias("test", "test_alias", FilterBuilders.termFilter("field", "value")).execute().actionGet();
 
 
         logger.info("--> closing the second node");
@@ -318,6 +320,8 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
         ClusterState state = node1.client().admin().cluster().prepareState().execute().actionGet().state();
         assertThat(state.metaData().index("test").mapping("type2"), notNullValue());
         assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*"));
+        assertThat(state.metaData().index("test").aliases().get("test_alias"), notNullValue());
+        assertThat(state.metaData().index("test").aliases().get("test_alias").filter(), notNullValue());
     }
 
     @Test