Gateway: Store cluster meta data in JSON (and not binary), closes #36.

modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -20,9 +20,14 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.google.common.collect.ImmutableMap;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.JsonToken;
 import org.elasticsearch.util.MapBuilder;
+import org.elasticsearch.util.Nullable;
 import org.elasticsearch.util.Preconditions;
 import org.elasticsearch.util.concurrent.Immutable;
+import org.elasticsearch.util.json.JsonBuilder;
+import org.elasticsearch.util.json.ToJson;
 import org.elasticsearch.util.settings.ImmutableSettings;
 import org.elasticsearch.util.settings.Settings;
 
@@ -52,8 +57,8 @@ public class IndexMetaData {
     private transient final int totalNumberOfShards;
 
     private IndexMetaData(String index, Settings settings, ImmutableMap<String, String> mappings) {
-        Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards");
+        Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]");
-        Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas");
+        Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]");
         this.index = index;
         this.settings = settings;
         this.mappings = mappings;
@@ -115,7 +120,7 @@ public class IndexMetaData {
         }
 
         public Builder numberOfShards(int numberOfShards) {
-            settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
+            settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
             return this;
         }
 
@@ -124,7 +129,7 @@ public class IndexMetaData {
         }
 
         public Builder numberOfReplicas(int numberOfReplicas) {
-            settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
+            settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
             return this;
         }
 
@@ -132,6 +137,11 @@ public class IndexMetaData {
             return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
         }
 
+        public Builder settings(Settings.Builder settings) {
+            this.settings = settings.build();
+            return this;
+        }
+
         public Builder settings(Settings settings) {
             this.settings = settings;
             return this;
@@ -151,6 +161,68 @@ public class IndexMetaData {
             return new IndexMetaData(index, settings, mappings.immutableMap());
         }
 
+        public static void toJson(IndexMetaData indexMetaData, JsonBuilder builder, ToJson.Params params) throws IOException {
+            builder.startObject(indexMetaData.index());
+
+            builder.startObject("settings");
+            for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
+                builder.field(entry.getKey(), entry.getValue());
+            }
+            builder.endObject();
+
+            builder.startObject("mappings");
+            for (Map.Entry<String, String> entry : indexMetaData.mappings().entrySet()) {
+                builder.startObject(entry.getKey());
+                builder.field("source", entry.getValue());
+                builder.endObject();
+            }
+            builder.endObject();
+
+            builder.endObject();
+        }
+
+        public static IndexMetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
+            Builder builder = new Builder(jp.getCurrentName());
+
+            String currentFieldName = null;
+            JsonToken token = jp.nextToken();
+            while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                if (token == JsonToken.FIELD_NAME) {
+                    currentFieldName = jp.getCurrentName();
+                } else if (token == JsonToken.START_OBJECT) {
+                    if ("settings".equals(currentFieldName)) {
+                        ImmutableSettings.Builder settingsBuilder = settingsBuilder().globalSettings(globalSettings);
+                        while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                            String key = jp.getCurrentName();
+                            token = jp.nextToken();
+                            String value = jp.getText();
+                            settingsBuilder.put(key, value);
+                        }
+                        builder.settings(settingsBuilder.build());
+                    } else if ("mappings".equals(currentFieldName)) {
+                        while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                            String mappingType = jp.getCurrentName();
+                            String mappingSource = null;
+                            while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                                if (token == JsonToken.FIELD_NAME) {
+                                    if ("source".equals(jp.getCurrentName())) {
+                                        jp.nextToken();
+                                        mappingSource = jp.getText();
+                                    }
+                                }
+                            }
+                            if (mappingSource == null) {
+                                // crap, no mapping source, warn?
+                            } else {
+                                builder.putMapping(mappingType, mappingSource);
+                            }
+                        }
+                    }
+                }
+            }
+            return builder.build();
+        }
+
         public static IndexMetaData readFrom(DataInput in, Settings globalSettings) throws ClassNotFoundException, IOException {
             Builder builder = new Builder(in.readUTF());
             builder.settings(readSettingsFromStream(in, globalSettings));

modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java

@@ -21,9 +21,13 @@ package org.elasticsearch.cluster.metadata;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.UnmodifiableIterator;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.JsonToken;
 import org.elasticsearch.util.MapBuilder;
 import org.elasticsearch.util.Nullable;
 import org.elasticsearch.util.concurrent.Immutable;
+import org.elasticsearch.util.json.JsonBuilder;
+import org.elasticsearch.util.json.ToJson;
 import org.elasticsearch.util.settings.Settings;
 
 import java.io.DataInput;
@@ -31,6 +35,7 @@ import java.io.DataOutput;
 import java.io.IOException;
 
 import static org.elasticsearch.util.MapBuilder.*;
+import static org.elasticsearch.util.json.JsonBuilder.*;
 
 /**
  * @author kimchy (Shay Banon)
@@ -120,6 +125,50 @@ public class MetaData implements Iterable<IndexMetaData> {
             return new MetaData(indices.immutableMap(), maxNumberOfShardsPerNode);
         }
 
+        public static String toJson(MetaData metaData) throws IOException {
+            JsonBuilder builder = jsonBuilder().prettyPrint();
+            builder.startObject();
+            toJson(metaData, builder, ToJson.EMPTY_PARAMS);
+            builder.endObject();
+            return builder.string();
+        }
+
+        public static void toJson(MetaData metaData, JsonBuilder builder, ToJson.Params params) throws IOException {
+            builder.startObject("meta-data");
+            builder.field("maxNumberOfShardsPerNode", metaData.maxNumberOfShardsPerNode());
+
+            builder.startObject("indices");
+            for (IndexMetaData indexMetaData : metaData) {
+                IndexMetaData.Builder.toJson(indexMetaData, builder, params);
+            }
+            builder.endObject();
+
+            builder.endObject();
+        }
+
+        public static MetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
+            Builder builder = new Builder();
+
+            String currentFieldName = null;
+            JsonToken token = jp.nextToken();
+            while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                if (token == JsonToken.FIELD_NAME) {
+                    currentFieldName = jp.getCurrentName();
+                } else if (token == JsonToken.START_OBJECT) {
+                    if ("indices".equals(currentFieldName)) {
+                        while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
+                            builder.put(IndexMetaData.Builder.fromJson(jp, globalSettings));
+                        }
+                    }
+                } else if (token == JsonToken.VALUE_NUMBER_INT) {
+                    if ("maxNumberOfShardsPerNode".equals(currentFieldName)) {
+                        builder.maxNumberOfShardsPerNode(jp.getIntValue());
+                    }
+                }
+            }
+            return builder.build();
+        }
+
         public static MetaData readFrom(DataInput in, @Nullable Settings globalSettings) throws IOException, ClassNotFoundException {
             Builder builder = new Builder();
             builder.maxNumberOfShardsPerNode(in.readInt());

modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java

@@ -21,6 +21,8 @@ package org.elasticsearch.gateway.fs;
 
 import com.google.inject.Inject;
 import com.google.inject.Module;
+import org.codehaus.jackson.JsonEncoding;
+import org.codehaus.jackson.JsonParser;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -30,12 +32,16 @@ import org.elasticsearch.gateway.GatewayException;
 import org.elasticsearch.index.gateway.fs.FsIndexGatewayModule;
 import org.elasticsearch.util.component.AbstractComponent;
 import org.elasticsearch.util.component.Lifecycle;
-import org.elasticsearch.util.io.FastDataOutputStream;
 import org.elasticsearch.util.io.FileSystemUtils;
+import org.elasticsearch.util.json.Jackson;
+import org.elasticsearch.util.json.JsonBuilder;
+import org.elasticsearch.util.json.ToJson;
 import org.elasticsearch.util.settings.Settings;
 
 import java.io.*;
 
+import static org.elasticsearch.util.io.FileSystemUtils.*;
+
 /**
  * @author kimchy (Shay Banon)
  */
@@ -150,13 +156,17 @@ public class FsGateway extends AbstractComponent implements Gateway {
             }
 
             FileOutputStream fileStream = new FileOutputStream(file);
-            FastDataOutputStream outStream = new FastDataOutputStream(fileStream);
 
-            MetaData.Builder.writeTo(metaData, outStream);
+            JsonBuilder builder = new JsonBuilder(Jackson.defaultJsonFactory().createJsonGenerator(fileStream, JsonEncoding.UTF8));
+            builder.prettyPrint();
+            builder.startObject();
+            MetaData.Builder.toJson(metaData, builder, ToJson.EMPTY_PARAMS);
+            builder.endObject();
+            builder.close();
 
-            outStream.close();
+            fileStream.close();
 
-            FileSystemUtils.syncFile(file);
+            syncFile(file);
 
             currentIndex++;
 
@@ -187,11 +197,12 @@ public class FsGateway extends AbstractComponent implements Gateway {
             }
 
             FileInputStream fileStream = new FileInputStream(file);
-            DataInputStream inStream = new DataInputStream(fileStream);
 
-            MetaData metaData = MetaData.Builder.readFrom(inStream, settings);
+            JsonParser jp = Jackson.defaultJsonFactory().createJsonParser(fileStream);
+            MetaData metaData = MetaData.Builder.fromJson(jp, settings);
+            jp.close();
 
-            inStream.close();
+            fileStream.close();
 
             return metaData;
 

modules/elasticsearch/src/main/java/org/elasticsearch/util/json/JsonBuilder.java

@@ -21,6 +21,7 @@ package org.elasticsearch.util.json;
 
 import org.apache.lucene.util.UnicodeUtil;
 import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.util.concurrent.NotThreadSafe;
 import org.elasticsearch.util.io.FastCharArrayWriter;
@@ -94,6 +95,12 @@ public class JsonBuilder {
         this.generator = factory.createJsonGenerator(writer);
     }
 
+    public JsonBuilder(JsonGenerator generator) throws IOException {
+        this.writer = new FastCharArrayWriter();
+        this.generator = generator;
+        this.factory = null;
+    }
+
     public JsonBuilder prettyPrint() {
         generator.useDefaultPrettyPrinter();
         return this;

modules/elasticsearch/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.util.json.Jackson;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
+import static org.elasticsearch.cluster.metadata.MetaData.*;
+import static org.elasticsearch.util.settings.ImmutableSettings.*;
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+@Test
+public class ToAndFromJsonMetaDataTests {
+
+    @Test
+    public void testSimpleJsonFromAndTo() throws IOException {
+        MetaData metaData = newMetaDataBuilder()
+                .maxNumberOfShardsPerNode(2)
+                .put(newIndexMetaDataBuilder("test1")
+                        .numberOfShards(1)
+                        .numberOfReplicas(2))
+                .put(newIndexMetaDataBuilder("test2")
+                        .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
+                        .numberOfShards(2)
+                        .numberOfReplicas(3))
+                .put(newIndexMetaDataBuilder("test3")
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .putMapping("mapping1", MAPPING_SOURCE1))
+                .put(newIndexMetaDataBuilder("test4")
+                        .settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
+                        .numberOfShards(1)
+                        .numberOfReplicas(2)
+                        .putMapping("mapping1", MAPPING_SOURCE1)
+                        .putMapping("mapping2", MAPPING_SOURCE2))
+                .build();
+
+        String metaDataSource = MetaData.Builder.toJson(metaData);
+        System.out.println("ToJson: " + metaDataSource);
+
+        MetaData parsedMetaData = MetaData.Builder.fromJson(Jackson.defaultJsonFactory().createJsonParser(metaDataSource), null);
+        assertThat(parsedMetaData.maxNumberOfShardsPerNode(), equalTo(2));
+
+        IndexMetaData indexMetaData = metaData.index("test1");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().size(), equalTo(0));
+
+        indexMetaData = metaData.index("test2");
+        assertThat(indexMetaData.numberOfShards(), equalTo(2));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(3));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
+        assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
+        assertThat(indexMetaData.mappings().size(), equalTo(0));
+
+        indexMetaData = metaData.index("test3");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().size(), equalTo(1));
+        assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
+
+        indexMetaData = metaData.index("test4");
+        assertThat(indexMetaData.numberOfShards(), equalTo(1));
+        assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
+        assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
+        assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
+        assertThat(indexMetaData.mappings().size(), equalTo(2));
+        assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
+        assertThat(indexMetaData.mappings().get("mapping2"), equalTo(MAPPING_SOURCE2));
+    }
+
+    private static final String MAPPING_SOURCE1 = "{ text1: { type : \"string\" } }";
+    private static final String MAPPING_SOURCE2 = "{ text2: { type : \"string\" } }";
+}