honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Gateway: Store cluster meta data in JSON (and not binary), closes #36.

This commit is contained in:
kimchy 2010-02-22 20:08:33 +02:00
parent 575250e223
commit 67d86de7ea
5 changed files with 253 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -20,9 +20,14 @@
package org.elasticsearch.cluster.metadata;
import com.google.common.collect.ImmutableMap;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
import org.elasticsearch.util.MapBuilder;
import org.elasticsearch.util.Nullable;
import org.elasticsearch.util.Preconditions;
import org.elasticsearch.util.concurrent.Immutable;
import org.elasticsearch.util.json.JsonBuilder;
import org.elasticsearch.util.json.ToJson;
import org.elasticsearch.util.settings.ImmutableSettings;
import org.elasticsearch.util.settings.Settings;
@ -52,8 +57,8 @@ public class IndexMetaData {
private transient final int totalNumberOfShards;
private IndexMetaData(String index, Settings settings, ImmutableMap<String, String> mappings) {
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards");
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas");
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]");
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]");
this.index = index;
this.settings = settings;
this.mappings = mappings;
@ -115,7 +120,7 @@ public class IndexMetaData {
}
public Builder numberOfShards(int numberOfShards) {
settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
return this;
}
@ -124,7 +129,7 @@ public class IndexMetaData {
}
public Builder numberOfReplicas(int numberOfReplicas) {
settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
return this;
}
@ -132,6 +137,11 @@ public class IndexMetaData {
return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
}
public Builder settings(Settings.Builder settings) {
this.settings = settings.build();
return this;
}
public Builder settings(Settings settings) {
this.settings = settings;
return this;
@ -151,6 +161,68 @@ public class IndexMetaData {
return new IndexMetaData(index, settings, mappings.immutableMap());
}
public static void toJson(IndexMetaData indexMetaData, JsonBuilder builder, ToJson.Params params) throws IOException {
builder.startObject(indexMetaData.index());
builder.startObject("settings");
for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
builder.startObject("mappings");
for (Map.Entry<String, String> entry : indexMetaData.mappings().entrySet()) {
builder.startObject(entry.getKey());
builder.field("source", entry.getValue());
builder.endObject();
}
builder.endObject();
builder.endObject();
}
public static IndexMetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
Builder builder = new Builder(jp.getCurrentName());
String currentFieldName = null;
JsonToken token = jp.nextToken();
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
if (token == JsonToken.FIELD_NAME) {
currentFieldName = jp.getCurrentName();
} else if (token == JsonToken.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
ImmutableSettings.Builder settingsBuilder = settingsBuilder().globalSettings(globalSettings);
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
String key = jp.getCurrentName();
token = jp.nextToken();
String value = jp.getText();
settingsBuilder.put(key, value);
}
builder.settings(settingsBuilder.build());
} else if ("mappings".equals(currentFieldName)) {
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
String mappingType = jp.getCurrentName();
String mappingSource = null;
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
if (token == JsonToken.FIELD_NAME) {
if ("source".equals(jp.getCurrentName())) {
jp.nextToken();
mappingSource = jp.getText();
}
}
}
if (mappingSource == null) {
// crap, no mapping source, warn?
} else {
builder.putMapping(mappingType, mappingSource);
}
}
}
}
}
return builder.build();
}
public static IndexMetaData readFrom(DataInput in, Settings globalSettings) throws ClassNotFoundException, IOException {
Builder builder = new Builder(in.readUTF());
builder.settings(readSettingsFromStream(in, globalSettings));

49
modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View File

@ -21,9 +21,13 @@ package org.elasticsearch.cluster.metadata;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.UnmodifiableIterator;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
import org.elasticsearch.util.MapBuilder;
import org.elasticsearch.util.Nullable;
import org.elasticsearch.util.concurrent.Immutable;
import org.elasticsearch.util.json.JsonBuilder;
import org.elasticsearch.util.json.ToJson;
import org.elasticsearch.util.settings.Settings;
import java.io.DataInput;
@ -31,6 +35,7 @@ import java.io.DataOutput;
import java.io.IOException;
import static org.elasticsearch.util.MapBuilder.*;
import static org.elasticsearch.util.json.JsonBuilder.*;
/**
* @author kimchy (Shay Banon)
@ -120,6 +125,50 @@ public class MetaData implements Iterable<IndexMetaData> {
return new MetaData(indices.immutableMap(), maxNumberOfShardsPerNode);
}
public static String toJson(MetaData metaData) throws IOException {
JsonBuilder builder = jsonBuilder().prettyPrint();
builder.startObject();
toJson(metaData, builder, ToJson.EMPTY_PARAMS);
builder.endObject();
return builder.string();
}
public static void toJson(MetaData metaData, JsonBuilder builder, ToJson.Params params) throws IOException {
builder.startObject("meta-data");
builder.field("maxNumberOfShardsPerNode", metaData.maxNumberOfShardsPerNode());
builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData) {
IndexMetaData.Builder.toJson(indexMetaData, builder, params);
}
builder.endObject();
builder.endObject();
}
public static MetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
Builder builder = new Builder();
String currentFieldName = null;
JsonToken token = jp.nextToken();
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
if (token == JsonToken.FIELD_NAME) {
currentFieldName = jp.getCurrentName();
} else if (token == JsonToken.START_OBJECT) {
if ("indices".equals(currentFieldName)) {
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
builder.put(IndexMetaData.Builder.fromJson(jp, globalSettings));
}
}
} else if (token == JsonToken.VALUE_NUMBER_INT) {
if ("maxNumberOfShardsPerNode".equals(currentFieldName)) {
builder.maxNumberOfShardsPerNode(jp.getIntValue());
}
}
}
return builder.build();
}
public static MetaData readFrom(DataInput in, @Nullable Settings globalSettings) throws IOException, ClassNotFoundException {
Builder builder = new Builder();
builder.maxNumberOfShardsPerNode(in.readInt());

27
modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.gateway.fs;
import com.google.inject.Inject;
import com.google.inject.Module;
import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonParser;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.metadata.MetaData;
@ -30,12 +32,16 @@ import org.elasticsearch.gateway.GatewayException;
import org.elasticsearch.index.gateway.fs.FsIndexGatewayModule;
import org.elasticsearch.util.component.AbstractComponent;
import org.elasticsearch.util.component.Lifecycle;
import org.elasticsearch.util.io.FastDataOutputStream;
import org.elasticsearch.util.io.FileSystemUtils;
import org.elasticsearch.util.json.Jackson;
import org.elasticsearch.util.json.JsonBuilder;
import org.elasticsearch.util.json.ToJson;
import org.elasticsearch.util.settings.Settings;
import java.io.*;
import static org.elasticsearch.util.io.FileSystemUtils.*;
/**
* @author kimchy (Shay Banon)
*/
@ -150,13 +156,17 @@ public class FsGateway extends AbstractComponent implements Gateway {
}
FileOutputStream fileStream = new FileOutputStream(file);
FastDataOutputStream outStream = new FastDataOutputStream(fileStream);
MetaData.Builder.writeTo(metaData, outStream);
JsonBuilder builder = new JsonBuilder(Jackson.defaultJsonFactory().createJsonGenerator(fileStream, JsonEncoding.UTF8));
builder.prettyPrint();
builder.startObject();
MetaData.Builder.toJson(metaData, builder, ToJson.EMPTY_PARAMS);
builder.endObject();
builder.close();
outStream.close();
fileStream.close();
FileSystemUtils.syncFile(file);
syncFile(file);
currentIndex++;
@ -187,11 +197,12 @@ public class FsGateway extends AbstractComponent implements Gateway {
}
FileInputStream fileStream = new FileInputStream(file);
DataInputStream inStream = new DataInputStream(fileStream);
MetaData metaData = MetaData.Builder.readFrom(inStream, settings);
JsonParser jp = Jackson.defaultJsonFactory().createJsonParser(fileStream);
MetaData metaData = MetaData.Builder.fromJson(jp, settings);
jp.close();
inStream.close();
fileStream.close();
return metaData;

7
modules/elasticsearch/src/main/java/org/elasticsearch/util/json/JsonBuilder.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.util.json;
import org.apache.lucene.util.UnicodeUtil;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.util.concurrent.NotThreadSafe;
import org.elasticsearch.util.io.FastCharArrayWriter;
@ -94,6 +95,12 @@ public class JsonBuilder {
this.generator = factory.createJsonGenerator(writer);
}
public JsonBuilder(JsonGenerator generator) throws IOException {
this.writer = new FastCharArrayWriter();
this.generator = generator;
this.factory = null;
}
public JsonBuilder prettyPrint() {
generator.useDefaultPrettyPrinter();
return this;

102
modules/elasticsearch/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.util.json.Jackson;
import org.testng.annotations.Test;
import java.io.IOException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
import static org.elasticsearch.cluster.metadata.MetaData.*;
import static org.elasticsearch.util.settings.ImmutableSettings.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
@Test
public class ToAndFromJsonMetaDataTests {
@Test
public void testSimpleJsonFromAndTo() throws IOException {
MetaData metaData = newMetaDataBuilder()
.maxNumberOfShardsPerNode(2)
.put(newIndexMetaDataBuilder("test1")
.numberOfShards(1)
.numberOfReplicas(2))
.put(newIndexMetaDataBuilder("test2")
.settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
.numberOfShards(2)
.numberOfReplicas(3))
.put(newIndexMetaDataBuilder("test3")
.numberOfShards(1)
.numberOfReplicas(2)
.putMapping("mapping1", MAPPING_SOURCE1))
.put(newIndexMetaDataBuilder("test4")
.settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
.numberOfShards(1)
.numberOfReplicas(2)
.putMapping("mapping1", MAPPING_SOURCE1)
.putMapping("mapping2", MAPPING_SOURCE2))
.build();
String metaDataSource = MetaData.Builder.toJson(metaData);
System.out.println("ToJson: " + metaDataSource);
MetaData parsedMetaData = MetaData.Builder.fromJson(Jackson.defaultJsonFactory().createJsonParser(metaDataSource), null);
assertThat(parsedMetaData.maxNumberOfShardsPerNode(), equalTo(2));
IndexMetaData indexMetaData = metaData.index("test1");
assertThat(indexMetaData.numberOfShards(), equalTo(1));
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
assertThat(indexMetaData.mappings().size(), equalTo(0));
indexMetaData = metaData.index("test2");
assertThat(indexMetaData.numberOfShards(), equalTo(2));
assertThat(indexMetaData.numberOfReplicas(), equalTo(3));
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
assertThat(indexMetaData.mappings().size(), equalTo(0));
indexMetaData = metaData.index("test3");
assertThat(indexMetaData.numberOfShards(), equalTo(1));
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
assertThat(indexMetaData.mappings().size(), equalTo(1));
assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
indexMetaData = metaData.index("test4");
assertThat(indexMetaData.numberOfShards(), equalTo(1));
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
assertThat(indexMetaData.mappings().size(), equalTo(2));
assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
assertThat(indexMetaData.mappings().get("mapping2"), equalTo(MAPPING_SOURCE2));
}
private static final String MAPPING_SOURCE1 = "{ text1: { type : \"string\" } }";
private static final String MAPPING_SOURCE2 = "{ text2: { type : \"string\" } }";
}