honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

by default, index metadata to be stored in smile format and store binary format mapping and alias filter to improve the cost it takes to persist them

This commit is contained in:
Shay Banon 2012-01-25 11:58:29 +02:00
parent 01d8305af3
commit 68bb5d1434
6 changed files with 103 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
View File

@ -139,6 +139,11 @@ public class AliasMetaData {
return alias;
}
public Builder filter(CompressedString filter) {
this.filter = filter;
return this;
}
public Builder filter(String filter) {
if (!Strings.hasLength(filter)) {
this.filter = null;
@ -201,12 +206,18 @@ public class AliasMetaData {
public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(aliasMetaData.alias(), XContentBuilder.FieldCaseConversion.NONE);
boolean binary = params.paramAsBoolean("binary", false);
if (aliasMetaData.filter() != null) {
byte[] data = aliasMetaData.filter().uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> filter = parser.mapOrdered();
parser.close();
builder.field("filter", filter);
if (binary) {
builder.field("filter", aliasMetaData.filter.compressed());
} else {
byte[] data = aliasMetaData.filter().uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> filter = parser.mapOrdered();
parser.close();
builder.field("filter", filter);
}
}
if (aliasMetaData.indexRouting() != null) {
builder.field("index_routing", aliasMetaData.indexRouting());
@ -235,6 +246,10 @@ public class AliasMetaData {
Map<String, Object> filter = parser.mapOrdered();
builder.filter(filter);
}
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
if ("filter".equals(currentFieldName)) {
builder.filter(new CompressedString(parser.binaryValue()));
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("routing".equals(currentFieldName)) {
builder.routing(parser.text());

45
src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Preconditions;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.regex.Regex;
@ -346,7 +347,7 @@ public class IndexMetaData {
public Builder putMapping(String type, String source) throws IOException {
XContentParser parser = XContentFactory.xContent(source).createParser(source);
try {
putMapping(new MappingMetaData(type, parser.map()));
putMapping(new MappingMetaData(type, parser.mapOrdered()));
} finally {
parser.close();
}
@ -413,6 +414,8 @@ public class IndexMetaData {
builder.field("version", indexMetaData.version());
builder.field("state", indexMetaData.state().toString().toLowerCase());
boolean binary = params.paramAsBoolean("binary", false);
builder.startObject("settings");
for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
@ -421,11 +424,15 @@ public class IndexMetaData {
builder.startArray("mappings");
for (Map.Entry<String, MappingMetaData> entry : indexMetaData.mappings().entrySet()) {
byte[] data = entry.getValue().source().uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> mapping = parser.mapOrdered();
parser.close();
builder.map(mapping);
if (binary) {
builder.value(entry.getValue().source().compressed());
} else {
byte[] data = entry.getValue().source().uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> mapping = parser.mapOrdered();
parser.close();
builder.map(mapping);
}
}
builder.endArray();
@ -462,10 +469,14 @@ public class IndexMetaData {
builder.settings(settingsBuilder.build());
} else if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
}
}
}
} else if ("aliases".equals(currentFieldName)) {
@ -473,6 +484,20 @@ public class IndexMetaData {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
}
}
}
}
} else if (token.isValue()) {
if ("state".equals(currentFieldName)) {
builder.state(State.fromString(parser.text()));

25
src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
@ -250,9 +251,9 @@ public class MappingMetaData {
private final CompressedString source;
private final Id id;
private final Routing routing;
private final Timestamp timestamp;
private Id id;
private Routing routing;
private Timestamp timestamp;
public MappingMetaData(DocumentMapper docMapper) {
this.type = docMapper.type();
@ -262,6 +263,20 @@ public class MappingMetaData {
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
}
public MappingMetaData(CompressedString mapping) throws IOException {
this.source = mapping;
Map<String, Object> mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose();
if (mappingMap.size() != 1) {
throw new ElasticSearchIllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
}
this.type = mappingMap.keySet().iterator().next();
initMappers((Map<String, Object>) mappingMap.get(this.type));
}
public MappingMetaData(Map<String, Object> mapping) throws IOException {
this(mapping.keySet().iterator().next(), mapping);
}
public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
this.type = type;
this.source = new CompressedString(XContentFactory.jsonBuilder().map(mapping).string());
@ -269,6 +284,10 @@ public class MappingMetaData {
if (mapping.size() == 1 && mapping.containsKey(type)) {
withoutType = (Map<String, Object>) mapping.get(type);
}
initMappers(withoutType);
}
private void initMappers(Map<String, Object> withoutType) {
if (withoutType.containsKey("_id")) {
String path = null;
Map<String, Object> routingNode = (Map<String, Object>) withoutType.get("_id");

4
src/main/java/org/elasticsearch/common/compress/CompressedString.java
View File

@ -40,6 +40,10 @@ public class CompressedString implements Streamable {
CompressedString() {
}
public CompressedString(byte[] compressed) {
this.bytes = compressed;
}
public CompressedString(String str) throws IOException {
UnicodeUtil.UTF8Result result = Unicode.unsafeFromStringAsUtf8(str);
this.bytes = LZFEncoder.encode(result.result, result.length);

23
src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gateway.local.state.meta;
import com.google.common.collect.Maps;
import com.google.common.io.Closeables;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterChangedEvent;
@ -42,6 +43,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
/**
@ -53,13 +55,24 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
private volatile MetaData currentMetaData;
private final XContentType format;
private final ToXContent.Params formatParams;
@Inject
public LocalGatewayMetaState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayMetaState nodesListGatewayMetaState) throws Exception {
super(settings);
this.nodeEnv = nodeEnv;
this.format = XContentType.fromRestContentType(settings.get("format", "smile"));
nodesListGatewayMetaState.init(this);
if (this.format == XContentType.SMILE) {
Map<String, String> params = Maps.newHashMap();
params.put("binary", "true");
formatParams = new ToXContent.MapParams(params);
} else {
formatParams = ToXContent.EMPTY_PARAMS;
}
try {
pre019Upgrade();
long start = System.currentTimeMillis();
@ -150,9 +163,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason);
CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry();
try {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes());
XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes());
builder.startObject();
IndexMetaData.Builder.toXContent(indexMetaData, builder, ToXContent.EMPTY_PARAMS);
IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams);
builder.endObject();
builder.flush();
@ -201,9 +214,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry();
try {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes());
XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes());
builder.startObject();
MetaData.Builder.toXContent(globalMetaData, builder, ToXContent.EMPTY_PARAMS);
MetaData.Builder.toXContent(globalMetaData, builder, formatParams);
builder.endObject();
builder.flush();

4
src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.gateway.Gateway;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
@ -295,6 +296,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
.startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
.endObject().endObject().endObject())
.execute().actionGet();
node2.client().admin().indices().prepareAliases().addAlias("test", "test_alias", FilterBuilders.termFilter("field", "value")).execute().actionGet();
logger.info("--> closing the second node");
@ -318,6 +320,8 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
ClusterState state = node1.client().admin().cluster().prepareState().execute().actionGet().state();
assertThat(state.metaData().index("test").mapping("type2"), notNullValue());
assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*"));
assertThat(state.metaData().index("test").aliases().get("test_alias"), notNullValue());
assertThat(state.metaData().index("test").aliases().get("test_alias").filter(), notNullValue());
}
@Test