honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup/optimize metadata stage parsing and generalize it

This commit is contained in:
Shay Banon 2011-08-30 15:36:09 +03:00
parent e5e96a86a0
commit a5aac3a5c8
3 changed files with 225 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Required;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
@ -588,19 +587,18 @@ public class IndexRequest extends ShardReplicationOperationRequest {
}
// extract values if needed
if (mappingMd != null) {
boolean shouldParseRouting = (routing == null && mappingMd.routing().hasPath());
boolean shouldParseTimestamp = (timestamp == null && mappingMd.timestamp().hasPath());
MappingMetaData.ParseContext parseContext = mappingMd.createParseContext(routing, timestamp);
if (shouldParseRouting || shouldParseTimestamp) {
if (parseContext.shouldParse()) {
XContentParser parser = null;
try {
parser = XContentFactory.xContent(source, sourceOffset, sourceLength).createParser(source, sourceOffset, sourceLength);
Tuple<String, String> parseResult = mappingMd.parseRoutingAndTimestamp(parser, shouldParseRouting, shouldParseTimestamp);
if (shouldParseRouting) {
routing = parseResult.v1();
mappingMd.parse(parser, parseContext);
if (parseContext.shouldParseRouting()) {
routing = parseContext.routing();
}
if (shouldParseTimestamp) {
timestamp = parseResult.v2();
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
}
} catch (Exception e) {

179
modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.cluster.metadata;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -229,19 +228,22 @@ public class MappingMetaData {
return this.timestamp;
}
public Tuple<String, String> parseRoutingAndTimestamp(XContentParser parser,
boolean shouldParseRouting,
boolean shouldParseTimestamp) throws IOException {
return parseRoutingAndTimestamp(parser, 0, 0, null, null, shouldParseRouting, shouldParseTimestamp);
public ParseContext createParseContext(@Nullable String routing, @Nullable String timestamp) {
return new ParseContext(
routing == null && routing().hasPath(),
timestamp == null && timestamp().hasPath()
);
}
private Tuple<String, String> parseRoutingAndTimestamp(XContentParser parser,
int locationRouting,
int locationTimestamp,
@Nullable String routingValue,
@Nullable String timestampValue,
boolean shouldParseRouting,
boolean shouldParseTimestamp) throws IOException {
public void parse(XContentParser parser, ParseContext parseContext) throws IOException {
innerParse(parser, parseContext);
}
private void innerParse(XContentParser parser, ParseContext context) throws IOException {
if (!context.parsingStillNeeded()) {
return;
}
XContentParser.Token t = parser.currentToken();
if (t == null) {
t = parser.nextToken();
@ -249,8 +251,8 @@ public class MappingMetaData {
if (t == XContentParser.Token.START_OBJECT) {
t = parser.nextToken();
}
String routingPart = shouldParseRouting ? routing().pathElements()[locationRouting] : null;
String timestampPart = shouldParseTimestamp ? timestamp().pathElements()[locationTimestamp] : null;
String routingPart = context.routingParsingStillNeeded() ? routing().pathElements()[context.locationRouting] : null;
String timestampPart = context.timestampParsingStillNeeded() ? timestamp().pathElements()[context.locationTimestamp] : null;
for (; t == XContentParser.Token.FIELD_NAME; t = parser.nextToken()) {
// Must point to field name
@ -260,18 +262,18 @@ public class MappingMetaData {
boolean incLocationRouting = false;
boolean incLocationTimestamp = false;
if (shouldParseRouting && routingPart.equals(fieldName)) {
if (locationRouting + 1 == routing.pathElements().length) {
routingValue = parser.textOrNull();
shouldParseRouting = false;
if (context.routingParsingStillNeeded() && fieldName.equals(routingPart)) {
if (context.locationRouting + 1 == routing.pathElements().length) {
context.routing = parser.textOrNull();
context.routingResolved = true;
} else {
incLocationRouting = true;
}
}
if (shouldParseTimestamp && timestampPart.equals(fieldName)) {
if (locationTimestamp + 1 == timestamp.pathElements().length) {
timestampValue = parser.textOrNull();
shouldParseTimestamp = false;
if (context.timestampParsingStillNeeded() && fieldName.equals(timestampPart)) {
if (context.locationTimestamp + 1 == timestamp.pathElements().length) {
context.timestamp = parser.textOrNull();
context.timestampResolved = true;
} else {
incLocationTimestamp = true;
}
@ -279,37 +281,20 @@ public class MappingMetaData {
if (incLocationRouting || incLocationTimestamp) {
if (t == XContentParser.Token.START_OBJECT) {
locationRouting += incLocationRouting ? 1 : 0;
locationTimestamp += incLocationTimestamp ? 1 : 0;
Tuple<String, String> result = parseRoutingAndTimestamp(parser, locationRouting, locationTimestamp, routingValue, timestampValue,
shouldParseRouting, shouldParseTimestamp);
routingValue = result.v1();
timestampValue = result.v2();
if (incLocationRouting) {
if (routingValue != null) {
shouldParseRouting = false;
} else {
locationRouting--;
}
}
if (incLocationTimestamp) {
if (timestampValue != null) {
shouldParseTimestamp = false;
} else {
locationTimestamp--;
}
}
context.locationRouting += incLocationRouting ? 1 : 0;
context.locationTimestamp += incLocationTimestamp ? 1 : 0;
innerParse(parser, context);
context.locationRouting -= incLocationRouting ? 1 : 0;
context.locationTimestamp -= incLocationTimestamp ? 1 : 0;
}
} else {
parser.skipChildren();
}
if (!shouldParseRouting && !shouldParseTimestamp) {
return Tuple.create(routingValue, timestampValue);
if (!context.parsingStillNeeded()) {
return;
}
}
return Tuple.create(routingValue, timestampValue);
}
public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws IOException {
@ -343,4 +328,106 @@ public class MappingMetaData {
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readUTF() : null, in.readUTF());
return new MappingMetaData(type, source, routing, timestamp);
}
public static class ParseResult {
public final String routing;
public final boolean routingResolved;
public final String timestamp;
public final boolean timestampResolved;
public ParseResult(String routing, boolean routingResolved, String timestamp, boolean timestampResolved) {
this.routing = routing;
this.routingResolved = routingResolved;
this.timestamp = timestamp;
this.timestampResolved = timestampResolved;
}
}
public static class ParseContext {
final boolean shouldParseRouting;
final boolean shouldParseTimestamp;
int locationRouting = 0;
int locationTimestamp = 0;
boolean routingResolved;
boolean timestampResolved;
String routing;
String timestamp;
public ParseContext(boolean shouldParseRouting, boolean shouldParseTimestamp) {
this.shouldParseRouting = shouldParseRouting;
this.shouldParseTimestamp = shouldParseTimestamp;
}
/**
* The routing value parsed, <tt>null</tt> if does not require parsing, or not resolved.
*/
public String routing() {
return routing;
}
/**
* Does routing parsing really needed at all?
*/
public boolean shouldParseRouting() {
return shouldParseRouting;
}
/**
* Has routing been resolved during the parsing phase.
*/
public boolean routingResolved() {
return routingResolved;
}
/**
* Is routing parsing still needed?
*/
public boolean routingParsingStillNeeded() {
return shouldParseRouting && !routingResolved;
}
/**
* The timestamp value parsed, <tt>null</tt> if does not require parsing, or not resolved.
*/
public String timestamp() {
return timestamp;
}
/**
* Does timestamp parsing really needed at all?
*/
public boolean shouldParseTimestamp() {
return shouldParseTimestamp;
}
/**
* Has timestamp been resolved during the parsing phase.
*/
public boolean timestampResolved() {
return timestampResolved;
}
/**
* Is timestamp parsing still needed?
*/
public boolean timestampParsingStillNeeded() {
return shouldParseTimestamp && !timestampResolved;
}
/**
* Do we really need parsing?
*/
public boolean shouldParse() {
return shouldParseRouting || shouldParseTimestamp;
}
/**
* Is parsing still needed?
*/
public boolean parsingStillNeeded() {
return routingParsingStillNeeded() || timestampParsingStillNeeded();
}
}
}

126
modules/elasticsearch/src/test/java/org/elasticsearch/cluster/metadata/ParseRoutingTimestampTests.java → modules/elasticsearch/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.testng.annotations.Test;
@ -28,10 +27,8 @@ import static org.elasticsearch.common.xcontent.XContentFactory.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
public class ParseRoutingTimestampTests {
@Test
public class MappingMetaDataParserTests {
@Test public void testParseRoutingAlone() throws Exception {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
@ -39,9 +36,12 @@ public class ParseRoutingTimestampTests {
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"));
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("routing", "routing_value").field("timestamp", "1").endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, false);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo(null));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "1");
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.routingResolved(), equalTo(true));
assertThat(parseContext.timestamp(), nullValue());
assertThat(parseContext.timestampResolved(), equalTo(false));
}
@Test public void testParseTimestampAlone() throws Exception {
@ -50,9 +50,10 @@ public class ParseRoutingTimestampTests {
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"));
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("routing", "routing_value").field("timestamp", "1").endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), false, true);
assertThat(parsed.v1(), equalTo(null));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext("routing_value1", null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), nullValue());
assertThat(parseContext.timestamp(), equalTo("1"));
}
@Test public void testParseRoutingAndTimestamp() throws Exception {
@ -61,9 +62,10 @@ public class ParseRoutingTimestampTests {
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"));
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("routing", "routing_value").field("timestamp", "1").endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.timestamp(), equalTo("1"));
}
@Test public void testParseRoutingAndTimestampWithPath() throws Exception {
@ -75,9 +77,44 @@ public class ParseRoutingTimestampTests {
.startObject("obj1").field("routing", "routing_value").endObject()
.startObject("obj2").field("timestamp", "1").endObject()
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.timestamp(), equalTo("1"));
}
@Test public void testParseRoutingWithPath() throws Exception {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"));
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("routing", "routing_value").endObject()
.startObject("obj2").field("timestamp", "1").endObject()
.endObject().copiedBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "2");
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.routingResolved(), equalTo(true));
assertThat(parseContext.timestamp(), nullValue());
assertThat(parseContext.timestampResolved(), equalTo(false));
}
@Test public void testParseTimestampWithPath() throws Exception {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"));
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("routing", "routing_value").endObject()
.startObject("obj2").field("timestamp", "1").endObject()
.endObject().copiedBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext("routing_value1", null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), nullValue());
assertThat(parseContext.routingResolved(), equalTo(false));
assertThat(parseContext.timestamp(), equalTo("1"));
assertThat(parseContext.timestampResolved(), equalTo(true));
}
@Test public void testParseRoutingAndTimestampWithinSamePath() throws Exception {
@ -89,9 +126,10 @@ public class ParseRoutingTimestampTests {
.startObject("obj1").field("routing", "routing_value").field("timestamp", "1").endObject()
.startObject("obj2").field("field1", "value1").endObject()
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.timestamp(), equalTo("1"));
}
@Test public void testParseRoutingAndTimestampWithinSamePathAndMoreLevels() throws Exception {
@ -101,18 +139,19 @@ public class ParseRoutingTimestampTests {
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1")
.startObject("obj2")
.field("routing", "routing_value")
.endObject()
.startObject("obj3")
.field("timestamp", "1")
.endObject()
.startObject("obj2")
.field("routing", "routing_value")
.endObject()
.startObject("obj3")
.field("timestamp", "1")
.endObject()
.endObject()
.startObject("obj2").field("field1", "value1").endObject()
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.timestamp(), equalTo("1"));
}
@ -125,11 +164,13 @@ public class ParseRoutingTimestampTests {
.startObject("obj1").field("routing", "routing_value").endObject()
.startObject("obj1").field("timestamp", "1").endObject()
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("routing_value"));
assertThat(parsed.v2(), equalTo("1"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("routing_value"));
assertThat(parseContext.timestamp(), equalTo("1"));
}
//
@Test public void testParseRoutingTimestampWithRepeatedField() throws Exception {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Routing(true, "field1.field1"),
@ -144,9 +185,10 @@ public class ParseRoutingTimestampTests {
.field("zzz", "wr")
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo(null));
assertThat(parsed.v2(), equalTo("foo"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), nullValue());
assertThat(parseContext.timestamp(), equalTo("foo"));
}
@Test public void testParseRoutingWithRepeatedFieldAndObject() throws Exception {
@ -163,9 +205,10 @@ public class ParseRoutingTimestampTests {
.field("zzz", "wr")
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo(null));
assertThat(parsed.v2(), equalTo("foo"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), nullValue());
assertThat(parseContext.timestamp(), equalTo("foo"));
}
@Test public void testParseRoutingWithRepeatedFieldAndValidRouting() throws Exception {
@ -182,8 +225,9 @@ public class ParseRoutingTimestampTests {
.field("zzz", "wr")
.endObject().copiedBytes();
Tuple<String, String> parsed = md.parseRoutingAndTimestamp(XContentFactory.xContent(bytes).createParser(bytes), true, true);
assertThat(parsed.v1(), equalTo("bar"));
assertThat(parsed.v2(), equalTo("foo"));
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null);
md.parse(XContentFactory.xContent(bytes).createParser(bytes), parseContext);
assertThat(parseContext.routing(), equalTo("bar"));
assertThat(parseContext.timestamp(), equalTo("foo"));
}
}