honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add XContentHelper.childBytes() method (#54287) 

We have a number of places where we want to read a fairly complex object from
XContent, but aren't interested in its contents; for example, mappings are often
serialized and deserialized between several objects before they are actually built
into a MappingMetaData object. This means that potentially large maps of maps
are constructed several times, only to immediately be re-serialized again.

This commit adds a new helper method to XContentHelper that reads the children
of an xcontent object directly to a BytesReference, serialized via the same xcontenttype
as the parent parser, avoiding the construction of intermediary maps or lists.
This commit is contained in:
Alan Woodward 2020-03-27 14:21:13 +00:00
parent d9d11f6d16
commit 461f1307d6
2 changed files with 144 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
View File

@ -384,4 +384,23 @@ public class XContentHelper {
BytesRef br = bytes.toBytesRef();
return XContentFactory.xContentType(br.bytes, br.offset, br.length);
}
/**
* Returns the contents of an object as an unparsed BytesReference
*
* This is useful for things like mappings where we're copying bytes around but don't
* actually need to parse their contents, and so avoids building large maps of maps
* unnecessarily
*/
public static BytesReference childBytes(XContentParser parser) throws IOException {
if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new XContentParseException(parser.getTokenLocation(),
"Expected [START_OBJECT] but got [" + parser.currentToken() + "]");
}
}
XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent());
builder.copyCurrentStructure(parser);
return BytesReference.bytes(builder);
}
}

125
server/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
View File

@ -20,10 +20,12 @@
package org.elasticsearch.common.xcontent.support;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
@ -105,4 +107,127 @@ public class XContentHelperTests extends ESTestCase {
}
}
}
public void testChildBytes() throws IOException {
for (XContentType xContentType : XContentType.values()) {
XContentBuilder builder = XContentBuilder.builder(xContentType.xContent());
builder.startObject().startObject("level1");
builder.startObject("level2")
.startObject("object").field("text", "string").field("number", 10).endObject()
.startObject("object2").field("boolean", true).nullField("null")
.startArray("array_of_strings").value("string1").value("string2").endArray().endObject().endObject();
builder.field("field", "value");
builder.endObject().endObject();
BytesReference input = BytesReference.bytes(builder);
BytesReference bytes;
try (XContentParser parser = xContentType.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, input.streamInput())) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("level2", parser.currentName());
// Extract everything under 'level2' as a bytestream
bytes = XContentHelper.childBytes(parser);
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("field", parser.currentName());
}
// now parse the contents of 'level2'
try (XContentParser parser = xContentType.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput())) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("object", parser.currentName());
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("text", parser.currentName());
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
assertEquals("string", parser.text());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("number", parser.currentName());
assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
assertEquals(10, parser.numberValue());
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("object2", parser.currentName());
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("boolean", parser.currentName());
assertEquals(XContentParser.Token.VALUE_BOOLEAN, parser.nextToken());
assertTrue(parser.booleanValue());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("null", parser.currentName());
assertEquals(XContentParser.Token.VALUE_NULL, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("array_of_strings", parser.currentName());
assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
assertEquals("string1", parser.text());
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
assertEquals("string2", parser.text());
assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken());
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
}
}
}
public void testEmbeddedObject() throws IOException {
// Need to test this separately as XContentType.JSON never produces VALUE_EMBEDDED_OBJECT
XContentBuilder builder = XContentBuilder.builder(XContentType.CBOR.xContent());
builder.startObject().startObject("root");
CompressedXContent embedded = new CompressedXContent("{\"field\":\"value\"}");
builder.field("bytes", embedded.compressed());
builder.endObject().endObject();
BytesReference bytes = BytesReference.bytes(builder);
BytesReference inner;
try (XContentParser parser = XContentType.CBOR.xContent().createParser(
NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput())) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
inner = XContentHelper.childBytes(parser);
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
}
try (XContentParser parser = XContentType.CBOR.xContent().createParser(
NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inner.streamInput())) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals("bytes", parser.currentName());
assertEquals(XContentParser.Token.VALUE_EMBEDDED_OBJECT, parser.nextToken());
assertEquals(embedded, new CompressedXContent(parser.binaryValue()));
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
}
}
public void testEmptyChildBytes() throws IOException {
String inputJson = "{ \"mappings\" : {} }";
try (XContentParser parser = XContentType.JSON.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputJson)) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
BytesReference bytes = XContentHelper.childBytes(parser);
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
assertEquals("{}", bytes.utf8ToString());
}
}
}