Add Map to XContentParser Wrapper (#44036)

In some cases we need to parse some XContent that is already parsed into
a map. This is currently happening in handling source in SQL and ingest
processors as well as parsing null_value values in geo mappings. To avoid
re-serializing and parsing the value again or writing another map-based
parser this commit adds an iterator that iterates over a map as if it was
XContent. This makes reusing existing XContent parser on maps possible.

Relates to #43554
This commit is contained in:
Igor Motov 2019-07-10 16:53:17 -04:00
parent f01a9eeb34
commit 66a9b721f5
5 changed files with 603 additions and 35 deletions

View File

@ -0,0 +1,440 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.xcontent.support;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.CharBuffer;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* Wraps a map generated by XContentParser's map() method into XContent Parser
*/
public class MapXContentParser extends AbstractXContentParser {
private XContentType xContentType;
private TokenIterator iterator;
private boolean closed;
public MapXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, Map<String, Object> map,
XContentType xContentType) {
super(xContentRegistry, deprecationHandler);
this.xContentType = xContentType;
this.iterator = new MapIterator(null, null, map);
}
@Override
protected boolean doBooleanValue() throws IOException {
if (iterator != null && iterator.currentValue() instanceof Boolean) {
return (Boolean) iterator.currentValue();
} else {
throw new IllegalStateException("Cannot get boolean value for the current token " + currentToken());
}
}
@Override
protected short doShortValue() throws IOException {
return numberValue().shortValue();
}
@Override
protected int doIntValue() throws IOException {
return numberValue().intValue();
}
@Override
protected long doLongValue() throws IOException {
return numberValue().longValue();
}
@Override
protected float doFloatValue() throws IOException {
return numberValue().floatValue();
}
@Override
protected double doDoubleValue() throws IOException {
return numberValue().doubleValue();
}
@Override
public XContentType contentType() {
return xContentType;
}
@Override
public Token nextToken() throws IOException {
if (iterator == null) {
return null;
} else {
iterator = iterator.next();
}
return currentToken();
}
@Override
public void skipChildren() throws IOException {
Token token = currentToken();
if (token == Token.START_OBJECT || token == Token.START_ARRAY) {
iterator = iterator.skipChildren();
}
}
@Override
public Token currentToken() {
if (iterator == null) {
return null;
} else {
return iterator.currentToken();
}
}
@Override
public String currentName() throws IOException {
if (iterator == null) {
return null;
} else {
return iterator.currentName();
}
}
@Override
public String text() throws IOException {
if (iterator != null) {
if (currentToken() == Token.VALUE_STRING || currentToken() == Token.VALUE_NUMBER || currentToken() == Token.VALUE_BOOLEAN) {
return iterator.currentValue().toString();
} else if (currentToken() == Token.FIELD_NAME) {
return iterator.currentName();
} else {
return null;
}
} else {
throw new IllegalStateException("Cannot get text for the current token " + currentToken());
}
}
@Override
public CharBuffer charBuffer() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public Object objectText() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public Object objectBytes() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public boolean hasTextCharacters() {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public char[] textCharacters() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public int textLength() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public int textOffset() throws IOException {
throw new UnsupportedOperationException("use text() instead");
}
@Override
public Number numberValue() throws IOException {
if (iterator != null && currentToken() == Token.VALUE_NUMBER) {
return (Number) iterator.currentValue();
} else {
throw new IllegalStateException("Cannot get numeric value for the current token " + currentToken());
}
}
@Override
public NumberType numberType() throws IOException {
Number number = numberValue();
if (number instanceof Integer) {
return NumberType.INT;
} else if (number instanceof BigInteger) {
return NumberType.BIG_INTEGER;
} else if (number instanceof Long) {
return NumberType.LONG;
} else if (number instanceof Float) {
return NumberType.FLOAT;
} else if (number instanceof Double) {
return NumberType.DOUBLE;
} else if (number instanceof BigDecimal) {
return NumberType.BIG_DECIMAL;
}
throw new IllegalStateException("No matching token for number_type [" + number.getClass() + "]");
}
@Override
public byte[] binaryValue() throws IOException {
if (iterator != null && iterator.currentValue() instanceof byte[]) {
return (byte[]) iterator.currentValue();
} else {
throw new IllegalStateException("Cannot get binary value for the current token " + currentToken());
}
}
@Override
public XContentLocation getTokenLocation() {
return new XContentLocation(0, 0);
}
@Override
public boolean isClosed() {
return closed;
}
@Override
public void close() throws IOException {
closed = true;
}
/**
* Iterator over the elements of the map
*/
private abstract static class TokenIterator {
protected final TokenIterator parent;
protected final String name;
protected Token currentToken;
protected State state = State.BEFORE;
TokenIterator(TokenIterator parent, String name) {
this.parent = parent;
this.name = name;
}
public abstract TokenIterator next();
public abstract TokenIterator skipChildren();
public Token currentToken() {
return currentToken;
}
public abstract Object currentValue();
/**
* name of the field name of the current element
*/
public abstract String currentName();
/**
* field name that the child element needs to inherit.
*
* In most cases this is the same as currentName() except with embedded arrays. In "foo": [[42]] the first START_ARRAY
* token will have the name "foo", but the second START_ARRAY will have no name.
*/
public abstract String childName();
@SuppressWarnings("unchecked")
TokenIterator processValue(Object value) {
if (value instanceof Map) {
return new MapIterator(this, childName(), (Map<String, Object>) value).next();
} else if (value instanceof List) {
return new ArrayIterator(this, childName(), (List<Object>) value).next();
} else if (value instanceof Number) {
currentToken = Token.VALUE_NUMBER;
} else if (value instanceof String) {
currentToken = Token.VALUE_STRING;
} else if (value instanceof Boolean) {
currentToken = Token.VALUE_BOOLEAN;
} else if (value instanceof byte[]) {
currentToken = Token.VALUE_EMBEDDED_OBJECT;
} else if (value == null) {
currentToken = Token.VALUE_NULL;
}
return this;
}
}
private enum State {
BEFORE,
NAME,
VALUE,
AFTER
}
/**
* Iterator over the map
*/
private static class MapIterator extends TokenIterator {
private final Iterator<Map.Entry<String, Object>> iterator;
private Map.Entry<String, Object> entry;
MapIterator(TokenIterator parent, String name, Map<String, Object> map) {
super(parent, name);
iterator = map.entrySet().iterator();
}
@Override
public TokenIterator next() {
switch (state) {
case BEFORE:
state = State.NAME;
currentToken = Token.START_OBJECT;
return this;
case NAME:
if (iterator.hasNext()) {
state = State.VALUE;
entry = iterator.next();
currentToken = Token.FIELD_NAME;
return this;
} else {
state = State.AFTER;
entry = null;
currentToken = Token.END_OBJECT;
return this;
}
case VALUE:
state = State.NAME;
return processValue(entry.getValue());
case AFTER:
currentToken = null;
if (parent == null) {
return null;
} else {
return parent.next();
}
default:
throw new IllegalArgumentException("Unknown state " + state);
}
}
@Override
public TokenIterator skipChildren() {
state = State.AFTER;
entry = null;
currentToken = Token.END_OBJECT;
return this;
}
@Override
public Object currentValue() {
if (entry == null) {
throw new IllegalStateException("Cannot get value for non-value token " + currentToken);
}
return entry.getValue();
}
@Override
public String currentName() {
if (entry == null) {
return name;
}
return entry.getKey();
}
@Override
public String childName() {
return currentName();
}
}
private static class ArrayIterator extends TokenIterator {
private final Iterator<Object> iterator;
private Object value;
private ArrayIterator(TokenIterator parent, String name, List<Object> list) {
super(parent, name);
iterator = list.iterator();
}
@Override
public TokenIterator next() {
switch (state) {
case BEFORE:
state = State.VALUE;
currentToken = Token.START_ARRAY;
return this;
case VALUE:
if (iterator.hasNext()) {
value = iterator.next();
return processValue(value);
} else {
state = State.AFTER;
value = null;
currentToken = Token.END_ARRAY;
return this;
}
case AFTER:
currentToken = null;
if (parent == null) {
return null;
} else {
return parent.next();
}
default:
throw new IllegalArgumentException("Unknown state " + state);
}
}
@Override
public TokenIterator skipChildren() {
state = State.AFTER;
value = null;
currentToken = Token.END_ARRAY;
return this;
}
@Override
public Object currentValue() {
return value;
}
@Override
public String currentName() {
if (parent == null || (currentToken != Token.START_ARRAY && currentToken != Token.END_ARRAY)) {
return null;
} else {
return name;
}
}
@Override
public String childName() {
return null;
}
}
}

View File

@ -0,0 +1,147 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.xcontent;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.support.MapXContentParser;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentParserTests.generateRandomObject;
public class MapXContentParserTests extends ESTestCase {
public void testSimpleMap() throws IOException {
compareTokens(builder -> {
builder.startObject();
builder.field("string", "foo");
builder.field("number", 42);
builder.field("double", 42.5);
builder.field("bool", false);
builder.startArray("arr");
{
builder.value(10).value(20.0).value("30");
builder.startArray();
builder.value(30);
builder.endArray();
}
builder.endArray();
builder.startArray("nested_arr");
{
builder.startArray();
builder.value(10);
builder.endArray();
}
builder.endArray();
builder.startObject("obj");
{
builder.field("inner_string", "bar");
builder.startObject("inner_empty_obj");
builder.field("f", "a");
builder.endObject();
}
builder.endObject();
builder.field("bytes", new byte[]{1, 2, 3});
builder.nullField("nothing");
builder.endObject();
});
}
public void testRandomObject() throws IOException {
compareTokens(builder -> generateRandomObject(builder, randomIntBetween(0, 10)));
}
public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer) throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());
try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
consumer.accept(builder);
final Map<String, Object> map;
try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) {
map = parser.mapOrdered();
}
try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) {
try (XContentParser mapParser = new MapXContentParser(
xContentRegistry(), LoggingDeprecationHandler.INSTANCE, map, xContentType)) {
assertEquals(parser.contentType(), mapParser.contentType());
XContentParser.Token token;
assertEquals(parser.currentToken(), mapParser.currentToken());
assertEquals(parser.currentName(), mapParser.currentName());
do {
token = parser.nextToken();
XContentParser.Token mapToken = mapParser.nextToken();
assertEquals(token, mapToken);
assertEquals(parser.currentName(), mapParser.currentName());
if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) {
assertEquals(parser.textOrNull(), mapParser.textOrNull());
switch (token) {
case VALUE_STRING:
assertEquals(parser.text(), mapParser.text());
break;
case VALUE_NUMBER:
assertEquals(parser.numberType(), mapParser.numberType());
assertEquals(parser.numberValue(), mapParser.numberValue());
if (parser.numberType() == XContentParser.NumberType.LONG ||
parser.numberType() == XContentParser.NumberType.INT) {
assertEquals(parser.longValue(), mapParser.longValue());
if (parser.longValue() <= Integer.MAX_VALUE && parser.longValue() >= Integer.MIN_VALUE) {
assertEquals(parser.intValue(), mapParser.intValue());
if (parser.longValue() <= Short.MAX_VALUE && parser.longValue() >= Short.MIN_VALUE) {
assertEquals(parser.shortValue(), mapParser.shortValue());
}
}
} else {
assertEquals(parser.doubleValue(), mapParser.doubleValue(), 0.000001);
}
break;
case VALUE_BOOLEAN:
assertEquals(parser.booleanValue(), mapParser.booleanValue());
break;
case VALUE_EMBEDDED_OBJECT:
assertArrayEquals(parser.binaryValue(), mapParser.binaryValue());
break;
case VALUE_NULL:
assertNull(mapParser.textOrNull());
break;
}
assertEquals(parser.currentName(), mapParser.currentName());
assertEquals(parser.isClosed(), mapParser.isClosed());
} else if (token == XContentParser.Token.START_ARRAY || token == XContentParser.Token.START_OBJECT) {
if (randomInt(5) == 0) {
parser.skipChildren();
mapParser.skipChildren();
}
}
} while (token != null);
assertEquals(parser.nextToken(), mapParser.nextToken());
parser.close();
mapParser.close();
assertEquals(parser.isClosed(), mapParser.isClosed());
assertTrue(mapParser.isClosed());
}
}
}
}
}

View File

@ -512,7 +512,7 @@ public class XContentParserTests extends ESTestCase {
*
* Returns the number of tokens in the marked field
*/
private int generateRandomObjectForMarking(XContentBuilder builder) throws IOException {
private static int generateRandomObjectForMarking(XContentBuilder builder) throws IOException {
builder.startObject()
.field("first_field", "foo")
.field("marked_field");
@ -521,7 +521,7 @@ public class XContentParserTests extends ESTestCase {
return numberOfTokens;
}
private int generateRandomObject(XContentBuilder builder, int level) throws IOException {
public static int generateRandomObject(XContentBuilder builder, int level) throws IOException {
int tokens = 2;
builder.startObject();
int numberOfElements = randomInt(5);
@ -533,7 +533,7 @@ public class XContentParserTests extends ESTestCase {
return tokens;
}
private int generateRandomValue(XContentBuilder builder, int level) throws IOException {
private static int generateRandomValue(XContentBuilder builder, int level) throws IOException {
@SuppressWarnings("unchecked") CheckedSupplier<Integer, IOException> fieldGenerator = randomFrom(
() -> {
builder.value(randomInt());
@ -568,7 +568,7 @@ public class XContentParserTests extends ESTestCase {
return fieldGenerator.get();
}
private int generateRandomArray(XContentBuilder builder, int level) throws IOException {
private static int generateRandomArray(XContentBuilder builder, int level) throws IOException {
int tokens = 2;
int arraySize = randomInt(3);
builder.startArray();

View File

@ -23,15 +23,13 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.apache.lucene.util.SloppyMath;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.XContentSubParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.MapXContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.geo.geometry.Rectangle;
import org.elasticsearch.geo.utils.Geohash;
@ -43,7 +41,7 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
public class GeoUtils {
@ -376,21 +374,12 @@ public class GeoUtils {
* Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true
*/
public static GeoPoint parseGeoPoint(Object value, final boolean ignoreZValue) throws ElasticsearchParseException {
try {
XContentBuilder content = JsonXContent.contentBuilder();
content.startObject();
content.field("null_value", value);
content.endObject();
try (InputStream stream = BytesReference.bytes(content).streamInput();
XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value
return parseGeoPoint(parser, new GeoPoint(), ignoreZValue);
}
try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
Collections.singletonMap("null_value", value), null)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value
return parseGeoPoint(parser, new GeoPoint(), ignoreZValue);
} catch (IOException ex) {
throw new ElasticsearchParseException("error parsing geopoint", ex);
}

View File

@ -20,18 +20,16 @@ package org.elasticsearch.common.geo.parsers;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.MapXContentParser;
import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
/**
* first point of entry for a shape parser
@ -75,14 +73,8 @@ public interface ShapeParser {
}
static ShapeBuilder parse(Object value) throws IOException {
XContentBuilder content = JsonXContent.contentBuilder();
content.startObject();
content.field("value", value);
content.endObject();
try (InputStream stream = BytesReference.bytes(content).streamInput();
XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
Collections.singletonMap("value", value), null)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value