From 66a9b721f540f1692ea17de8ba8269d7ac6af910 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 10 Jul 2019 16:53:17 -0400 Subject: [PATCH] Add Map to XContentParser Wrapper (#44036) In some cases we need to parse some XContent that is already parsed into a map. This is currently happening in handling source in SQL and ingest processors as well as parsing null_value values in geo mappings. To avoid re-serializing and parsing the value again or writing another map-based parser this commit adds an iterator that iterates over a map as if it was XContent. This makes reusing existing XContent parser on maps possible. Relates to #43554 --- .../xcontent/support/MapXContentParser.java | 440 ++++++++++++++++++ .../xcontent/MapXContentParserTests.java | 147 ++++++ .../common/xcontent/XContentParserTests.java | 8 +- .../elasticsearch/common/geo/GeoUtils.java | 27 +- .../common/geo/parsers/ShapeParser.java | 16 +- 5 files changed, 603 insertions(+), 35 deletions(-) create mode 100644 libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java create mode 100644 libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java new file mode 100644 index 00000000000..c54e71634d6 --- /dev/null +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java @@ -0,0 +1,440 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support; + +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.CharBuffer; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Wraps a map generated by XContentParser's map() method into XContent Parser + */ +public class MapXContentParser extends AbstractXContentParser { + + private XContentType xContentType; + private TokenIterator iterator; + private boolean closed; + + public MapXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, Map map, + XContentType xContentType) { + super(xContentRegistry, deprecationHandler); + this.xContentType = xContentType; + this.iterator = new MapIterator(null, null, map); + } + + + @Override + protected boolean doBooleanValue() throws IOException { + if (iterator != null && iterator.currentValue() instanceof Boolean) { + return (Boolean) iterator.currentValue(); + } else { + throw new IllegalStateException("Cannot get boolean value for the current token " + currentToken()); + } + } + + @Override + protected short doShortValue() throws IOException { + return numberValue().shortValue(); + } + + @Override + protected int doIntValue() throws IOException { + return numberValue().intValue(); + } + + @Override + protected long doLongValue() throws IOException { + return numberValue().longValue(); + } + + @Override + protected float doFloatValue() throws IOException { + return numberValue().floatValue(); + } + + @Override + protected double doDoubleValue() throws IOException { + return numberValue().doubleValue(); + } + + @Override + public XContentType contentType() { + return xContentType; + } + + @Override + public Token nextToken() throws IOException { + if (iterator == null) { + return null; + } else { + iterator = iterator.next(); + } + return currentToken(); + } + + @Override + public void skipChildren() throws IOException { + Token token = currentToken(); + if (token == Token.START_OBJECT || token == Token.START_ARRAY) { + iterator = iterator.skipChildren(); + } + } + + @Override + public Token currentToken() { + if (iterator == null) { + return null; + } else { + return iterator.currentToken(); + } + } + + @Override + public String currentName() throws IOException { + if (iterator == null) { + return null; + } else { + return iterator.currentName(); + } + } + + @Override + public String text() throws IOException { + if (iterator != null) { + if (currentToken() == Token.VALUE_STRING || currentToken() == Token.VALUE_NUMBER || currentToken() == Token.VALUE_BOOLEAN) { + return iterator.currentValue().toString(); + } else if (currentToken() == Token.FIELD_NAME) { + return iterator.currentName(); + } else { + return null; + } + } else { + throw new IllegalStateException("Cannot get text for the current token " + currentToken()); + } + } + + @Override + public CharBuffer charBuffer() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public Object objectText() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public Object objectBytes() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public boolean hasTextCharacters() { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public char[] textCharacters() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public int textLength() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public int textOffset() throws IOException { + throw new UnsupportedOperationException("use text() instead"); + } + + @Override + public Number numberValue() throws IOException { + if (iterator != null && currentToken() == Token.VALUE_NUMBER) { + return (Number) iterator.currentValue(); + } else { + throw new IllegalStateException("Cannot get numeric value for the current token " + currentToken()); + } + } + + @Override + public NumberType numberType() throws IOException { + Number number = numberValue(); + if (number instanceof Integer) { + return NumberType.INT; + } else if (number instanceof BigInteger) { + return NumberType.BIG_INTEGER; + } else if (number instanceof Long) { + return NumberType.LONG; + } else if (number instanceof Float) { + return NumberType.FLOAT; + } else if (number instanceof Double) { + return NumberType.DOUBLE; + } else if (number instanceof BigDecimal) { + return NumberType.BIG_DECIMAL; + } + throw new IllegalStateException("No matching token for number_type [" + number.getClass() + "]"); + } + + @Override + public byte[] binaryValue() throws IOException { + if (iterator != null && iterator.currentValue() instanceof byte[]) { + return (byte[]) iterator.currentValue(); + } else { + throw new IllegalStateException("Cannot get binary value for the current token " + currentToken()); + } + } + + @Override + public XContentLocation getTokenLocation() { + return new XContentLocation(0, 0); + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public void close() throws IOException { + closed = true; + } + + /** + * Iterator over the elements of the map + */ + private abstract static class TokenIterator { + protected final TokenIterator parent; + protected final String name; + protected Token currentToken; + protected State state = State.BEFORE; + + TokenIterator(TokenIterator parent, String name) { + this.parent = parent; + this.name = name; + } + + public abstract TokenIterator next(); + + public abstract TokenIterator skipChildren(); + + public Token currentToken() { + return currentToken; + } + + public abstract Object currentValue(); + + /** + * name of the field name of the current element + */ + public abstract String currentName(); + + /** + * field name that the child element needs to inherit. + * + * In most cases this is the same as currentName() except with embedded arrays. In "foo": [[42]] the first START_ARRAY + * token will have the name "foo", but the second START_ARRAY will have no name. + */ + public abstract String childName(); + + @SuppressWarnings("unchecked") + TokenIterator processValue(Object value) { + if (value instanceof Map) { + return new MapIterator(this, childName(), (Map) value).next(); + } else if (value instanceof List) { + return new ArrayIterator(this, childName(), (List) value).next(); + } else if (value instanceof Number) { + currentToken = Token.VALUE_NUMBER; + } else if (value instanceof String) { + currentToken = Token.VALUE_STRING; + } else if (value instanceof Boolean) { + currentToken = Token.VALUE_BOOLEAN; + } else if (value instanceof byte[]) { + currentToken = Token.VALUE_EMBEDDED_OBJECT; + } else if (value == null) { + currentToken = Token.VALUE_NULL; + } + return this; + } + + } + + private enum State { + BEFORE, + NAME, + VALUE, + AFTER + } + + /** + * Iterator over the map + */ + private static class MapIterator extends TokenIterator { + + private final Iterator> iterator; + + private Map.Entry entry; + + MapIterator(TokenIterator parent, String name, Map map) { + super(parent, name); + iterator = map.entrySet().iterator(); + } + + @Override + public TokenIterator next() { + switch (state) { + case BEFORE: + state = State.NAME; + currentToken = Token.START_OBJECT; + return this; + case NAME: + if (iterator.hasNext()) { + state = State.VALUE; + entry = iterator.next(); + currentToken = Token.FIELD_NAME; + return this; + } else { + state = State.AFTER; + entry = null; + currentToken = Token.END_OBJECT; + return this; + } + case VALUE: + state = State.NAME; + return processValue(entry.getValue()); + case AFTER: + currentToken = null; + if (parent == null) { + return null; + } else { + return parent.next(); + } + default: + throw new IllegalArgumentException("Unknown state " + state); + + } + } + + @Override + public TokenIterator skipChildren() { + state = State.AFTER; + entry = null; + currentToken = Token.END_OBJECT; + return this; + } + + @Override + public Object currentValue() { + if (entry == null) { + throw new IllegalStateException("Cannot get value for non-value token " + currentToken); + } + return entry.getValue(); + } + + @Override + public String currentName() { + if (entry == null) { + return name; + } + return entry.getKey(); + } + + @Override + public String childName() { + return currentName(); + } + } + + private static class ArrayIterator extends TokenIterator { + private final Iterator iterator; + + private Object value; + + private ArrayIterator(TokenIterator parent, String name, List list) { + super(parent, name); + iterator = list.iterator(); + } + + @Override + public TokenIterator next() { + switch (state) { + case BEFORE: + state = State.VALUE; + currentToken = Token.START_ARRAY; + return this; + case VALUE: + if (iterator.hasNext()) { + value = iterator.next(); + return processValue(value); + } else { + state = State.AFTER; + value = null; + currentToken = Token.END_ARRAY; + return this; + } + case AFTER: + currentToken = null; + if (parent == null) { + return null; + } else { + return parent.next(); + } + default: + throw new IllegalArgumentException("Unknown state " + state); + } + } + + @Override + public TokenIterator skipChildren() { + state = State.AFTER; + value = null; + currentToken = Token.END_ARRAY; + return this; + } + + @Override + public Object currentValue() { + return value; + } + + @Override + public String currentName() { + if (parent == null || (currentToken != Token.START_ARRAY && currentToken != Token.END_ARRAY)) { + return null; + } else { + return name; + } + } + + @Override + public String childName() { + return null; + } + } +} diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java new file mode 100644 index 00000000000..0d2113152eb --- /dev/null +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.support.MapXContentParser; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentParserTests.generateRandomObject; + +public class MapXContentParserTests extends ESTestCase { + + public void testSimpleMap() throws IOException { + compareTokens(builder -> { + builder.startObject(); + builder.field("string", "foo"); + builder.field("number", 42); + builder.field("double", 42.5); + builder.field("bool", false); + builder.startArray("arr"); + { + builder.value(10).value(20.0).value("30"); + builder.startArray(); + builder.value(30); + builder.endArray(); + } + builder.endArray(); + builder.startArray("nested_arr"); + { + builder.startArray(); + builder.value(10); + builder.endArray(); + } + builder.endArray(); + builder.startObject("obj"); + { + builder.field("inner_string", "bar"); + builder.startObject("inner_empty_obj"); + builder.field("f", "a"); + builder.endObject(); + } + builder.endObject(); + builder.field("bytes", new byte[]{1, 2, 3}); + builder.nullField("nothing"); + builder.endObject(); + }); + } + + + public void testRandomObject() throws IOException { + compareTokens(builder -> generateRandomObject(builder, randomIntBetween(0, 10))); + } + + public void compareTokens(CheckedConsumer consumer) throws IOException { + final XContentType xContentType = randomFrom(XContentType.values()); + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + consumer.accept(builder); + final Map map; + try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { + map = parser.mapOrdered(); + } + + try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { + try (XContentParser mapParser = new MapXContentParser( + xContentRegistry(), LoggingDeprecationHandler.INSTANCE, map, xContentType)) { + assertEquals(parser.contentType(), mapParser.contentType()); + XContentParser.Token token; + assertEquals(parser.currentToken(), mapParser.currentToken()); + assertEquals(parser.currentName(), mapParser.currentName()); + do { + token = parser.nextToken(); + XContentParser.Token mapToken = mapParser.nextToken(); + assertEquals(token, mapToken); + assertEquals(parser.currentName(), mapParser.currentName()); + if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) { + assertEquals(parser.textOrNull(), mapParser.textOrNull()); + switch (token) { + case VALUE_STRING: + assertEquals(parser.text(), mapParser.text()); + break; + case VALUE_NUMBER: + assertEquals(parser.numberType(), mapParser.numberType()); + assertEquals(parser.numberValue(), mapParser.numberValue()); + if (parser.numberType() == XContentParser.NumberType.LONG || + parser.numberType() == XContentParser.NumberType.INT) { + assertEquals(parser.longValue(), mapParser.longValue()); + if (parser.longValue() <= Integer.MAX_VALUE && parser.longValue() >= Integer.MIN_VALUE) { + assertEquals(parser.intValue(), mapParser.intValue()); + if (parser.longValue() <= Short.MAX_VALUE && parser.longValue() >= Short.MIN_VALUE) { + assertEquals(parser.shortValue(), mapParser.shortValue()); + } + } + } else { + assertEquals(parser.doubleValue(), mapParser.doubleValue(), 0.000001); + } + break; + case VALUE_BOOLEAN: + assertEquals(parser.booleanValue(), mapParser.booleanValue()); + break; + case VALUE_EMBEDDED_OBJECT: + assertArrayEquals(parser.binaryValue(), mapParser.binaryValue()); + break; + case VALUE_NULL: + assertNull(mapParser.textOrNull()); + break; + } + assertEquals(parser.currentName(), mapParser.currentName()); + assertEquals(parser.isClosed(), mapParser.isClosed()); + } else if (token == XContentParser.Token.START_ARRAY || token == XContentParser.Token.START_OBJECT) { + if (randomInt(5) == 0) { + parser.skipChildren(); + mapParser.skipChildren(); + } + } + } while (token != null); + assertEquals(parser.nextToken(), mapParser.nextToken()); + parser.close(); + mapParser.close(); + assertEquals(parser.isClosed(), mapParser.isClosed()); + assertTrue(mapParser.isClosed()); + } + } + + } + } +} diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java index 8493877b148..cc5804a6ce7 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java @@ -512,7 +512,7 @@ public class XContentParserTests extends ESTestCase { * * Returns the number of tokens in the marked field */ - private int generateRandomObjectForMarking(XContentBuilder builder) throws IOException { + private static int generateRandomObjectForMarking(XContentBuilder builder) throws IOException { builder.startObject() .field("first_field", "foo") .field("marked_field"); @@ -521,7 +521,7 @@ public class XContentParserTests extends ESTestCase { return numberOfTokens; } - private int generateRandomObject(XContentBuilder builder, int level) throws IOException { + public static int generateRandomObject(XContentBuilder builder, int level) throws IOException { int tokens = 2; builder.startObject(); int numberOfElements = randomInt(5); @@ -533,7 +533,7 @@ public class XContentParserTests extends ESTestCase { return tokens; } - private int generateRandomValue(XContentBuilder builder, int level) throws IOException { + private static int generateRandomValue(XContentBuilder builder, int level) throws IOException { @SuppressWarnings("unchecked") CheckedSupplier fieldGenerator = randomFrom( () -> { builder.value(randomInt()); @@ -568,7 +568,7 @@ public class XContentParserTests extends ESTestCase { return fieldGenerator.get(); } - private int generateRandomArray(XContentBuilder builder, int level) throws IOException { + private static int generateRandomArray(XContentBuilder builder, int level) throws IOException { int tokens = 2; int arraySize = randomInt(3); builder.startArray(); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index f990a9750e0..33e4140a08e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -23,15 +23,13 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.XContentSubParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.MapXContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.geo.geometry.Rectangle; import org.elasticsearch.geo.utils.Geohash; @@ -43,7 +41,7 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; import java.io.IOException; -import java.io.InputStream; +import java.util.Collections; public class GeoUtils { @@ -376,21 +374,12 @@ public class GeoUtils { * Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true */ public static GeoPoint parseGeoPoint(Object value, final boolean ignoreZValue) throws ElasticsearchParseException { - try { - XContentBuilder content = JsonXContent.contentBuilder(); - content.startObject(); - content.field("null_value", value); - content.endObject(); - - try (InputStream stream = BytesReference.bytes(content).streamInput(); - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - parser.nextToken(); // start object - parser.nextToken(); // field name - parser.nextToken(); // field value - return parseGeoPoint(parser, new GeoPoint(), ignoreZValue); - } - + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("null_value", value), null)) { + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + return parseGeoPoint(parser, new GeoPoint(), ignoreZValue); } catch (IOException ex) { throw new ElasticsearchParseException("error parsing geopoint", ex); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java index 9299edc459c..4a976d19b23 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java @@ -20,18 +20,16 @@ package org.elasticsearch.common.geo.parsers; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.MapXContentParser; import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import java.io.IOException; -import java.io.InputStream; +import java.util.Collections; /** * first point of entry for a shape parser @@ -75,14 +73,8 @@ public interface ShapeParser { } static ShapeBuilder parse(Object value) throws IOException { - XContentBuilder content = JsonXContent.contentBuilder(); - content.startObject(); - content.field("value", value); - content.endObject(); - - try (InputStream stream = BytesReference.bytes(content).streamInput(); - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("value", value), null)) { parser.nextToken(); // start object parser.nextToken(); // field name parser.nextToken(); // field value