From d40831397409ccea5352e2d52e6701a6b8dc0eae Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 9 Dec 2015 21:52:39 +0100 Subject: [PATCH] Make XContentGenerator.writeRaw* safer. This method currently allows to write arbitrary bytes in an xcontent stream. I changed it so that it can only write data to the same stream as the xcontent (the bos parameter is removed) and that it yells at you if you try to write raw bytes that can't be recognized as xcontent. Also the logic to copy the structure instead of appending the bytes directly if the source and target are of a different xcontent type have been moved to the low-level XContentGenerator. --- .../percolate/PercolateSourceBuilder.java | 12 +- .../common/xcontent/XContent.java | 5 - .../common/xcontent/XContentBuilder.java | 33 ++--- .../common/xcontent/XContentGenerator.java | 9 +- .../common/xcontent/XContentHelper.java | 74 +---------- .../common/xcontent/cbor/CborXContent.java | 9 +- .../xcontent/cbor/CborXContentGenerator.java | 51 +------- .../common/xcontent/json/JsonXContent.java | 9 +- .../xcontent/json/JsonXContentGenerator.java | 111 ++++++++-------- .../common/xcontent/smile/SmileXContent.java | 9 +- .../smile/SmileXContentGenerator.java | 51 +------- .../common/xcontent/yaml/YamlXContent.java | 9 +- .../xcontent/yaml/YamlXContentGenerator.java | 51 +------- .../rest/action/get/RestGetSourceAction.java | 3 +- .../GetTermVectorsCheckDocFreqIT.java | 9 +- .../common/xcontent/BaseXContentTestCase.java | 115 +++++++++++++++++ .../builder/BuilderRawFieldTests.java | 122 ------------------ .../builder/XContentBuilderTests.java | 19 +-- .../xcontent/cbor/CborXContentTests.java | 32 +++++ .../xcontent/json/JsonXContentTests.java | 32 +++++ .../xcontent/smile/SmileXContentTests.java | 32 +++++ ...bstractFilteringJsonGeneratorTestCase.java | 11 +- .../xcontent/yaml/YamlXContentTests.java | 32 +++++ .../elasticsearch/test/ESIntegTestCase.java | 2 +- 24 files changed, 357 insertions(+), 485 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java delete mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java create mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java create mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java create mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java create mode 100644 core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java index 967d5a0a2b7..79f51db59f7 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java @@ -238,17 +238,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - XContentType contentType = XContentFactory.xContentType(doc); - if (contentType == builder.contentType()) { - builder.rawField("doc", doc); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(doc)) { - parser.nextToken(); - builder.field("doc"); - builder.copyCurrentStructure(parser); - } - } - return builder; + return builder.rawField("doc", doc); } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 101098d67a4..50c04930901 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -45,11 +45,6 @@ public interface XContent { */ XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException; - /** - * Creates a new generator using the provided writer. - */ - XContentGenerator createGenerator(Writer writer) throws IOException; - /** * Creates a parser over the provided string content. */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 2bd4b9e3206..af8e7534692 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -920,23 +920,18 @@ public final class XContentBuilder implements BytesStream, Releasable { return this; } - public XContentBuilder rawField(String fieldName, byte[] content) throws IOException { - generator.writeRawField(fieldName, content, bos); - return this; - } - - public XContentBuilder rawField(String fieldName, byte[] content, int offset, int length) throws IOException { - generator.writeRawField(fieldName, content, offset, length, bos); - return this; - } - - public XContentBuilder rawField(String fieldName, InputStream content, XContentType contentType) throws IOException { - generator.writeRawField(fieldName, content, bos, contentType); + public XContentBuilder rawField(String fieldName, InputStream content) throws IOException { + generator.writeRawField(fieldName, content); return this; } public XContentBuilder rawField(String fieldName, BytesReference content) throws IOException { - generator.writeRawField(fieldName, content, bos); + generator.writeRawField(fieldName, content); + return this; + } + + public XContentBuilder rawValue(BytesReference content) throws IOException { + generator.writeRawValue(content); return this; } @@ -1202,24 +1197,12 @@ public final class XContentBuilder implements BytesStream, Releasable { return this.generator; } - public OutputStream stream() { - return this.bos; - } - @Override public BytesReference bytes() { close(); return ((BytesStream) bos).bytes(); } - /** - * Returns the actual stream used. - */ - public BytesStream bytesStream() throws IOException { - close(); - return (BytesStream) bos; - } - /** * Returns a string representation of the builder (only applicable for text based xcontent). */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 9843a19948e..11a42e37279 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.bytes.BytesReference; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; /** * @@ -112,13 +111,11 @@ public interface XContentGenerator extends Closeable { void writeObjectFieldStart(XContentString fieldName) throws IOException; - void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException; + void writeRawField(String fieldName, InputStream content) throws IOException; - void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException; + void writeRawField(String fieldName, BytesReference content) throws IOException; - void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException; - - void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException; + void writeRawValue(BytesReference content) throws IOException; void copyCurrentStructure(XContentParser parser) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index f16332f07dd..4466d295735 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.ToXContent.Params; import java.io.BufferedInputStream; @@ -102,9 +101,7 @@ public class XContentHelper { BytesArray bytesArray = bytes.toBytesArray(); return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), StandardCharsets.UTF_8); } - XContentParser parser = null; - try { - parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput()); + try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) { parser.nextToken(); XContentBuilder builder = XContentFactory.jsonBuilder(); if (prettyPrint) { @@ -112,10 +109,6 @@ public class XContentHelper { } builder.copyCurrentStructure(parser); return builder.string(); - } finally { - if (parser != null) { - parser.close(); - } } } @@ -128,9 +121,7 @@ public class XContentHelper { if (xContentType == XContentType.JSON && !reformatJson) { return new String(data, offset, length, StandardCharsets.UTF_8); } - XContentParser parser = null; - try { - parser = XContentFactory.xContent(xContentType).createParser(data, offset, length); + try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(data, offset, length)) { parser.nextToken(); XContentBuilder builder = XContentFactory.jsonBuilder(); if (prettyPrint) { @@ -138,10 +129,6 @@ public class XContentHelper { } builder.copyCurrentStructure(parser); return builder.string(); - } finally { - if (parser != null) { - parser.close(); - } } } @@ -378,38 +365,6 @@ public class XContentHelper { } } - /** - * Directly writes the source to the output builder - */ - public static void writeDirect(BytesReference source, XContentBuilder rawBuilder, ToXContent.Params params) throws IOException { - Compressor compressor = CompressorFactory.compressor(source); - if (compressor != null) { - InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - if (contentType == rawBuilder.contentType()) { - Streams.copy(compressedStreamInput, rawBuilder.stream()); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) { - parser.nextToken(); - rawBuilder.copyCurrentStructure(parser); - } - } - } else { - XContentType contentType = XContentFactory.xContentType(source); - if (contentType == rawBuilder.contentType()) { - source.writeTo(rawBuilder.stream()); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) { - parser.nextToken(); - rawBuilder.copyCurrentStructure(parser); - } - } - } - } - /** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}. @@ -418,30 +373,9 @@ public class XContentHelper { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - if (contentType == builder.contentType()) { - builder.rawField(field, compressedStreamInput, contentType); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) { - parser.nextToken(); - builder.field(field); - builder.copyCurrentStructure(parser); - } - } + builder.rawField(field, compressedStreamInput); } else { - XContentType contentType = XContentFactory.xContentType(source); - if (contentType == builder.contentType()) { - builder.rawField(field, source); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) { - parser.nextToken(); - builder.field(field); - builder.copyCurrentStructure(parser); - } - } + builder.rawField(field, source); } } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index 5417675738e..5f8dddc5b56 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -61,17 +61,12 @@ public class CborXContent implements XContent { @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8)); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), filters); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return new CborXContentGenerator(cborFactory.createGenerator(writer)); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java index a3ca669b156..517266b8170 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java @@ -20,13 +20,9 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.dataformat.cbor.CBORParser; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class CborXContentGenerator extends JsonXContentGenerator { - public CborXContentGenerator(JsonGenerator jsonGenerator, String... filters) { - super(jsonGenerator, filters); + public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class CborXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - CBORParser parser; - if (content.hasArray()) { - parser = CborXContent.cborFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = CborXContent.cborFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 671dee086f6..86f5bc28a7d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -65,17 +65,12 @@ public class JsonXContent implements XContent { @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8)); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), filters); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return new JsonXContentGenerator(jsonFactory.createGenerator(writer)); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 7f3cc6334b8..0854f7a6d2b 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -26,13 +26,13 @@ import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; import com.fasterxml.jackson.core.io.SerializedString; import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -58,11 +58,14 @@ public class JsonXContentGenerator implements XContentGenerator { */ private final FilteringGeneratorDelegate filter; + private final OutputStream os; + private boolean writeLineFeedAtEnd; private static final SerializedString LF = new SerializedString("\n"); private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); + private boolean prettyPrint = false; - public JsonXContentGenerator(JsonGenerator jsonGenerator, String... filters) { + public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { if (jsonGenerator instanceof GeneratorBase) { this.base = (GeneratorBase) jsonGenerator; } else { @@ -76,6 +79,8 @@ public class JsonXContentGenerator implements XContentGenerator { this.filter = new FilteringGeneratorDelegate(jsonGenerator, new FilterPathBasedFilter(filters), true, true); this.generator = this.filter; } + + this.os = os; } @Override @@ -86,6 +91,7 @@ public class JsonXContentGenerator implements XContentGenerator { @Override public final void usePrettyPrint() { generator.setPrettyPrinter(new DefaultPrettyPrinter().withObjectIndenter(INDENTER)); + prettyPrint = true; } @Override @@ -323,22 +329,16 @@ public class JsonXContentGenerator implements XContentGenerator { } @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeRawField(fieldName, new BytesArray(content), bos); - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeRawField(fieldName, new BytesArray(content, offset, length), bos); - } - - @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { - if (isFiltered() || (contentType != contentType())) { - // When the current generator is filtered (ie filter != null) - // or the content is in a different format than the current generator, - // we need to copy the whole structure so that it will be correctly - // filtered or converted + public void writeRawField(String fieldName, InputStream content) throws IOException { + if (content.markSupported() == false) { + // needed for the XContentFactory.xContentType call + content = new BufferedInputStream(content); + } + XContentType contentType = XContentFactory.xContentType(content); + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) { parser.nextToken(); writeFieldName(fieldName); @@ -347,49 +347,59 @@ public class JsonXContentGenerator implements XContentGenerator { } else { writeStartRaw(fieldName); flush(); - Streams.copy(content, bos); + Streams.copy(content, os); writeEndRaw(); } } @Override - public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException { + public final void writeRawField(String fieldName, BytesReference content) throws IOException { XContentType contentType = XContentFactory.xContentType(content); - if (contentType != null) { - if (isFiltered() || (contentType != contentType())) { - // When the current generator is filtered (ie filter != null) - // or the content is in a different format than the current generator, - // we need to copy the whole structure so that it will be correctly - // filtered or converted - copyRawField(fieldName, content, contentType.xContent()); - } else { - // Otherwise, the generator is not filtered and has the same type: we can potentially optimize the write - writeObjectRaw(fieldName, content, bos); - } - } else { + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { writeFieldName(fieldName); - // we could potentially optimize this to not rely on exception logic... - String sValue = content.toUtf8(); - try { - writeNumber(Long.parseLong(sValue)); - } catch (NumberFormatException e) { - try { - writeNumber(Double.parseDouble(sValue)); - } catch (NumberFormatException e1) { - writeString(sValue); - } - } + copyRawValue(content, contentType.xContent()); + } else { + writeStartRaw(fieldName); + flush(); + content.writeTo(os); + writeEndRaw(); } } - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeStartRaw(fieldName); - flush(); - content.writeTo(bos); - writeEndRaw(); + public final void writeRawValue(BytesReference content) throws IOException { + XContentType contentType = XContentFactory.xContentType(content); + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { + copyRawValue(content, contentType.xContent()); + } else { + flush(); + content.writeTo(os); + writeEndRaw(); + } } - protected void copyRawField(String fieldName, BytesReference content, XContent xContent) throws IOException { + private boolean mayWriteRawData(XContentType contentType) { + // When the current generator is filtered (ie filter != null) + // or the content is in a different format than the current generator, + // we need to copy the whole structure so that it will be correctly + // filtered or converted + return supportsRawWrites() + && isFiltered() == false + && contentType == contentType() + && prettyPrint == false; + } + + /** Whether this generator supports writing raw data directly */ + protected boolean supportsRawWrites() { + return true; + } + + protected void copyRawValue(BytesReference content, XContent xContent) throws IOException { XContentParser parser = null; try { if (content.hasArray()) { @@ -397,9 +407,6 @@ public class JsonXContentGenerator implements XContentGenerator { } else { parser = xContent.createParser(content.streamInput()); } - if (fieldName != null) { - writeFieldName(fieldName); - } copyCurrentStructure(parser); } finally { if (parser != null) { diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index cb1bdd3bd4d..51b85900f99 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -62,17 +62,12 @@ public class SmileXContent implements XContent { @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8)); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), filters); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return new SmileXContentGenerator(smileFactory.createGenerator(writer)); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java index 5002cfac645..451abab33d5 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java @@ -20,13 +20,9 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.dataformat.smile.SmileParser; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class SmileXContentGenerator extends JsonXContentGenerator { - public SmileXContentGenerator(JsonGenerator jsonGenerator, String... filters) { - super(jsonGenerator, filters); + public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class SmileXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - SmileParser parser; - if (content.hasArray()) { - parser = SmileXContent.smileFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = SmileXContent.smileFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index d79a6a8ba80..c24ddb7f296 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -60,17 +60,12 @@ public class YamlXContent implements XContent { @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8)); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), filters); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return new YamlXContentGenerator(yamlFactory.createGenerator(writer)); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java index e1a71c143b0..dcb2155c82e 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java @@ -20,13 +20,9 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class YamlXContentGenerator extends JsonXContentGenerator { - public YamlXContentGenerator(JsonGenerator jsonGenerator, String... filters) { - super(jsonGenerator, filters); + public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class YamlXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - YAMLParser parser; - if (content.hasArray()) { - parser = YamlXContent.yamlFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = YamlXContent.yamlFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java index 1fe07156d99..c0e45fc6aeb 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java @@ -26,7 +26,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestResponseListener; import org.elasticsearch.search.fetch.source.FetchSourceContext; @@ -77,7 +76,7 @@ public class RestGetSourceAction extends BaseRestHandler { if (!response.isExists()) { return new BytesRestResponse(NOT_FOUND, builder); } else { - XContentHelper.writeDirect(response.getSourceInternal(), builder, request); + builder.rawValue(response.getSourceInternal()); return new BytesRestResponse(OK, builder); } } diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java index 28d4b0f4c45..0eb7c0757e7 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java @@ -141,8 +141,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -198,8 +197,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -258,8 +256,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java new file mode 100644 index 00000000000..1b917263b79 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public abstract class BaseXContentTestCase extends ESTestCase { + + public abstract XContentType xcontentType(); + + public void testBasics() throws IOException { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeStartObject(); + generator.writeEndObject(); + } + byte[] data = os.toByteArray(); + assertEquals(xcontentType(), XContentFactory.xContentType(data)); + } + + public void testRawField() throws Exception { + for (boolean useStream : new boolean[] {false, true}) { + for (XContentType xcontentType : XContentType.values()) { + doTestRawField(xcontentType.xContent(), useStream); + } + } + } + + void doTestRawField(XContent source, boolean useStream) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = source.createGenerator(os)) { + generator.writeStartObject(); + generator.writeFieldName("foo"); + generator.writeNull(); + generator.writeEndObject(); + } + final byte[] rawData = os.toByteArray(); + + os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeStartObject(); + if (useStream) { + generator.writeRawField("bar", new ByteArrayInputStream(rawData)); + } else { + generator.writeRawField("bar", new BytesArray(rawData)); + } + generator.writeEndObject(); + } + + XContentParser parser = xcontentType().xContent().createParser(os.toByteArray()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("bar", parser.currentName()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("foo", parser.currentName()); + assertEquals(Token.VALUE_NULL, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } + + public void testRawValue() throws Exception { + for (XContentType xcontentType : XContentType.values()) { + doTestRawValue(xcontentType.xContent()); + } + } + + void doTestRawValue(XContent source) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = source.createGenerator(os)) { + generator.writeStartObject(); + generator.writeFieldName("foo"); + generator.writeNull(); + generator.writeEndObject(); + } + final byte[] rawData = os.toByteArray(); + + os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeRawValue(new BytesArray(rawData)); + } + + XContentParser parser = xcontentType().xContent().createParser(os.toByteArray()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("foo", parser.currentName()); + assertEquals(Token.VALUE_NULL, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java deleted file mode 100644 index 9bb26b635ef..00000000000 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.builder; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -public class BuilderRawFieldTests extends ESTestCase { - public void testJsonRawField() throws IOException { - testRawField(XContentType.JSON); - } - - public void testSmileRawField() throws IOException { - testRawField(XContentType.SMILE); - } - - public void testYamlRawField() throws IOException { - testRawField(XContentType.YAML); - } - - public void testCborRawField() throws IOException { - testRawField(XContentType.CBOR); - } - - private void testRawField(XContentType type) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(type); - builder.startObject(); - builder.field("field1", "value1"); - builder.rawField("_source", XContentFactory.contentBuilder(type).startObject().field("s_field", "s_value").endObject().bytes()); - builder.field("field2", "value2"); - builder.rawField("payload_i", new BytesArray(Long.toString(1))); - builder.field("field3", "value3"); - builder.rawField("payload_d", new BytesArray(Double.toString(1.1))); - builder.field("field4", "value4"); - builder.rawField("payload_s", new BytesArray("test")); - builder.field("field5", "value5"); - builder.endObject(); - - XContentParser parser = XContentFactory.xContent(type).createParser(builder.bytes()); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field1")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value1")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("_source")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("s_field")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("s_value")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field2")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value2")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_i")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER)); - assertThat(parser.numberType(), equalTo(XContentParser.NumberType.INT)); - assertThat(parser.longValue(), equalTo(1l)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field3")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value3")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_d")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER)); - assertThat(parser.numberType(), equalTo(XContentParser.NumberType.DOUBLE)); - assertThat(parser.doubleValue(), equalTo(1.1d)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field4")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value4")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_s")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("test")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field5")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value5")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java index d6cec176fc0..7ffafc004ab 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.xcontent.builder; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.FastCharArrayWriter; @@ -32,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; @@ -54,8 +56,8 @@ import static org.hamcrest.Matchers.equalTo; */ public class XContentBuilderTests extends ESTestCase { public void testPrettyWithLfAtEnd() throws Exception { - FastCharArrayWriter writer = new FastCharArrayWriter(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); generator.usePrettyPrint(); generator.usePrintLineFeedAtEnd(); @@ -68,27 +70,28 @@ public class XContentBuilderTests extends ESTestCase { // double close, and check there is no error... generator.close(); - assertThat(writer.unsafeCharArray()[writer.size() - 1], equalTo('\n')); + byte[] bytes = os.toByteArray(); + assertThat((char) bytes[bytes.length - 1], equalTo('\n')); } public void testReuseJsonGenerator() throws Exception { - FastCharArrayWriter writer = new FastCharArrayWriter(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); generator.flush(); - assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}")); + assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef("{\"test\":\"value\"}"))); // try again... - writer.reset(); + os.reset(); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); generator.flush(); // we get a space at the start here since it thinks we are not in the root object (fine, we will ignore it in the real code we use) - assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}")); + assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef(" {\"test\":\"value\"}"))); } public void testRaw() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java new file mode 100644 index 00000000000..928b8a6a5a9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.cbor; + +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; + +public class CborXContentTests extends BaseXContentTestCase { + + @Override + public XContentType xcontentType() { + return XContentType.CBOR; + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java new file mode 100644 index 00000000000..8a739eef4b8 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.json; + +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; + +public class JsonXContentTests extends BaseXContentTestCase { + + @Override + public XContentType xcontentType() { + return XContentType.JSON; + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java new file mode 100644 index 00000000000..6961e84416d --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.smile; + +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; + +public class SmileXContentTests extends BaseXContentTestCase { + + @Override + public XContentType xcontentType() { + return XContentType.SMILE; + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index 9216f48bb26..ed7aee33eba 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -475,15 +475,10 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw).endObject()); - // Test method: rawField(String fieldName, byte[] content) - assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - // Test method: rawField(String fieldName, InputStream content) - assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); + assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); } public void testArrays() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java new file mode 100644 index 00000000000..17c2a590ec1 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.yaml; + +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; + +public class YamlXContentTests extends BaseXContentTestCase { + + @Override + public XContentType xcontentType() { + return XContentType.YAML; + } + +} diff --git a/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index cc5348bc6bc..05dab1c35dc 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -450,7 +450,7 @@ public abstract class ESIntegTestCase extends ESTestCase { .setOrder(0) .setSettings(randomSettingsBuilder); if (mappings != null) { - logger.info("test using _default_ mappings: [{}]", mappings.bytesStream().bytes().toUtf8()); + logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet());