Merge pull request #15358 from jpountz/fix/safe_write_raw

Make XContentGenerator.writeRaw* safer.

core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java

@@ -238,17 +238,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            XContentType contentType = XContentFactory.xContentType(doc);
+            return builder.rawField("doc", doc);
-            if (contentType == builder.contentType()) {
-                builder.rawField("doc", doc);
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(doc)) {
-                    parser.nextToken();
-                    builder.field("doc");
-                    builder.copyCurrentStructure(parser);
-                }
-            }
-            return builder;
         }
     }
 

core/src/main/java/org/elasticsearch/common/xcontent/XContent.java

@@ -45,11 +45,6 @@ public interface XContent {
      */
     XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException;
 
-    /**
-     * Creates a new generator using the provided writer.
-     */
-    XContentGenerator createGenerator(Writer writer) throws IOException;
-
     /**
      * Creates a parser over the provided string content.
      */

core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -920,23 +920,18 @@ public final class XContentBuilder implements BytesStream, Releasable {
         return this;
     }
 
-    public XContentBuilder rawField(String fieldName, byte[] content) throws IOException {
+    public XContentBuilder rawField(String fieldName, InputStream content) throws IOException {
-        generator.writeRawField(fieldName, content, bos);
+        generator.writeRawField(fieldName, content);
-        return this;
-    }
-
-    public XContentBuilder rawField(String fieldName, byte[] content, int offset, int length) throws IOException {
-        generator.writeRawField(fieldName, content, offset, length, bos);
-        return this;
-    }
-
-    public XContentBuilder rawField(String fieldName, InputStream content, XContentType contentType) throws IOException {
-        generator.writeRawField(fieldName, content, bos, contentType);
         return this;
     }
 
     public XContentBuilder rawField(String fieldName, BytesReference content) throws IOException {
-        generator.writeRawField(fieldName, content, bos);
+        generator.writeRawField(fieldName, content);
+        return this;
+    }
+
+    public XContentBuilder rawValue(BytesReference content) throws IOException {
+        generator.writeRawValue(content);
         return this;
     }
 
@@ -1202,24 +1197,12 @@ public final class XContentBuilder implements BytesStream, Releasable {
         return this.generator;
     }
 
-    public OutputStream stream() {
-        return this.bos;
-    }
-
     @Override
     public BytesReference bytes() {
         close();
         return ((BytesStream) bos).bytes();
     }
 
-    /**
-     * Returns the actual stream used.
-     */
-    public BytesStream bytesStream() throws IOException {
-        close();
-        return (BytesStream) bos;
-    }
-
     /**
      * Returns a string representation of the builder (only applicable for text based xcontent).
      */

core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java

@@ -24,7 +24,6 @@ import org.elasticsearch.common.bytes.BytesReference;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 
 /**
  *
@@ -112,13 +111,11 @@ public interface XContentGenerator extends Closeable {
 
     void writeObjectFieldStart(XContentString fieldName) throws IOException;
 
-    void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException;
+    void writeRawField(String fieldName, InputStream content) throws IOException;
 
-    void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException;
+    void writeRawField(String fieldName, BytesReference content) throws IOException;
 
-    void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException;
+    void writeRawValue(BytesReference content) throws IOException;
-
-    void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException;
 
     void copyCurrentStructure(XContentParser parser) throws IOException;
 

core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java

@@ -27,7 +27,6 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.ToXContent.Params;
 
 import java.io.BufferedInputStream;
@@ -102,9 +101,7 @@ public class XContentHelper {
             BytesArray bytesArray = bytes.toBytesArray();
             return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), StandardCharsets.UTF_8);
         }
-        XContentParser parser = null;
+        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) {
-        try {
-            parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput());
             parser.nextToken();
             XContentBuilder builder = XContentFactory.jsonBuilder();
             if (prettyPrint) {
@@ -112,10 +109,6 @@ public class XContentHelper {
             }
             builder.copyCurrentStructure(parser);
             return builder.string();
-        } finally {
-            if (parser != null) {
-                parser.close();
-            }
         }
     }
 
@@ -128,9 +121,7 @@ public class XContentHelper {
         if (xContentType == XContentType.JSON && !reformatJson) {
             return new String(data, offset, length, StandardCharsets.UTF_8);
         }
-        XContentParser parser = null;
+        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(data, offset, length)) {
-        try {
-            parser = XContentFactory.xContent(xContentType).createParser(data, offset, length);
             parser.nextToken();
             XContentBuilder builder = XContentFactory.jsonBuilder();
             if (prettyPrint) {
@@ -138,10 +129,6 @@ public class XContentHelper {
             }
             builder.copyCurrentStructure(parser);
             return builder.string();
-        } finally {
-            if (parser != null) {
-                parser.close();
-            }
         }
     }
 
@@ -378,38 +365,6 @@ public class XContentHelper {
         }
     }
 
-    /**
-     * Directly writes the source to the output builder
-     */
-    public static void writeDirect(BytesReference source, XContentBuilder rawBuilder, ToXContent.Params params) throws IOException {
-        Compressor compressor = CompressorFactory.compressor(source);
-        if (compressor != null) {
-            InputStream compressedStreamInput = compressor.streamInput(source.streamInput());
-            if (compressedStreamInput.markSupported() == false) {
-                compressedStreamInput = new BufferedInputStream(compressedStreamInput);
-            }
-            XContentType contentType = XContentFactory.xContentType(compressedStreamInput);
-            if (contentType == rawBuilder.contentType()) {
-                Streams.copy(compressedStreamInput, rawBuilder.stream());
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) {
-                    parser.nextToken();
-                    rawBuilder.copyCurrentStructure(parser);
-                }
-            }
-        } else {
-            XContentType contentType = XContentFactory.xContentType(source);
-            if (contentType == rawBuilder.contentType()) {
-                source.writeTo(rawBuilder.stream());
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) {
-                    parser.nextToken();
-                    rawBuilder.copyCurrentStructure(parser);
-                }
-            }
-        }
-    }
-
     /**
      * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
      * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}.
@@ -418,30 +373,9 @@ public class XContentHelper {
         Compressor compressor = CompressorFactory.compressor(source);
         if (compressor != null) {
             InputStream compressedStreamInput = compressor.streamInput(source.streamInput());
-            if (compressedStreamInput.markSupported() == false) {
+            builder.rawField(field, compressedStreamInput);
-                compressedStreamInput = new BufferedInputStream(compressedStreamInput);
-            }
-            XContentType contentType = XContentFactory.xContentType(compressedStreamInput);
-            if (contentType == builder.contentType()) {
-                builder.rawField(field, compressedStreamInput, contentType);
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) {
-                    parser.nextToken();
-                    builder.field(field);
-                    builder.copyCurrentStructure(parser);
-                }
-            }
         } else {
-            XContentType contentType = XContentFactory.xContentType(source);
+            builder.rawField(field, source);
-            if (contentType == builder.contentType()) {
-                builder.rawField(field, source);
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) {
-                    parser.nextToken();
-                    builder.field(field);
-                    builder.copyCurrentStructure(parser);
-                }
-            }
         }
     }
 }

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java

@@ -61,17 +61,12 @@ public class CborXContent implements XContent {
 
     @Override
     public XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8));
+        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os);
     }
 
     @Override
     public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), filters);
+        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters);
-    }
-
-    @Override
-    public XContentGenerator createGenerator(Writer writer) throws IOException {
-        return new CborXContentGenerator(cborFactory.createGenerator(writer));
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java

@@ -20,13 +20,9 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.dataformat.cbor.CBORParser;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 
 /**
@@ -34,8 +30,8 @@ import java.io.OutputStream;
  */
 public class CborXContentGenerator extends JsonXContentGenerator {
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, String... filters) {
+    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        super(jsonGenerator, filters);
+        super(jsonGenerator, os, filters);
     }
 
     @Override
@@ -49,46 +45,7 @@ public class CborXContentGenerator extends JsonXContentGenerator {
     }
 
     @Override
-    public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException {
+    protected boolean supportsRawWrites() {
-        writeFieldName(fieldName);
+        return false;
-        try (CBORParser parser = CborXContent.cborFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (CBORParser parser = CborXContent.cborFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        CBORParser parser;
-        if (content.hasArray()) {
-            parser = CborXContent.cborFactory.createParser(content.array(), content.arrayOffset(), content.length());
-        } else {
-            parser = CborXContent.cborFactory.createParser(content.streamInput());
-        }
-        try {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        } finally {
-            parser.close();
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (CBORParser parser = CborXContent.cborFactory.createParser(content, offset, length)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
     }
 }

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -65,17 +65,12 @@ public class JsonXContent implements XContent {
 
     @Override
     public XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8));
+        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os);
     }
 
     @Override
     public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), filters);
+        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters);
-    }
-
-    @Override
-    public XContentGenerator createGenerator(Writer writer) throws IOException {
-        return new JsonXContentGenerator(jsonFactory.createGenerator(writer));
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java

@@ -26,13 +26,13 @@ import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate;
 import com.fasterxml.jackson.core.io.SerializedString;
 import com.fasterxml.jackson.core.util.DefaultIndenter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.*;
 import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -58,11 +58,14 @@ public class JsonXContentGenerator implements XContentGenerator {
      */
     private final FilteringGeneratorDelegate filter;
 
+    private final OutputStream os;
+
     private boolean writeLineFeedAtEnd;
     private static final SerializedString LF = new SerializedString("\n");
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
+    private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, String... filters) {
+    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
         if (jsonGenerator instanceof GeneratorBase) {
             this.base = (GeneratorBase) jsonGenerator;
         } else {
@@ -76,6 +79,8 @@ public class JsonXContentGenerator implements XContentGenerator {
             this.filter = new FilteringGeneratorDelegate(jsonGenerator, new FilterPathBasedFilter(filters), true, true);
             this.generator = this.filter;
         }
+
+        this.os = os;
     }
 
     @Override
@@ -86,6 +91,7 @@ public class JsonXContentGenerator implements XContentGenerator {
     @Override
     public final void usePrettyPrint() {
         generator.setPrettyPrinter(new DefaultPrettyPrinter().withObjectIndenter(INDENTER));
+        prettyPrint = true;
     }
 
     @Override
@@ -323,22 +329,16 @@ public class JsonXContentGenerator implements XContentGenerator {
     }
 
     @Override
-    public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException {
+    public void writeRawField(String fieldName, InputStream content) throws IOException {
-        writeRawField(fieldName, new BytesArray(content), bos);
+        if (content.markSupported() == false) {
-    }
+            // needed for the XContentFactory.xContentType call
-
+            content = new BufferedInputStream(content);
-    @Override
+        }
-    public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException {
+        XContentType contentType = XContentFactory.xContentType(content);
-        writeRawField(fieldName, new BytesArray(content, offset, length), bos);
+        if (contentType == null) {
-    }
+            throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
-
+        }
-    @Override
+        if (mayWriteRawData(contentType) == false) {
-    public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException {
-        if (isFiltered() || (contentType != contentType())) {
-            // When the current generator is filtered (ie filter != null)
-            // or the content is in a different format than the current generator,
-            // we need to copy the whole structure so that it will be correctly
-            // filtered or converted
             try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) {
                 parser.nextToken();
                 writeFieldName(fieldName);
@@ -347,49 +347,59 @@ public class JsonXContentGenerator implements XContentGenerator {
         } else {
             writeStartRaw(fieldName);
             flush();
-            Streams.copy(content, bos);
+            Streams.copy(content, os);
             writeEndRaw();
         }
     }
 
     @Override
-    public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
+    public final void writeRawField(String fieldName, BytesReference content) throws IOException {
         XContentType contentType = XContentFactory.xContentType(content);
-        if (contentType != null) {
+        if (contentType == null) {
-            if (isFiltered() || (contentType != contentType())) {
+            throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
-                // When the current generator is filtered (ie filter != null)
+        }
-                // or the content is in a different format than the current generator,
+        if (mayWriteRawData(contentType) == false) {
-                // we need to copy the whole structure so that it will be correctly
-                // filtered or converted
-                copyRawField(fieldName, content, contentType.xContent());
-            } else {
-                // Otherwise, the generator is not filtered and has the same type: we can potentially optimize the write
-                writeObjectRaw(fieldName, content, bos);
-            }
-        } else {
             writeFieldName(fieldName);
-            // we could potentially optimize this to not rely on exception logic...
+            copyRawValue(content, contentType.xContent());
-            String sValue = content.toUtf8();
+        } else {
-            try {
+            writeStartRaw(fieldName);
-                writeNumber(Long.parseLong(sValue));
+            flush();
-            } catch (NumberFormatException e) {
+            content.writeTo(os);
-                try {
+            writeEndRaw();
-                    writeNumber(Double.parseDouble(sValue));
-                } catch (NumberFormatException e1) {
-                    writeString(sValue);
-                }
-            }
         }
     }
 
-    protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
+    public final void writeRawValue(BytesReference content) throws IOException {
-        writeStartRaw(fieldName);
+        XContentType contentType = XContentFactory.xContentType(content);
-        flush();
+        if (contentType == null) {
-        content.writeTo(bos);
+            throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
-        writeEndRaw();
+        }
+        if (mayWriteRawData(contentType) == false) {
+            copyRawValue(content, contentType.xContent());
+        } else {
+            flush();
+            content.writeTo(os);
+            writeEndRaw();
+        }
     }
 
-    protected void copyRawField(String fieldName, BytesReference content, XContent xContent) throws IOException {
+    private boolean mayWriteRawData(XContentType contentType) {
+        // When the current generator is filtered (ie filter != null)
+        // or the content is in a different format than the current generator,
+        // we need to copy the whole structure so that it will be correctly
+        // filtered or converted
+        return supportsRawWrites()
+                && isFiltered() == false
+                && contentType == contentType()
+                && prettyPrint == false;
+    }
+
+    /** Whether this generator supports writing raw data directly */
+    protected boolean supportsRawWrites() {
+        return true;
+    }
+
+    protected void copyRawValue(BytesReference content, XContent xContent) throws IOException {
         XContentParser parser = null;
         try {
             if (content.hasArray()) {
@@ -397,9 +407,6 @@ public class JsonXContentGenerator implements XContentGenerator {
             } else {
                 parser = xContent.createParser(content.streamInput());
             }
-            if (fieldName != null) {
-                writeFieldName(fieldName);
-            }
             copyCurrentStructure(parser);
         } finally {
             if (parser != null) {

core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java

@@ -62,17 +62,12 @@ public class SmileXContent implements XContent {
 
     @Override
     public XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8));
+        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os);
     }
 
     @Override
     public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), filters);
+        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters);
-    }
-
-    @Override
-    public XContentGenerator createGenerator(Writer writer) throws IOException {
-        return new SmileXContentGenerator(smileFactory.createGenerator(writer));
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java

@@ -20,13 +20,9 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.dataformat.smile.SmileParser;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 
 /**
@@ -34,8 +30,8 @@ import java.io.OutputStream;
  */
 public class SmileXContentGenerator extends JsonXContentGenerator {
 
-    public SmileXContentGenerator(JsonGenerator jsonGenerator, String... filters) {
+    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        super(jsonGenerator, filters);
+        super(jsonGenerator, os, filters);
     }
 
     @Override
@@ -49,46 +45,7 @@ public class SmileXContentGenerator extends JsonXContentGenerator {
     }
 
     @Override
-    public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException {
+    protected boolean supportsRawWrites() {
-        writeFieldName(fieldName);
+        return false;
-        try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        SmileParser parser;
-        if (content.hasArray()) {
-            parser = SmileXContent.smileFactory.createParser(content.array(), content.arrayOffset(), content.length());
-        } else {
-            parser = SmileXContent.smileFactory.createParser(content.streamInput());
-        }
-        try {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        } finally {
-            parser.close();
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (SmileParser parser = SmileXContent.smileFactory.createParser(content, offset, length)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
     }
 }

core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java

@@ -60,17 +60,12 @@ public class YamlXContent implements XContent {
 
     @Override
     public XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8));
+        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os);
     }
 
     @Override
     public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), filters);
+        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, filters);
-    }
-
-    @Override
-    public XContentGenerator createGenerator(Writer writer) throws IOException {
-        return new YamlXContentGenerator(yamlFactory.createGenerator(writer));
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java

@@ -20,13 +20,9 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.dataformat.yaml.YAMLParser;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 
 /**
@@ -34,8 +30,8 @@ import java.io.OutputStream;
  */
 public class YamlXContentGenerator extends JsonXContentGenerator {
 
-    public YamlXContentGenerator(JsonGenerator jsonGenerator, String... filters) {
+    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        super(jsonGenerator, filters);
+        super(jsonGenerator, os, filters);
     }
 
     @Override
@@ -49,46 +45,7 @@ public class YamlXContentGenerator extends JsonXContentGenerator {
     }
 
     @Override
-    public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException {
+    protected boolean supportsRawWrites() {
-        writeFieldName(fieldName);
+        return false;
-        try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
-    }
-
-    @Override
-    protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        YAMLParser parser;
-        if (content.hasArray()) {
-            parser = YamlXContent.yamlFactory.createParser(content.array(), content.arrayOffset(), content.length());
-        } else {
-            parser = YamlXContent.yamlFactory.createParser(content.streamInput());
-        }
-        try {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        } finally {
-            parser.close();
-        }
-    }
-
-    @Override
-    public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException {
-        writeFieldName(fieldName);
-        try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content, offset, length)) {
-            parser.nextToken();
-            generator.copyCurrentStructure(parser);
-        }
     }
 }

core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java

@@ -26,7 +26,6 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.rest.*;
 import org.elasticsearch.rest.action.support.RestResponseListener;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
@@ -77,7 +76,7 @@ public class RestGetSourceAction extends BaseRestHandler {
                 if (!response.isExists()) {
                     return new BytesRestResponse(NOT_FOUND, builder);
                 } else {
-                    XContentHelper.writeDirect(response.getSourceInternal(), builder, request);
+                    builder.rawValue(response.getSourceInternal());
                     return new BytesRestResponse(OK, builder);
                 }
             }

core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java

@@ -141,8 +141,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, null);
         xBuilder.endObject();
-        BytesStream bytesStream = xBuilder.bytesStream();
+        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
-        String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";
@@ -198,8 +197,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, null);
         xBuilder.endObject();
-        BytesStream bytesStream = xBuilder.bytesStream();
+        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
-        String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";
@@ -258,8 +256,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS);
         xBuilder.endObject();
-        BytesStream bytesStream = xBuilder.bytesStream();
+        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
-        String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";

core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public abstract class BaseXContentTestCase extends ESTestCase {
+
+    public abstract XContentType xcontentType();
+
+    public void testBasics() throws IOException {
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+            generator.writeStartObject();
+            generator.writeEndObject();
+        }
+        byte[] data = os.toByteArray();
+        assertEquals(xcontentType(), XContentFactory.xContentType(data));
+    }
+
+    public void testRawField() throws Exception {
+        for (boolean useStream : new boolean[] {false, true}) {
+            for (XContentType xcontentType : XContentType.values()) {
+                doTestRawField(xcontentType.xContent(), useStream);
+            }
+        }
+    }
+
+    void doTestRawField(XContent source, boolean useStream) throws Exception {
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = source.createGenerator(os)) {
+            generator.writeStartObject();
+            generator.writeFieldName("foo");
+            generator.writeNull();
+            generator.writeEndObject();
+        }
+        final byte[] rawData = os.toByteArray();
+
+        os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+            generator.writeStartObject();
+            if (useStream) {
+                generator.writeRawField("bar", new ByteArrayInputStream(rawData));
+            } else {
+                generator.writeRawField("bar", new BytesArray(rawData));
+            }
+            generator.writeEndObject();
+        }
+
+        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        assertEquals(Token.FIELD_NAME, parser.nextToken());
+        assertEquals("bar", parser.currentName());
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        assertEquals(Token.FIELD_NAME, parser.nextToken());
+        assertEquals("foo", parser.currentName());
+        assertEquals(Token.VALUE_NULL, parser.nextToken());
+        assertEquals(Token.END_OBJECT, parser.nextToken());
+        assertEquals(Token.END_OBJECT, parser.nextToken());
+        assertNull(parser.nextToken());
+    }
+
+    public void testRawValue() throws Exception {
+        for (XContentType xcontentType : XContentType.values()) {
+            doTestRawValue(xcontentType.xContent());
+        }
+    }
+
+    void doTestRawValue(XContent source) throws Exception {
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = source.createGenerator(os)) {
+            generator.writeStartObject();
+            generator.writeFieldName("foo");
+            generator.writeNull();
+            generator.writeEndObject();
+        }
+        final byte[] rawData = os.toByteArray();
+
+        os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+            generator.writeRawValue(new BytesArray(rawData));
+        }
+
+        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        assertEquals(Token.FIELD_NAME, parser.nextToken());
+        assertEquals("foo", parser.currentName());
+        assertEquals(Token.VALUE_NULL, parser.nextToken());
+        assertEquals(Token.END_OBJECT, parser.nextToken());
+        assertNull(parser.nextToken());
+    }
+}

core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java

@@ -1,122 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.xcontent.builder;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class BuilderRawFieldTests extends ESTestCase {
-    public void testJsonRawField() throws IOException {
-        testRawField(XContentType.JSON);
-    }
-
-    public void testSmileRawField() throws IOException {
-        testRawField(XContentType.SMILE);
-    }
-
-    public void testYamlRawField() throws IOException {
-        testRawField(XContentType.YAML);
-    }
-
-    public void testCborRawField() throws IOException {
-        testRawField(XContentType.CBOR);
-    }
-
-    private void testRawField(XContentType type) throws IOException {
-        XContentBuilder builder = XContentFactory.contentBuilder(type);
-        builder.startObject();
-        builder.field("field1", "value1");
-        builder.rawField("_source", XContentFactory.contentBuilder(type).startObject().field("s_field", "s_value").endObject().bytes());
-        builder.field("field2", "value2");
-        builder.rawField("payload_i", new BytesArray(Long.toString(1)));
-        builder.field("field3", "value3");
-        builder.rawField("payload_d", new BytesArray(Double.toString(1.1)));
-        builder.field("field4", "value4");
-        builder.rawField("payload_s", new BytesArray("test"));
-        builder.field("field5", "value5");
-        builder.endObject();
-
-        XContentParser parser = XContentFactory.xContent(type).createParser(builder.bytes());
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("field1"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("value1"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("_source"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("s_field"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("s_value"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("field2"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("value2"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("payload_i"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
-        assertThat(parser.numberType(), equalTo(XContentParser.NumberType.INT));
-        assertThat(parser.longValue(), equalTo(1l));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("field3"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("value3"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("payload_d"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
-        assertThat(parser.numberType(), equalTo(XContentParser.NumberType.DOUBLE));
-        assertThat(parser.doubleValue(), equalTo(1.1d));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("field4"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("value4"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("payload_s"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("test"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-        assertThat(parser.currentName(), equalTo("field5"));
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-        assertThat(parser.text(), equalTo("value5"));
-
-        assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
-    }
-}

core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.xcontent.builder;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.FastCharArrayWriter;
@@ -32,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -54,8 +56,8 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class XContentBuilderTests extends ESTestCase {
     public void testPrettyWithLfAtEnd() throws Exception {
-        FastCharArrayWriter writer = new FastCharArrayWriter();
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
-        XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer);
+        XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os);
         generator.usePrettyPrint();
         generator.usePrintLineFeedAtEnd();
 
@@ -68,27 +70,28 @@ public class XContentBuilderTests extends ESTestCase {
         // double close, and check there is no error...
         generator.close();
 
-        assertThat(writer.unsafeCharArray()[writer.size() - 1], equalTo('\n'));
+        byte[] bytes = os.toByteArray();
+        assertThat((char) bytes[bytes.length - 1], equalTo('\n'));
     }
 
     public void testReuseJsonGenerator() throws Exception {
-        FastCharArrayWriter writer = new FastCharArrayWriter();
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
-        XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer);
+        XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os);
         generator.writeStartObject();
         generator.writeStringField("test", "value");
         generator.writeEndObject();
         generator.flush();
 
-        assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}"));
+        assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef("{\"test\":\"value\"}")));
 
         // try again...
-        writer.reset();
+        os.reset();
         generator.writeStartObject();
         generator.writeStringField("test", "value");
         generator.writeEndObject();
         generator.flush();
         // we get a space at the start here since it thinks we are not in the root object (fine, we will ignore it in the real code we use)
-        assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}"));
+        assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef(" {\"test\":\"value\"}")));
     }
 
     public void testRaw() throws IOException {

core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.cbor;
+
+import org.elasticsearch.common.xcontent.BaseXContentTestCase;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class CborXContentTests extends BaseXContentTestCase {
+
+    @Override
+    public XContentType xcontentType() {
+        return XContentType.CBOR;
+    }
+
+}

core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.json;
+
+import org.elasticsearch.common.xcontent.BaseXContentTestCase;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class JsonXContentTests extends BaseXContentTestCase {
+
+    @Override
+    public XContentType xcontentType() {
+        return XContentType.JSON;
+    }
+
+}

core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.smile;
+
+import org.elasticsearch.common.xcontent.BaseXContentTestCase;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class SmileXContentTests extends BaseXContentTestCase {
+
+    @Override
+    public XContentType xcontentType() {
+        return XContentType.SMILE;
+    }
+
+}

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java

@@ -475,15 +475,10 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
         assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
 
-        // Test method: rawField(String fieldName, byte[] content)
-        assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject());
-
         // Test method: rawField(String fieldName, InputStream content)
-        assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject());
+        assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject());
+        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject());
+        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
     }
 
     public void testArrays() throws Exception {

core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.yaml;
+
+import org.elasticsearch.common.xcontent.BaseXContentTestCase;
+import org.elasticsearch.common.xcontent.XContentType;
+
+public class YamlXContentTests extends BaseXContentTestCase {
+
+    @Override
+    public XContentType xcontentType() {
+        return XContentType.YAML;
+    }
+
+}

test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -450,7 +450,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                     .setOrder(0)
                     .setSettings(randomSettingsBuilder);
             if (mappings != null) {
-                logger.info("test using _default_ mappings: [{}]", mappings.bytesStream().bytes().toUtf8());
+                logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8());
                 putTemplate.addMapping("_default_", mappings);
             }
             assertAcked(putTemplate.execute().actionGet());