diff --git a/pom.xml b/pom.xml index ebc78fa8fd9..c6147ceb822 100644 --- a/pom.xml +++ b/pom.xml @@ -232,6 +232,13 @@ compile + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + 2.3.2 + compile + + io.netty netty @@ -554,6 +561,7 @@ com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.dataformat:jackson-dataformat-smile com.fasterxml.jackson.dataformat:jackson-dataformat-yaml + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor joda-time:joda-time org.joda:joda-convert io.netty:netty diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index 68498af254a..ca3e2f7bcd2 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -19,11 +19,13 @@ package org.elasticsearch.common.xcontent; +import com.fasterxml.jackson.dataformat.cbor.CBORConstants; import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.cbor.CborXContent; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.smile.SmileXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; @@ -82,6 +84,20 @@ public class XContentFactory { return new XContentBuilder(YamlXContent.yamlXContent, os); } + /** + * Returns a content builder using CBOR format ({@link org.elasticsearch.common.xcontent.XContentType#CBOR}. + */ + public static XContentBuilder cborBuilder() throws IOException { + return contentBuilder(XContentType.CBOR); + } + + /** + * Constructs a new cbor builder that will output the result into the provided output stream. + */ + public static XContentBuilder cborBuilder(OutputStream os) throws IOException { + return new XContentBuilder(CborXContent.cborXContent, os); + } + /** * Constructs a xcontent builder that will output the result into the provided output stream. */ @@ -92,6 +108,8 @@ public class XContentFactory { return smileBuilder(outputStream); } else if (type == XContentType.YAML) { return yamlBuilder(outputStream); + } else if (type == XContentType.CBOR) { + return cborBuilder(outputStream); } throw new ElasticsearchIllegalArgumentException("No matching content type for " + type); } @@ -106,6 +124,8 @@ public class XContentFactory { return SmileXContent.contentBuilder(); } else if (type == XContentType.YAML) { return YamlXContent.contentBuilder(); + } else if (type == XContentType.CBOR) { + return CborXContent.contentBuilder(); } throw new ElasticsearchIllegalArgumentException("No matching content type for " + type); } @@ -137,6 +157,8 @@ public class XContentFactory { return XContentType.YAML; } + // CBOR is not supported + for (int i = 0; i < length; i++) { char c = content.charAt(i); if (c == '{') { @@ -209,6 +231,9 @@ public class XContentFactory { return XContentType.YAML; } } + if (first == (CBORConstants.BYTE_OBJECT_INDEFINITE & 0xff)){ + return XContentType.CBOR; + } for (int i = 2; i < GUESS_HEADER_LENGTH; i++) { int val = si.read(); if (val == -1) { @@ -254,6 +279,9 @@ public class XContentFactory { if (length > 2 && first == '-' && bytes.get(1) == '-' && bytes.get(2) == '-') { return XContentType.YAML; } + if (first == CBORConstants.BYTE_OBJECT_INDEFINITE){ + return XContentType.CBOR; + } for (int i = 0; i < length; i++) { if (bytes.get(i) == '{') { return XContentType.JSON; diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index b658199a762..4acee241603 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.xcontent.cbor.CborXContent; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.smile.SmileXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; @@ -84,7 +85,26 @@ public enum XContentType { public XContent xContent() { return YamlXContent.yamlXContent; } - }; + }, + /** + * A CBOR based content type. + */ + CBOR(3) { + @Override + public String restContentType() { + return "application/cbor"; + } + + @Override + public String shortName() { + return "cbor"; + } + + @Override + public XContent xContent() { + return CborXContent.cborXContent; + } + },; public static XContentType fromRestContentType(String contentType) { if (contentType == null) { @@ -102,6 +122,10 @@ public enum XContentType { return YAML; } + if ("application/cbor".equals(contentType) || "cbor".equalsIgnoreCase(contentType)) { + return CBOR; + } + return null; } diff --git a/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java new file mode 100644 index 00000000000..ac5a7696e0b --- /dev/null +++ b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.cbor; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.dataformat.cbor.CBORFactory; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.FastStringReader; +import org.elasticsearch.common.xcontent.*; + +import java.io.*; + +/** + * A CBOR based content implementation using Jackson. + */ +public class CborXContent implements XContent { + + public static XContentBuilder contentBuilder() throws IOException { + return XContentBuilder.builder(cborXContent); + } + + final static CBORFactory cborFactory; + public final static CborXContent cborXContent; + + static { + cborFactory = new CBORFactory(); + cborXContent = new CborXContent(); + } + + private CborXContent() { + } + + @Override + public XContentType type() { + return XContentType.CBOR; + } + + @Override + public byte streamSeparator() { + throw new ElasticsearchParseException("cbor does not support stream parsing..."); + } + + @Override + public XContentGenerator createGenerator(OutputStream os) throws IOException { + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8)); + } + + @Override + public XContentGenerator createGenerator(Writer writer) throws IOException { + return new CborXContentGenerator(cborFactory.createGenerator(writer)); + } + + @Override + public XContentParser createParser(String content) throws IOException { + return new CborXContentParser(cborFactory.createParser(new FastStringReader(content))); + } + + @Override + public XContentParser createParser(InputStream is) throws IOException { + return new CborXContentParser(cborFactory.createParser(is)); + } + + @Override + public XContentParser createParser(byte[] data) throws IOException { + return new CborXContentParser(cborFactory.createParser(data)); + } + + @Override + public XContentParser createParser(byte[] data, int offset, int length) throws IOException { + return new CborXContentParser(cborFactory.createParser(data, offset, length)); + } + + @Override + public XContentParser createParser(BytesReference bytes) throws IOException { + if (bytes.hasArray()) { + return createParser(bytes.array(), bytes.arrayOffset(), bytes.length()); + } + return createParser(bytes.streamInput()); + } + + @Override + public XContentParser createParser(Reader reader) throws IOException { + return new CborXContentParser(cborFactory.createParser(reader)); + } +} diff --git a/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java new file mode 100644 index 00000000000..c410d777b0d --- /dev/null +++ b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.cbor; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.dataformat.cbor.CBORParser; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * + */ +public class CborXContentGenerator extends JsonXContentGenerator { + + public CborXContentGenerator(JsonGenerator generator) { + super(generator); + } + + @Override + public XContentType contentType() { + return XContentType.CBOR; + } + + @Override + public void usePrintLineFeedAtEnd() { + // nothing here + } + + @Override + public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { + writeFieldName(fieldName); + try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { + parser.nextToken(); + generator.copyCurrentStructure(parser); + } + } + + @Override + public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { + writeFieldName(fieldName); + try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { + parser.nextToken(); + generator.copyCurrentStructure(parser); + } + } + + @Override + protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { + writeFieldName(fieldName); + CBORParser parser; + if (content.hasArray()) { + parser = CborXContent.cborFactory.createParser(content.array(), content.arrayOffset(), content.length()); + } else { + parser = CborXContent.cborFactory.createParser(content.streamInput()); + } + try { + parser.nextToken(); + generator.copyCurrentStructure(parser); + } finally { + parser.close(); + } + } + + @Override + public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { + writeFieldName(fieldName); + try (CBORParser parser = CborXContent.cborFactory.createParser(content, offset, length)) { + parser.nextToken(); + generator.copyCurrentStructure(parser); + } + } +} diff --git a/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java new file mode 100644 index 00000000000..ed10ea47c0e --- /dev/null +++ b/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.cbor; + +import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContentParser; + +/** + * + */ +public class CborXContentParser extends JsonXContentParser { + + public CborXContentParser(JsonParser parser) { + super(parser); + } + + @Override + public XContentType contentType() { + return XContentType.CBOR; + } +} diff --git a/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java new file mode 100644 index 00000000000..e7629332253 --- /dev/null +++ b/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamInput; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +public class XContentFactoryTests extends ElasticsearchTestCase { + + + @Test + public void testGuessJson() throws IOException { + testGuessType(XContentType.JSON); + } + + @Test + public void testGuessSmile() throws IOException { + testGuessType(XContentType.SMILE); + } + + @Test + public void testGuessYaml() throws IOException { + testGuessType(XContentType.YAML); + } + + @Test + public void testGuessCbor() throws IOException { + testGuessType(XContentType.CBOR); + } + + private void testGuessType(XContentType type) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(type); + builder.startObject(); + builder.field("field1", "value1"); + builder.endObject(); + + assertThat(XContentFactory.xContentType(builder.bytes()), equalTo(type)); + BytesArray bytesArray = builder.bytes().toBytesArray(); + assertThat(XContentFactory.xContentType(new BytesStreamInput(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), false)), equalTo(type)); + + // CBOR is binary, cannot use String + if (type != XContentType.CBOR) { + assertThat(XContentFactory.xContentType(builder.string()), equalTo(type)); + } + } +} diff --git a/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java b/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java index c8911c6c4a6..d1f90ced81f 100644 --- a/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java +++ b/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java @@ -52,6 +52,11 @@ public class BuilderRawFieldTests extends ElasticsearchTestCase { testRawField(XContentType.YAML); } + @Test + public void testCborRawField() throws IOException { + testRawField(XContentType.CBOR); + } + private void testRawField(XContentType type) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(type); builder.startObject(); diff --git a/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java new file mode 100644 index 00000000000..f4982cd614b --- /dev/null +++ b/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.cbor; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentGenerator; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +/** + * + */ +public class JsonVsCborTests extends ElasticsearchTestCase { + + @Test + public void compareParsingTokens() throws IOException { + BytesStreamOutput xsonOs = new BytesStreamOutput(); + XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs); + + BytesStreamOutput jsonOs = new BytesStreamOutput(); + XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs); + + xsonGen.writeStartObject(); + jsonGen.writeStartObject(); + + xsonGen.writeStringField("test", "value"); + jsonGen.writeStringField("test", "value"); + + xsonGen.writeArrayFieldStart("arr"); + jsonGen.writeArrayFieldStart("arr"); + xsonGen.writeNumber(1); + jsonGen.writeNumber(1); + xsonGen.writeNull(); + jsonGen.writeNull(); + xsonGen.writeEndArray(); + jsonGen.writeEndArray(); + + xsonGen.writeEndObject(); + jsonGen.writeEndObject(); + + xsonGen.close(); + jsonGen.close(); + + verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes().toBytes())); + } + + private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException { + while (true) { + XContentParser.Token token1 = parser1.nextToken(); + XContentParser.Token token2 = parser2.nextToken(); + if (token1 == null) { + assertThat(token2, nullValue()); + return; + } + assertThat(token1, equalTo(token2)); + switch (token1) { + case FIELD_NAME: + assertThat(parser1.currentName(), equalTo(parser2.currentName())); + break; + case VALUE_STRING: + assertThat(parser1.text(), equalTo(parser2.text())); + break; + case VALUE_NUMBER: + assertThat(parser1.numberType(), equalTo(parser2.numberType())); + assertThat(parser1.numberValue(), equalTo(parser2.numberValue())); + break; + } + } + } +}