diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index d5480cd4e3a..2bd4b9e3206 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -930,8 +930,8 @@ public final class XContentBuilder implements BytesStream, Releasable { return this; } - public XContentBuilder rawField(String fieldName, InputStream content) throws IOException { - generator.writeRawField(fieldName, content, bos); + public XContentBuilder rawField(String fieldName, InputStream content, XContentType contentType) throws IOException { + generator.writeRawField(fieldName, content, bos, contentType); return this; } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index a84ee838e2e..9843a19948e 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -116,7 +116,7 @@ public interface XContentGenerator extends Closeable { void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException; - void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException; + void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException; void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 9fa14a3cc8d..f16332f07dd 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -423,7 +423,7 @@ public class XContentHelper { } XContentType contentType = XContentFactory.xContentType(compressedStreamInput); if (contentType == builder.contentType()) { - builder.rawField(field, compressedStreamInput); + builder.rawField(field, compressedStreamInput, contentType); } else { try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) { parser.nextToken(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index 5210a82527e..5417675738e 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -63,27 +59,19 @@ public class CborXContent implements XContent { throw new ElasticsearchParseException("cbor does not support stream parsing..."); } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new CborXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8)); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8)); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator cborGenerator = new FilteringJsonGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new CborXContentGenerator(cborGenerator); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), filters); } @Override public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(cborFactory.createGenerator(writer)); + return new CborXContentGenerator(cborFactory.createGenerator(writer)); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java index 70b92b0708c..a3ca669b156 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java @@ -19,10 +19,10 @@ package org.elasticsearch.common.xcontent.cbor; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.cbor.CBORParser; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.IOException; @@ -34,8 +34,8 @@ import java.io.OutputStream; */ public class CborXContentGenerator extends JsonXContentGenerator { - public CborXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public CborXContentGenerator(JsonGenerator jsonGenerator, String... filters) { + super(jsonGenerator, filters); } @Override @@ -49,7 +49,7 @@ public class CborXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { + public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { writeFieldName(fieldName); try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { parser.nextToken(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java deleted file mode 100644 index 0b485508c32..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.json; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.base.GeneratorBase; -import com.fasterxml.jackson.core.util.JsonGeneratorDelegate; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class BaseJsonGenerator extends JsonGeneratorDelegate { - - protected final GeneratorBase base; - - public BaseJsonGenerator(JsonGenerator generator, JsonGenerator base) { - super(generator, true); - if (base instanceof GeneratorBase) { - this.base = (GeneratorBase) base; - } else { - this.base = null; - } - } - - public BaseJsonGenerator(JsonGenerator generator) { - this(generator, generator); - } - - protected void writeStartRaw(String fieldName) throws IOException { - writeFieldName(fieldName); - writeRaw(':'); - } - - public void writeEndRaw() { - assert base != null : "JsonGenerator should be of instance GeneratorBase but was: " + delegate.getClass(); - if (base != null) { - base.getOutputContext().writeValue(); - } - } - - protected void writeRawValue(byte[] content, OutputStream bos) throws IOException { - flush(); - bos.write(content); - } - - protected void writeRawValue(byte[] content, int offset, int length, OutputStream bos) throws IOException { - flush(); - bos.write(content, offset, length); - } - - protected void writeRawValue(InputStream content, OutputStream bos) throws IOException { - flush(); - Streams.copy(content, bos); - } - - protected void writeRawValue(BytesReference content, OutputStream bos) throws IOException { - flush(); - content.writeTo(bos); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 47da7934939..671dee086f6 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -25,9 +25,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -65,27 +63,19 @@ public class JsonXContent implements XContent { return '\n'; } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new JsonXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8)); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8)); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator jsonGenerator = new FilteringJsonGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new JsonXContentGenerator(jsonGenerator); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), filters); } @Override public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(jsonFactory.createGenerator(writer)); + return new JsonXContentGenerator(jsonFactory.createGenerator(writer)); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index b4a39cc42af..7f3cc6334b8 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -19,11 +19,19 @@ package org.elasticsearch.common.xcontent.json; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonStreamContext; +import com.fasterxml.jackson.core.base.GeneratorBase; +import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; import com.fasterxml.jackson.core.io.SerializedString; import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter; import java.io.IOException; import java.io.InputStream; @@ -34,13 +42,40 @@ import java.io.OutputStream; */ public class JsonXContentGenerator implements XContentGenerator { - protected final BaseJsonGenerator generator; + /** Generator used to write content **/ + protected final JsonGenerator generator; + + /** + * Reference to base generator because + * writing raw values needs a specific method call. + */ + private final GeneratorBase base; + + /** + * Reference to filtering generator because + * writing an empty object '{}' when everything is filtered + * out needs a specific treatment + */ + private final FilteringGeneratorDelegate filter; + private boolean writeLineFeedAtEnd; private static final SerializedString LF = new SerializedString("\n"); - private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); + private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); - public JsonXContentGenerator(BaseJsonGenerator generator) { - this.generator = generator; + public JsonXContentGenerator(JsonGenerator jsonGenerator, String... filters) { + if (jsonGenerator instanceof GeneratorBase) { + this.base = (GeneratorBase) jsonGenerator; + } else { + this.base = null; + } + + if (CollectionUtils.isEmpty(filters)) { + this.generator = jsonGenerator; + this.filter = null; + } else { + this.filter = new FilteringGeneratorDelegate(jsonGenerator, new FilterPathBasedFilter(filters), true, true); + this.generator = this.filter; + } } @Override @@ -68,13 +103,35 @@ public class JsonXContentGenerator implements XContentGenerator { generator.writeEndArray(); } + protected boolean isFiltered() { + return filter != null; + } + + protected boolean inRoot() { + if (isFiltered()) { + JsonStreamContext context = filter.getFilterContext(); + return ((context != null) && (context.inRoot() && context.getCurrentName() == null)); + } + return false; + } + @Override public void writeStartObject() throws IOException { + if (isFiltered() && inRoot()) { + // Bypass generator to always write the root start object + filter.getDelegate().writeStartObject(); + return; + } generator.writeStartObject(); } @Override public void writeEndObject() throws IOException { + if (isFiltered() && inRoot()) { + // Bypass generator to always write the root end object + filter.getDelegate().writeEndObject(); + return; + } generator.writeEndObject(); } @@ -253,32 +310,62 @@ public class JsonXContentGenerator implements XContentGenerator { generator.writeStartObject(); } + private void writeStartRaw(String fieldName) throws IOException { + writeFieldName(fieldName); + generator.writeRaw(':'); + } + + public void writeEndRaw() { + assert base != null : "JsonGenerator should be of instance GeneratorBase but was: " + generator.getClass(); + if (base != null) { + base.getOutputContext().writeValue(); + } + } + @Override public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); + writeRawField(fieldName, new BytesArray(content), bos); } @Override public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, offset, length, bos); - generator.writeEndRaw(); + writeRawField(fieldName, new BytesArray(content, offset, length), bos); } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); + public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { + if (isFiltered() || (contentType != contentType())) { + // When the current generator is filtered (ie filter != null) + // or the content is in a different format than the current generator, + // we need to copy the whole structure so that it will be correctly + // filtered or converted + try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) { + parser.nextToken(); + writeFieldName(fieldName); + copyCurrentStructure(parser); + } + } else { + writeStartRaw(fieldName); + flush(); + Streams.copy(content, bos); + writeEndRaw(); + } } @Override public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException { XContentType contentType = XContentFactory.xContentType(content); if (contentType != null) { - writeObjectRaw(fieldName, content, bos); + if (isFiltered() || (contentType != contentType())) { + // When the current generator is filtered (ie filter != null) + // or the content is in a different format than the current generator, + // we need to copy the whole structure so that it will be correctly + // filtered or converted + copyRawField(fieldName, content, contentType.xContent()); + } else { + // Otherwise, the generator is not filtered and has the same type: we can potentially optimize the write + writeObjectRaw(fieldName, content, bos); + } } else { writeFieldName(fieldName); // we could potentially optimize this to not rely on exception logic... @@ -296,9 +383,29 @@ public class JsonXContentGenerator implements XContentGenerator { } protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); + writeStartRaw(fieldName); + flush(); + content.writeTo(bos); + writeEndRaw(); + } + + protected void copyRawField(String fieldName, BytesReference content, XContent xContent) throws IOException { + XContentParser parser = null; + try { + if (content.hasArray()) { + parser = xContent.createParser(content.array(), content.arrayOffset(), content.length()); + } else { + parser = xContent.createParser(content.streamInput()); + } + if (fieldName != null) { + writeFieldName(fieldName); + } + copyCurrentStructure(parser); + } finally { + if (parser != null) { + parser.close(); + } + } } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index 8a21ce1d93a..cb1bdd3bd4d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -64,27 +60,19 @@ public class SmileXContent implements XContent { return (byte) 0xFF; } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new SmileXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8)); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8)); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator smileGenerator = new FilteringJsonGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new SmileXContentGenerator(smileGenerator); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), filters); } @Override public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(smileFactory.createGenerator(writer)); + return new SmileXContentGenerator(smileFactory.createGenerator(writer)); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java index b8c1b3dad65..5002cfac645 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java @@ -19,10 +19,10 @@ package org.elasticsearch.common.xcontent.smile; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.smile.SmileParser; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.IOException; @@ -34,8 +34,8 @@ import java.io.OutputStream; */ public class SmileXContentGenerator extends JsonXContentGenerator { - public SmileXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public SmileXContentGenerator(JsonGenerator jsonGenerator, String... filters) { + super(jsonGenerator, filters); } @Override @@ -49,7 +49,7 @@ public class SmileXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { + public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { writeFieldName(fieldName); try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) { parser.nextToken(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java deleted file mode 100644 index 66f20cce435..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support.filtering; - -import com.fasterxml.jackson.core.JsonGenerator; -import org.elasticsearch.common.regex.Regex; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * A FilterContext contains the description of a field about to be written by a JsonGenerator. - */ -public class FilterContext { - - /** - * The field/property name to be write - */ - private String property; - - /** - * List of XContentFilter matched by the current filtering context - */ - private List matchings; - - /** - * Flag to indicate if the field/property must be written - */ - private Boolean write = null; - - /** - * Flag to indicate if the field/property match a filter - */ - private boolean match = false; - - /** - * Points to the parent context - */ - private FilterContext parent; - - /** - * Type of the field/property - */ - private Type type = Type.VALUE; - - protected enum Type { - VALUE, - OBJECT, - ARRAY, - ARRAY_OF_OBJECT - } - - public FilterContext(String property, FilterContext parent) { - this.property = property; - this.parent = parent; - } - - public void reset(String property) { - this.property = property; - this.write = null; - if (matchings != null) { - matchings.clear(); - } - this.match = false; - this.type = Type.VALUE; - } - - public void reset(String property, FilterContext parent) { - reset(property); - this.parent = parent; - if (parent.isMatch()) { - match = true; - } - } - - public FilterContext parent() { - return parent; - } - - public List matchings() { - return matchings; - } - - public void addMatching(String[] matching) { - if (matchings == null) { - matchings = new ArrayList<>(); - } - matchings.add(matching); - } - - public boolean isRoot() { - return parent == null; - } - - public boolean isArray() { - return Type.ARRAY.equals(type); - } - - public void initArray() { - this.type = Type.ARRAY; - } - - public boolean isObject() { - return Type.OBJECT.equals(type); - } - - public void initObject() { - this.type = Type.OBJECT; - } - - public boolean isArrayOfObject() { - return Type.ARRAY_OF_OBJECT.equals(type); - } - - public void initArrayOfObject() { - this.type = Type.ARRAY_OF_OBJECT; - } - - public boolean isMatch() { - return match; - } - - /** - * This method contains the logic to check if a field/property must be included - * or not. - */ - public boolean include() { - if (write == null) { - if (parent != null) { - // the parent context matches the end of a filter list: - // by default we include all the sub properties so we - // don't need to check if the sub properties also match - if (parent.isMatch()) { - write = true; - match = true; - return write; - } - - if (parent.matchings() != null) { - - // Iterates over the filters matched by the parent context - // and checks if the current context also match - for (String[] matcher : parent.matchings()) { - if (matcher.length > 0) { - String field = matcher[0]; - - if ("**".equals(field)) { - addMatching(matcher); - } - - if ((field != null) && (Regex.simpleMatch(field, property))) { - int remaining = matcher.length - 1; - - // the current context matches the end of a filter list: - // it must be written and it is flagged as a direct match - if (remaining == 0) { - write = true; - match = true; - return write; - } else { - String[] submatching = new String[remaining]; - System.arraycopy(matcher, 1, submatching, 0, remaining); - addMatching(submatching); - } - } - } - } - } - } else { - // Root object is always written - write = true; - } - - if (write == null) { - write = false; - } - } - return write; - } - - /** - * Ensure that the full path to the current field is write by the JsonGenerator - */ - public void writePath(JsonGenerator generator) throws IOException { - if (parent != null) { - parent.writePath(generator); - } - - if ((write == null) || (!write)) { - write = true; - - if (property == null) { - generator.writeStartObject(); - } else { - generator.writeFieldName(property); - if (isArray()) { - generator.writeStartArray(); - } else if (isObject() || isArrayOfObject()) { - generator.writeStartObject(); - } - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java new file mode 100644 index 00000000000..9d7961ec0b5 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.elasticsearch.common.xcontent.support.filtering; + +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class FilterPath { + + static final FilterPath EMPTY = new FilterPath(); + + private final String filter; + private final String segment; + private final FilterPath next; + private final boolean simpleWildcard; + private final boolean doubleWildcard; + + protected FilterPath(String filter, String segment, FilterPath next) { + this.filter = filter; + this.segment = segment; + this.next = next; + this.simpleWildcard = (segment != null) && (segment.length() == 1) && (segment.charAt(0) == '*'); + this.doubleWildcard = (segment != null) && (segment.length() == 2) && (segment.charAt(0) == '*') && (segment.charAt(1) == '*'); + } + + private FilterPath() { + this("", "", null); + } + + public FilterPath matchProperty(String name) { + if ((next != null) && (simpleWildcard || doubleWildcard || Regex.simpleMatch(segment, name))) { + return next; + } + return null; + } + + public boolean matches() { + return next == null; + } + + boolean isDoubleWildcard() { + return doubleWildcard; + } + + boolean isSimpleWildcard() { + return simpleWildcard; + } + + String getSegment() { + return segment; + } + + FilterPath getNext() { + return next; + } + + public static FilterPath[] compile(String... filters) { + if (CollectionUtils.isEmpty(filters)) { + return null; + } + + List paths = new ArrayList<>(); + for (String filter : filters) { + if (filter != null) { + filter = filter.trim(); + if (filter.length() > 0) { + paths.add(parse(filter, filter)); + } + } + } + return paths.toArray(new FilterPath[paths.size()]); + } + + private static FilterPath parse(final String filter, final String segment) { + int end = segment.length(); + + for (int i = 0; i < end; ) { + char c = segment.charAt(i); + + if (c == '.') { + String current = segment.substring(0, i).replaceAll("\\\\.", "."); + return new FilterPath(filter, current, parse(filter, segment.substring(i + 1))); + } + ++i; + if ((c == '\\') && (i < end) && (segment.charAt(i) == '.')) { + ++i; + } + } + return new FilterPath(filter, segment.replaceAll("\\\\.", "."), EMPTY); + } + + @Override + public String toString() { + return "FilterPath [filter=" + filter + ", segment=" + segment + "]"; + } +} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java new file mode 100644 index 00000000000..d2f28611bae --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import com.fasterxml.jackson.core.filter.TokenFilter; +import org.elasticsearch.common.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class FilterPathBasedFilter extends TokenFilter { + + /** + * Marker value that should be used to indicate that a property name + * or value matches one of the filter paths. + */ + private static final TokenFilter MATCHING = new TokenFilter(){}; + + /** + * Marker value that should be used to indicate that none of the + * property names/values matches one of the filter paths. + */ + private static final TokenFilter NO_MATCHING = new TokenFilter(){}; + + private final FilterPath[] filters; + + public FilterPathBasedFilter(FilterPath[] filters) { + if (CollectionUtils.isEmpty(filters)) { + throw new IllegalArgumentException("filters cannot be null or empty"); + } + this.filters = filters; + } + + public FilterPathBasedFilter(String[] filters) { + this(FilterPath.compile(filters)); + } + + /** + * Evaluates if a property name matches one of the given filter paths. + */ + private TokenFilter evaluate(String name, FilterPath[] filters) { + if (filters != null) { + List nextFilters = null; + + for (FilterPath filter : filters) { + FilterPath next = filter.matchProperty(name); + if (next != null) { + if (next.matches()) { + return MATCHING; + } else { + if (nextFilters == null) { + nextFilters = new ArrayList<>(); + } + if (filter.isDoubleWildcard()) { + nextFilters.add(filter); + } + nextFilters.add(next); + } + } + } + + if ((nextFilters != null) && (nextFilters.isEmpty() == false)) { + return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[nextFilters.size()])); + } + } + return NO_MATCHING; + } + + @Override + public TokenFilter includeProperty(String name) { + TokenFilter include = evaluate(name, filters); + if (include == MATCHING) { + return TokenFilter.INCLUDE_ALL; + } + if (include == NO_MATCHING) { + return null; + } + return include; + } + + @Override + protected boolean _includeScalar() { + for (FilterPath filter : filters) { + if (filter.matches()) { + return true; + } + } + return false; + } +} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java deleted file mode 100644 index b70a1ae9365..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support.filtering; - -import com.fasterxml.jackson.core.Base64Variant; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.SerializableString; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Queue; - -/** - * A FilteringJsonGenerator uses antpath-like filters to include/exclude fields when writing XContent streams. - * - * When writing a XContent stream, this class instantiates (or reuses) a FilterContext instance for each - * field (or property) that must be generated. This filter context is used to check if the field/property must be - * written according to the current list of XContentFilter filters. - */ -public class FilteringJsonGenerator extends BaseJsonGenerator { - - /** - * List of previous contexts - * (MAX_CONTEXTS contexts are kept around in order to be reused) - */ - private Queue contexts = new ArrayDeque<>(); - private static final int MAX_CONTEXTS = 10; - - /** - * Current filter context - */ - private FilterContext context; - - public FilteringJsonGenerator(JsonGenerator generator, String[] filters) { - super(generator); - - List builder = new ArrayList<>(); - if (filters != null) { - for (String filter : filters) { - String[] matcher = Strings.delimitedListToStringArray(filter, "."); - if (matcher != null) { - builder.add(matcher); - } - } - } - - // Creates a root context that matches all filtering rules - this.context = get(null, null, Collections.unmodifiableList(builder)); - } - - /** - * Get a new context instance (and reset it if needed) - */ - private FilterContext get(String property, FilterContext parent) { - FilterContext ctx = contexts.poll(); - if (ctx == null) { - ctx = new FilterContext(property, parent); - } else { - ctx.reset(property, parent); - } - return ctx; - } - - /** - * Get a new context instance (and reset it if needed) - */ - private FilterContext get(String property, FilterContext context, List matchings) { - FilterContext ctx = get(property, context); - if (matchings != null) { - for (String[] matching : matchings) { - ctx.addMatching(matching); - } - } - return ctx; - } - - /** - * Adds a context instance to the pool in order to reuse it if needed - */ - private void put(FilterContext ctx) { - if (contexts.size() <= MAX_CONTEXTS) { - contexts.offer(ctx); - } - } - - @Override - public void writeStartArray() throws IOException { - context.initArray(); - if (context.include()) { - super.writeStartArray(); - } - } - - @Override - public void writeStartArray(int size) throws IOException { - context.initArray(); - if (context.include()) { - super.writeStartArray(size); - } - } - - @Override - public void writeEndArray() throws IOException { - // Case of array of objects - if (context.isArrayOfObject()) { - // Release current context and go one level up - FilterContext parent = context.parent(); - put(context); - context = parent; - } - - if (context.include()) { - super.writeEndArray(); - } - } - - @Override - public void writeStartObject() throws IOException { - // Case of array of objects - if (context.isArray()) { - // Get a context for the anonymous object - context = get(null, context, context.matchings()); - context.initArrayOfObject(); - } - - if (!context.isArrayOfObject()) { - context.initObject(); - } - - if (context.include()) { - super.writeStartObject(); - } - - context = get(null, context); - } - - @Override - public void writeEndObject() throws IOException { - if (!context.isRoot()) { - // Release current context and go one level up - FilterContext parent = context.parent(); - put(context); - context = parent; - } - - if (context.include()) { - super.writeEndObject(); - } - } - - @Override - public void writeFieldName(String name) throws IOException { - context.reset(name); - - if (context.include()) { - // Ensure that the full path to the field is written - context.writePath(delegate); - super.writeFieldName(name); - } - } - - @Override - public void writeFieldName(SerializableString name) throws IOException { - context.reset(name.getValue()); - - if (context.include()) { - // Ensure that the full path to the field is written - context.writePath(delegate); - super.writeFieldName(name); - } - } - - @Override - public void writeString(String text) throws IOException { - if (context.include()) { - super.writeString(text); - } - } - - @Override - public void writeString(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeString(text, offset, len); - } - } - - @Override - public void writeString(SerializableString text) throws IOException { - if (context.include()) { - super.writeString(text); - } - } - - @Override - public void writeRawUTF8String(byte[] text, int offset, int length) throws IOException { - if (context.include()) { - super.writeRawUTF8String(text, offset, length); - } - } - - @Override - public void writeUTF8String(byte[] text, int offset, int length) throws IOException { - if (context.include()) { - super.writeUTF8String(text, offset, length); - } - } - - @Override - public void writeRaw(String text) throws IOException { - if (context.include()) { - super.writeRaw(text); - } - } - - @Override - public void writeRaw(String text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRaw(text, offset, len); - } - } - - @Override - public void writeRaw(SerializableString raw) throws IOException { - if (context.include()) { - super.writeRaw(raw); - } - } - - @Override - public void writeRaw(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRaw(text, offset, len); - } - } - - @Override - public void writeRaw(char c) throws IOException { - if (context.include()) { - super.writeRaw(c); - } - } - - @Override - public void writeRawValue(String text) throws IOException { - if (context.include()) { - super.writeRawValue(text); - } - } - - @Override - public void writeRawValue(String text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRawValue(text, offset, len); - } - } - - @Override - public void writeRawValue(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRawValue(text, offset, len); - } - } - - @Override - public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int len) throws IOException { - if (context.include()) { - super.writeBinary(b64variant, data, offset, len); - } - } - - @Override - public int writeBinary(Base64Variant b64variant, InputStream data, int dataLength) throws IOException { - if (context.include()) { - return super.writeBinary(b64variant, data, dataLength); - } - return 0; - } - - @Override - public void writeNumber(short v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(int v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(long v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(BigInteger v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(double v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(float v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(BigDecimal v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(String encodedValue) throws IOException, UnsupportedOperationException { - if (context.include()) { - super.writeNumber(encodedValue); - } - } - - @Override - public void writeBoolean(boolean state) throws IOException { - if (context.include()) { - super.writeBoolean(state); - } - } - - @Override - public void writeNull() throws IOException { - if (context.include()) { - super.writeNull(); - } - } - - @Override - public void copyCurrentEvent(JsonParser jp) throws IOException { - if (context.include()) { - super.copyCurrentEvent(jp); - } - } - - @Override - public void copyCurrentStructure(JsonParser jp) throws IOException { - if (context.include()) { - super.copyCurrentStructure(jp); - } - } - - @Override - protected void writeRawValue(byte[] content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - protected void writeRawValue(byte[] content, int offset, int length, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, offset, length, bos); - } - } - - @Override - protected void writeRawValue(InputStream content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - protected void writeRawValue(BytesReference content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - public void close() throws IOException { - contexts.clear(); - super.close(); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index 388cd992e2b..d79a6a8ba80 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -62,27 +58,19 @@ public class YamlXContent implements XContent { throw new ElasticsearchParseException("yaml does not support stream parsing..."); } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new YamlXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8)); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8)); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator yamlGenerator = new FilteringJsonGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new YamlXContentGenerator(yamlGenerator); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), filters); } @Override public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(yamlFactory.createGenerator(writer)); + return new YamlXContentGenerator(yamlFactory.createGenerator(writer)); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java index 62967247a82..e1a71c143b0 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java @@ -19,10 +19,10 @@ package org.elasticsearch.common.xcontent.yaml; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.IOException; @@ -34,8 +34,8 @@ import java.io.OutputStream; */ public class YamlXContentGenerator extends JsonXContentGenerator { - public YamlXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public YamlXContentGenerator(JsonGenerator jsonGenerator, String... filters) { + super(jsonGenerator, filters); } @Override @@ -49,7 +49,7 @@ public class YamlXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { + public void writeRawField(String fieldName, InputStream content, OutputStream bos, XContentType contentType) throws IOException { writeFieldName(fieldName); try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) { parser.nextToken(); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index 8073468ce8b..9216f48bb26 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -481,9 +481,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); // Test method: rawField(String fieldName, InputStream content) - assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); + assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); + assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes()), getXContentType()).endObject()); } public void testArrays() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java new file mode 100644 index 00000000000..0ca2eafa417 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class FilterPathGeneratorFilteringTests extends ESTestCase { + + private final JsonFactory JSON_FACTORY = new JsonFactory(); + + public void testFilters() throws Exception { + final String SAMPLE = "{'a':0,'b':true,'c':'c_value','d':[0,1,2],'e':[{'f1':'f1_value','f2':'f2_value'},{'g1':'g1_value','g2':'g2_value'}],'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"; + + assertResult(SAMPLE, "a", "{'a':0}"); + assertResult(SAMPLE, "b", "{'b':true}"); + assertResult(SAMPLE, "c", "{'c':'c_value'}"); + assertResult(SAMPLE, "d", "{'d':[0,1,2]}"); + assertResult(SAMPLE, "e", "{'e':[{'f1':'f1_value','f2':'f2_value'},{'g1':'g1_value','g2':'g2_value'}]}"); + assertResult(SAMPLE, "h", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "z", ""); + + assertResult(SAMPLE, "e.f1", "{'e':[{'f1':'f1_value'}]}"); + assertResult(SAMPLE, "e.f2", "{'e':[{'f2':'f2_value'}]}"); + assertResult(SAMPLE, "e.f*", "{'e':[{'f1':'f1_value','f2':'f2_value'}]}"); + assertResult(SAMPLE, "e.*2", "{'e':[{'f2':'f2_value'},{'g2':'g2_value'}]}"); + + assertResult(SAMPLE, "h.i", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "h.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "*.i", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.*.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "h.*.j.*.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "**.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "**.*2", "{'e':[{'f2':'f2_value'},{'g2':'g2_value'}]}"); + } + + public void testFiltersWithDots() throws Exception { + assertResult("{'a':0,'b.c':'value','b':{'c':'c_value'}}", "b.c", "{'b':{'c':'c_value'}}"); + assertResult("{'a':0,'b.c':'value','b':{'c':'c_value'}}", "b\\.c", "{'b.c':'value'}"); + } + + private void assertResult(String input, String filter, String expected) throws Exception { + try (BytesStreamOutput os = new BytesStreamOutput()) { + try (FilteringGeneratorDelegate generator = new FilteringGeneratorDelegate(JSON_FACTORY.createGenerator(os), new FilterPathBasedFilter(new String[]{filter}), true, true)) { + try (JsonParser parser = JSON_FACTORY.createParser(replaceQuotes(input))) { + while (parser.nextToken() != null) { + generator.copyCurrentStructure(parser); + } + } + } + assertThat(os.bytes().toUtf8(), equalTo(replaceQuotes(expected))); + } + } + + private String replaceQuotes(String s) { + return s.replace('\'', '"'); + } +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java new file mode 100644 index 00000000000..50683007717 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java @@ -0,0 +1,351 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.*; + +public class FilterPathTests extends ESTestCase { + + public void testSimpleFilterPath() { + final String input = "test"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("test")); + + FilterPath next = filterPath.getNext(); + assertNotNull(next); + assertThat(next.matches(), is(true)); + assertThat(next.getSegment(), isEmptyString()); + assertSame(next, FilterPath.EMPTY); + } + + public void testFilterPathWithSubField() { + final String input = "foo.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testFilterPathWithSubFields() { + final String input = "foo.bar.quz"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("quz")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testEmptyFilterPath() { + FilterPath[] filterPaths = FilterPath.compile(""); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(0)); + } + + public void testNullFilterPath() { + FilterPath[] filterPaths = FilterPath.compile((String) null); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(0)); + } + + public void testFilterPathWithEscapedDots() { + String input = "w.0.0.t"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + input = "w\\.0\\.0\\.t"; + + filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w.0.0.t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + + input = "w\\.0.0\\.t"; + + filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + filterPath = filterPaths[0]; + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w.0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0.t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testSimpleWildcardFilterPath() { + FilterPath[] filterPaths = FilterPath.compile("*"); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isSimpleWildcard(), is(true)); + assertThat(filterPath.getSegment(), equalTo("*")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertSame(next, FilterPath.EMPTY); + } + + public void testWildcardInNameFilterPath() { + String input = "f*o.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("f*o")); + assertThat(filterPath.matchProperty("foo"), notNullValue()); + assertThat(filterPath.matchProperty("flo"), notNullValue()); + assertThat(filterPath.matchProperty("foooo"), notNullValue()); + assertThat(filterPath.matchProperty("boo"), nullValue()); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testDoubleWildcardFilterPath() { + FilterPath[] filterPaths = FilterPath.compile("**"); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), is(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertSame(next, FilterPath.EMPTY); + } + + public void testStartsWithDoubleWildcardFilterPath() { + String input = "**.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("**")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertThat(next.matches(), is(false)); + assertThat(next.getSegment(), equalTo("bar")); + + next = next.getNext(); + assertNotNull(next); + assertThat(next.matches(), is(true)); + assertThat(next.getSegment(), isEmptyString()); + assertSame(next, FilterPath.EMPTY); + } + + public void testContainsDoubleWildcardFilterPath() { + String input = "foo.**.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testMultipleFilterPaths() { + String[] inputs = {"foo.**.bar.*", "test.dot\\.ted"}; + + FilterPath[] filterPaths = FilterPath.compile(inputs); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(2)); + + // foo.**.bar.* + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.isSimpleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("*")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + // test.dot\.ted + filterPath = filterPaths[1]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("test")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("dot.ted")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } +} \ No newline at end of file