diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 35579965f30..83facb00f00 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -19,12 +19,15 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.util.Collections; +import java.util.Set; /** * A generic abstraction on top of handling content, inspired by JSON and pull parsing. @@ -42,27 +45,20 @@ public interface XContent { * Creates a new generator using the provided output stream. */ default XContentGenerator createGenerator(OutputStream os) throws IOException { - return createGenerator(os, null, true); + return createGenerator(os, Collections.emptySet(), Collections.emptySet()); } /** - * Creates a new generator using the provided output stream and some - * inclusive filters. Same as createGenerator(os, filters, true). - */ - default XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - return createGenerator(os, filters, true); - } - - /** - * Creates a new generator using the provided output stream and some - * filters. + * Creates a new generator using the provided output stream and some inclusive and/or exclusive filters. When both exclusive and + * inclusive filters are provided, the underlying generator will first use exclusion filters to remove fields and then will check the + * remaining fields against the inclusive filters. * - * @param inclusive - * If true only paths matching a filter will be included in - * output. If false no path matching a filter will be included in - * output + * @param os the output stream + * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. + * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. */ - XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException; + XContentGenerator createGenerator(OutputStream os, Set includes, Set excludes) throws IOException; + /** * Creates a parser over the provided string content. */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 1d8ce366b96..5274773b994 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -19,21 +19,8 @@ package org.elasticsearch.common.xcontent; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.file.Path; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; - import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.BytesStream; @@ -47,6 +34,21 @@ import org.joda.time.ReadableInstant; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.file.Path; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + /** * A utility to build XContent (ie json). */ @@ -58,12 +60,8 @@ public final class XContentBuilder implements BytesStream, Releasable { return new XContentBuilder(xContent, new BytesStreamOutput()); } - public static XContentBuilder builder(XContent xContent, String[] filters) throws IOException { - return new XContentBuilder(xContent, new BytesStreamOutput(), filters); - } - - public static XContentBuilder builder(XContent xContent, String[] filters, boolean inclusive) throws IOException { - return new XContentBuilder(xContent, new BytesStreamOutput(), filters, inclusive); + public static XContentBuilder builder(XContent xContent, Set includes, Set excludes) throws IOException { + return new XContentBuilder(xContent, new BytesStreamOutput(), includes, excludes); } private XContentGenerator generator; @@ -77,7 +75,7 @@ public final class XContentBuilder implements BytesStream, Releasable { * to call {@link #close()} when the builder is done with. */ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { - this(xContent, bos, null); + this(xContent, bos, Collections.emptySet(), Collections.emptySet()); } /** @@ -86,20 +84,24 @@ public final class XContentBuilder implements BytesStream, Releasable { * filter will be written to the output stream. Make sure to call * {@link #close()} when the builder is done with. */ - public XContentBuilder(XContent xContent, OutputStream bos, String[] filters) throws IOException { - this(xContent, bos, filters, true); + public XContentBuilder(XContent xContent, OutputStream bos, Set includes) throws IOException { + this(xContent, bos, includes, Collections.emptySet()); } /** - * Constructs a new builder using the provided xcontent, an OutputStream and - * some filters. If {@code filters} are specified and {@code inclusive} is - * true, only those values matching a filter will be written to the output - * stream. If {@code inclusive} is false, those matching will be excluded. + * Creates a new builder using the provided XContent, output stream and some inclusive and/or exclusive filters. When both exclusive and + * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the + * remaining fields against the inclusive filters. + *

* Make sure to call {@link #close()} when the builder is done with. + * + * @param os the output stream + * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. + * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. */ - public XContentBuilder(XContent xContent, OutputStream bos, String[] filters, boolean inclusive) throws IOException { - this.bos = bos; - this.generator = xContent.createGenerator(bos, filters, inclusive); + public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes) throws IOException { + this.bos = os; + this.generator = xContent.createGenerator(bos, includes, excludes); } public XContentType contentType() { diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index c8715d03c3e..4224b5328a7 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.util.Set; /** * A CBOR based content implementation using Jackson. @@ -70,8 +71,8 @@ public class CborXContent implements XContent { } @Override - public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException { - return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive); + public XContentGenerator createGenerator(OutputStream os, Set includes, Set excludes) throws IOException { + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java index 9ec690f2d1f..e63a928109d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java @@ -20,23 +20,22 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonGenerator; - +import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.OutputStream; +import java.util.Collections; +import java.util.Set; -/** - * - */ public class CborXContentGenerator extends JsonXContentGenerator { - public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { - this(jsonGenerator, os, filters, true); + public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) { + this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet()); } - public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) { - super(jsonGenerator, os, filters, inclusive); + public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set includes, Set excludes) { + super(jsonGenerator, os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 4828d8a752e..c8afb94f781 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.util.Set; /** * A JSON based content implementation using Jackson. @@ -92,8 +93,8 @@ public class JsonXContent implements XContent { } @Override - public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException { - return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive); + public XContentGenerator createGenerator(OutputStream os, Set includes, Set excludes) throws IOException { + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index dd95e0d1df5..4a393b9dd10 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -27,10 +27,10 @@ import com.fasterxml.jackson.core.io.SerializedString; import com.fasterxml.jackson.core.json.JsonWriteContext; import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.core.util.JsonGeneratorDelegate; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentGenerator; @@ -43,6 +43,9 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; /** * @@ -72,23 +75,38 @@ public class JsonXContentGenerator implements XContentGenerator { private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); private boolean prettyPrint = false; - public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) { + public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) { + this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet()); + } + + public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set includes, Set excludes) { + Objects.requireNonNull(includes, "Including filters must not be null"); + Objects.requireNonNull(excludes, "Excluding filters must not be null"); + this.os = os; if (jsonGenerator instanceof GeneratorBase) { this.base = (GeneratorBase) jsonGenerator; } else { this.base = null; } - if (CollectionUtils.isEmpty(filters)) { - this.generator = jsonGenerator; - this.filter = null; - } else { - this.filter = new FilteringGeneratorDelegate(jsonGenerator, - new FilterPathBasedFilter(filters, inclusive), true, true); - this.generator = this.filter; + JsonGenerator generator = jsonGenerator; + + boolean hasExcludes = excludes.isEmpty() == false; + if (hasExcludes) { + generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(excludes, false), true, true); } - this.os = os; + boolean hasIncludes = includes.isEmpty() == false; + if (hasIncludes) { + generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(includes, true), true, true); + } + + if (hasExcludes || hasIncludes) { + this.filter = (FilteringGeneratorDelegate) generator; + } else { + this.filter = null; + } + this.generator = generator; } @Override @@ -122,23 +140,34 @@ public class JsonXContentGenerator implements XContentGenerator { generator.writeEndArray(); } - protected boolean isFiltered() { + private boolean isFiltered() { return filter != null; } - protected boolean inRoot() { + private JsonGenerator getLowLevelGenerator() { if (isFiltered()) { - JsonStreamContext context = filter.getFilterContext(); - return ((context != null) && (context.inRoot() && context.getCurrentName() == null)); + JsonGenerator delegate = filter.getDelegate(); + if (delegate instanceof JsonGeneratorDelegate) { + // In case of combined inclusion and exclusion filters, we have one and only one another delegating level + delegate = ((JsonGeneratorDelegate) delegate).getDelegate(); + assert delegate instanceof JsonGeneratorDelegate == false; + } + return delegate; } - return false; + return generator; + } + + private boolean inRoot() { + JsonStreamContext context = generator.getOutputContext(); + return ((context != null) && (context.inRoot() && context.getCurrentName() == null)); } @Override public void writeStartObject() throws IOException { - if (isFiltered() && inRoot()) { - // Bypass generator to always write the root start object - filter.getDelegate().writeStartObject(); + if (inRoot()) { + // Use the low level generator to write the startObject so that the root + // start object is always written even if a filtered generator is used + getLowLevelGenerator().writeStartObject(); return; } generator.writeStartObject(); @@ -146,9 +175,10 @@ public class JsonXContentGenerator implements XContentGenerator { @Override public void writeEndObject() throws IOException { - if (isFiltered() && inRoot()) { - // Bypass generator to always write the root end object - filter.getDelegate().writeEndObject(); + if (inRoot()) { + // Use the low level generator to write the startObject so that the root + // start object is always written even if a filtered generator is used + getLowLevelGenerator().writeEndObject(); return; } generator.writeEndObject(); @@ -390,7 +420,8 @@ public class JsonXContentGenerator implements XContentGenerator { } if (writeLineFeedAtEnd) { flush(); - generator.writeRaw(LF); + // Bypass generator to always write the line feed + getLowLevelGenerator().writeRaw(LF); } generator.close(); } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index 629f8612ea6..94ac9b94356 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.util.Set; /** * A Smile based content implementation using Jackson. @@ -71,8 +72,8 @@ public class SmileXContent implements XContent { } @Override - public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException { - return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive); + public XContentGenerator createGenerator(OutputStream os, Set includes, Set excludes) throws IOException { + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java index ac294c1db85..afa420805f7 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java @@ -20,23 +20,22 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonGenerator; - +import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.OutputStream; +import java.util.Collections; +import java.util.Set; -/** - * - */ public class SmileXContentGenerator extends JsonXContentGenerator { - public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { - this(jsonGenerator, os, filters, true); + public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) { + this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet()); } - public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) { - super(jsonGenerator, os, filters, inclusive); + public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set includes, Set excludes) { + super(jsonGenerator, os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java index 9d7961ec0b5..a70e385d520 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java @@ -21,10 +21,10 @@ package org.elasticsearch.common.xcontent.support.filtering; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.util.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.Set; public class FilterPath { @@ -75,8 +75,8 @@ public class FilterPath { return next; } - public static FilterPath[] compile(String... filters) { - if (CollectionUtils.isEmpty(filters)) { + public static FilterPath[] compile(Set filters) { + if (filters == null || filters.isEmpty()) { return null; } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java index 69e4e79110d..846e172ae66 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.util.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.Set; public class FilterPathBasedFilter extends TokenFilter { @@ -53,7 +54,7 @@ public class FilterPathBasedFilter extends TokenFilter { this.filters = filters; } - public FilterPathBasedFilter(String[] filters, boolean inclusive) { + public FilterPathBasedFilter(Set filters, boolean inclusive) { this(FilterPath.compile(filters), inclusive); } @@ -103,11 +104,6 @@ public class FilterPathBasedFilter extends TokenFilter { @Override protected boolean _includeScalar() { - for (FilterPath filter : filters) { - if (filter.matches()) { - return inclusive; - } - } return !inclusive; } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index 27a0dd46e00..54da03118d7 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.util.Set; /** * A YAML based content implementation using Jackson. @@ -66,8 +67,8 @@ public class YamlXContent implements XContent { } @Override - public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException { - return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive); + public XContentGenerator createGenerator(OutputStream os, Set includes, Set excludes) throws IOException { + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java index f801401a229..d2c53c8a020 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java @@ -20,23 +20,22 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonGenerator; - +import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; import java.io.OutputStream; +import java.util.Collections; +import java.util.Set; -/** - * - */ public class YamlXContentGenerator extends JsonXContentGenerator { - public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { - this(jsonGenerator, os, filters, true); + public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) { + this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet()); } - public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) { - super(jsonGenerator, os, filters, inclusive); + public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set includes, Set excludes) { + super(jsonGenerator, os, includes, excludes); } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index f5d4f4eb695..f146267c9be 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -26,9 +27,17 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import java.util.function.Predicate; + +import static java.util.stream.Collectors.toSet; public abstract class AbstractRestChannel implements RestChannel { + private static final Predicate INCLUDE_FILTER = f -> f.charAt(0) != '-'; + private static final Predicate EXCLUDE_FILTER = INCLUDE_FILTER.negate(); + protected final RestRequest request; protected final boolean detailedErrorsEnabled; @@ -41,7 +50,7 @@ public abstract class AbstractRestChannel implements RestChannel { @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.hasContent() ? request.content() : null, request.hasParam("filter_path")); + return newBuilder(request.hasContent() ? request.content() : null, true); } @Override @@ -64,8 +73,15 @@ public abstract class AbstractRestChannel implements RestChannel { contentType = XContentType.JSON; } - String[] filters = useFiltering ? request.paramAsStringArrayOrEmptyIfAll("filter_path") : null; - XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), filters); + Set includes = Collections.emptySet(); + Set excludes = Collections.emptySet(); + if (useFiltering) { + Set filters = Strings.splitStringByCommaToSet(request.param("filter_path", null)); + includes = filters.stream().filter(INCLUDE_FILTER).collect(toSet()); + excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet()); + } + + XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes); if (request.paramAsBoolean("pretty", false)) { builder.prettyPrint().lfAtEnd(); } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index b8b38a543f6..f7ffcac32b3 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.xcontent.support.filtering; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -28,7 +29,11 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Set; +import java.util.function.Function; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -86,12 +91,16 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase return XContentBuilder.builder(getXContentType().xContent()); } - private XContentBuilder newXContentBuilder(String filter, boolean inclusive) throws IOException { - return XContentBuilder.builder(getXContentType().xContent(), new String[] { filter }, inclusive); + private XContentBuilder newXContentBuilderWithIncludes(String filter) throws IOException { + return newXContentBuilder(singleton(filter), emptySet()); } - private XContentBuilder newXContentBuilder(String[] filters, boolean inclusive) throws IOException { - return XContentBuilder.builder(getXContentType().xContent(), filters, inclusive); + private XContentBuilder newXContentBuilderWithExcludes(String filter) throws IOException { + return newXContentBuilder(emptySet(), singleton(filter)); + } + + private XContentBuilder newXContentBuilder(Set includes, Set excludes) throws IOException { + return XContentBuilder.builder(getXContentType().xContent(), includes, excludes); } /** @@ -173,20 +182,22 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase return builder; } - /** - * Instanciates a new XContentBuilder with the given filters and builds a - * sample with it. - * @param inclusive - * Specifies if filters are inclusive or exclusive - */ - private XContentBuilder sample(String filter, boolean inclusive) throws IOException { - return sample(newXContentBuilder(filter, inclusive)); + /** Create a new {@link XContentBuilder} and use it to build the sample using the given inclusive filter **/ + private XContentBuilder sampleWithIncludes(String filter) throws IOException { + return sample(newXContentBuilderWithIncludes(filter)); } - private XContentBuilder sample(String[] filters, boolean inclusive) throws IOException { - return sample(newXContentBuilder(filters, inclusive)); + /** Create a new {@link XContentBuilder} and use it to build the sample using the given exclusive filter **/ + private XContentBuilder sampleWithExcludes(String filter) throws IOException { + return sample(newXContentBuilderWithExcludes(filter)); } + /** Create a new {@link XContentBuilder} and use it to build the sample using the given includes and exclusive filters **/ + private XContentBuilder sampleWithFilters(Set includes, Set excludes) throws IOException { + return sample(newXContentBuilder(includes, excludes)); + } + + /** Create a new {@link XContentBuilder} and use it to build the sample **/ private XContentBuilder sample() throws IOException { return sample(newXContentBuilder()); } @@ -195,23 +206,23 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase XContentBuilder expected = sample(); assertXContentBuilder(expected, sample()); - assertXContentBuilder(expected, sample("*", true)); - assertXContentBuilder(expected, sample("**", true)); - assertXContentBuilder(expected, sample("xyz", false)); + assertXContentBuilder(expected, sampleWithIncludes("*")); + assertXContentBuilder(expected, sampleWithIncludes("**")); + assertXContentBuilder(expected, sampleWithExcludes("xyz")); } public void testNoMatch() throws Exception { XContentBuilder expected = newXContentBuilder().startObject().endObject(); - assertXContentBuilder(expected, sample("xyz", true)); - assertXContentBuilder(expected, sample("*", false)); - assertXContentBuilder(expected, sample("**", false)); + assertXContentBuilder(expected, sampleWithIncludes("xyz")); + assertXContentBuilder(expected, sampleWithExcludes("*")); + assertXContentBuilder(expected, sampleWithExcludes("**")); } public void testSimpleFieldInclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject().field("title", "My awesome book").endObject(); - assertXContentBuilder(expected, sample("title", true)); + assertXContentBuilder(expected, sampleWithIncludes("title")); } public void testSimpleFieldExclusive() throws Exception { @@ -286,10 +297,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("title", false)); + assertXContentBuilder(expected, sampleWithExcludes("title")); } - public void testSimpleFieldWithWildcardInclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .field("price", 27.99) @@ -343,7 +353,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("pr*", true)); + assertXContentBuilder(expected, sampleWithIncludes("pr*")); } public void testSimpleFieldWithWildcardExclusive() throws Exception { @@ -370,7 +380,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endArray() .endObject(); - assertXContentBuilder(expected, sample("pr*", false)); + assertXContentBuilder(expected, sampleWithExcludes("pr*")); } public void testMultipleFieldsInclusive() throws Exception { @@ -379,7 +389,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .field("pages", 456) .endObject(); - assertXContentBuilder(expected, sample(new String[] { "title", "pages" }, true)); + assertXContentBuilder(expected, sampleWithFilters(Sets.newHashSet("title", "pages"), emptySet())); } public void testMultipleFieldsExclusive() throws Exception { @@ -453,10 +463,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample(new String[] { "title", "pages" }, false)); + assertXContentBuilder(expected, sample(newXContentBuilder(emptySet(), Sets.newHashSet("title", "pages")))); } - public void testSimpleArrayInclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("tags") @@ -465,7 +474,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endArray() .endObject(); - assertXContentBuilder(expected, sample("tags", true)); + assertXContentBuilder(expected, sampleWithIncludes("tags")); } public void testSimpleArrayExclusive() throws Exception { @@ -537,10 +546,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("tags", false)); + assertXContentBuilder(expected, sampleWithExcludes("tags")); } - public void testSimpleArrayOfObjectsInclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("authors") @@ -557,9 +565,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endArray() .endObject(); - assertXContentBuilder(expected, sample("authors", true)); - assertXContentBuilder(expected, sample("authors.*", true)); - assertXContentBuilder(expected, sample("authors.*name", true)); + assertXContentBuilder(expected, sampleWithIncludes("authors")); + assertXContentBuilder(expected, sampleWithIncludes("authors.*")); + assertXContentBuilder(expected, sampleWithIncludes("authors.*name")); } public void testSimpleArrayOfObjectsExclusive() throws Exception { @@ -623,9 +631,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("authors", false)); - assertXContentBuilder(expected, sample("authors.*", false)); - assertXContentBuilder(expected, sample("authors.*name", false)); + assertXContentBuilder(expected, sampleWithExcludes("authors")); + assertXContentBuilder(expected, sampleWithExcludes("authors.*")); + assertXContentBuilder(expected, sampleWithExcludes("authors.*name")); } public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception { @@ -640,8 +648,8 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endArray() .endObject(); - assertXContentBuilder(expected, sample("authors.lastname", true)); - assertXContentBuilder(expected, sample("authors.l*", true)); + assertXContentBuilder(expected, sampleWithIncludes("authors.lastname")); + assertXContentBuilder(expected, sampleWithIncludes("authors.l*")); } public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception { @@ -715,8 +723,8 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("authors.lastname", false)); - assertXContentBuilder(expected, sample("authors.l*", false)); + assertXContentBuilder(expected, sampleWithExcludes("authors.lastname")); + assertXContentBuilder(expected, sampleWithExcludes("authors.l*")); } public void testRecurseField1Inclusive() throws Exception { @@ -768,7 +776,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("**.name", true)); + assertXContentBuilder(expected, sampleWithIncludes("**.name")); } public void testRecurseField1Exclusive() throws Exception { @@ -831,7 +839,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("**.name", false)); + assertXContentBuilder(expected, sampleWithExcludes("**.name")); } public void testRecurseField2Inclusive() throws Exception { @@ -875,7 +883,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.**.name", true)); + assertXContentBuilder(expected, sampleWithIncludes("properties.**.name")); } public void testRecurseField2Exclusive() throws Exception { @@ -940,10 +948,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.**.name", false)); + assertXContentBuilder(expected, sampleWithExcludes("properties.**.name")); } - public void testRecurseField3Inclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startObject("properties") @@ -970,7 +977,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.*.en.**.name", true)); + assertXContentBuilder(expected, sampleWithIncludes("properties.*.en.**.name")); } public void testRecurseField3Exclusive() throws Exception { @@ -1040,10 +1047,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.*.en.**.name", false)); + assertXContentBuilder(expected, sampleWithExcludes("properties.*.en.**.name")); } - public void testRecurseField4Inclusive() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startObject("properties") @@ -1072,7 +1078,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.**.distributors.name", true)); + assertXContentBuilder(expected, sampleWithIncludes("properties.**.distributors.name")); } public void testRecurseField4Exclusive() throws Exception { @@ -1140,7 +1146,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase .endObject() .endObject(); - assertXContentBuilder(expected, sample("properties.**.distributors.name", false)); + assertXContentBuilder(expected, sampleWithExcludes("properties.**.distributors.name")); } public void testRawField() throws Exception { @@ -1155,24 +1161,24 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase // Test method: rawField(String fieldName, BytesReference content) assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldFiltered, - newXContentBuilder("f*", true).startObject().field("foo", 0).rawField("raw", raw).endObject()); + newXContentBuilderWithIncludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldFiltered, - newXContentBuilder("r*", false).startObject().field("foo", 0).rawField("raw", raw).endObject()); + newXContentBuilderWithExcludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, - newXContentBuilder("r*", true).startObject().field("foo", 0).rawField("raw", raw).endObject()); + newXContentBuilderWithIncludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, - newXContentBuilder("f*", false).startObject().field("foo", 0).rawField("raw", raw).endObject()); + newXContentBuilderWithExcludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject()); // Test method: rawField(String fieldName, InputStream content) assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*", true).startObject().field("foo", 0) + assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithIncludes("f*").startObject().field("foo", 0) .rawField("raw", raw.streamInput()).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("r*", false).startObject().field("foo", 0) + assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithExcludes("r*").startObject().field("foo", 0) .rawField("raw", raw.streamInput()).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*", true).startObject().field("foo", 0) + assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithIncludes("r*").startObject().field("foo", 0) .rawField("raw", raw.streamInput()).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("f*", false).startObject().field("foo", 0) + assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithExcludes("f*").startObject().field("foo", 0) .rawField("raw", raw.streamInput()).endObject()); } @@ -1180,48 +1186,209 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase // Test: Array of values (no filtering) XContentBuilder expected = newXContentBuilder().startObject().startArray("tags").value("lorem").value("ipsum").value("dolor") .endArray().endObject(); - assertXContentBuilder(expected, newXContentBuilder("t*", true).startObject().startArray("tags").value("lorem").value("ipsum") + assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").value("lorem").value("ipsum") .value("dolor").endArray().endObject()); - assertXContentBuilder(expected, newXContentBuilder("tags", true).startObject().startArray("tags").value("lorem").value("ipsum") - .value("dolor").endArray().endObject()); - assertXContentBuilder(expected, newXContentBuilder("a", false).startObject().startArray("tags").value("lorem").value("ipsum") + assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").value("lorem") + .value("ipsum").value("dolor").endArray().endObject()); + assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").value("lorem").value("ipsum") .value("dolor").endArray().endObject()); // Test: Array of values (with filtering) - assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("foo", true).startObject() + assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithIncludes("foo").startObject() .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject()); - assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("t*", false).startObject() + assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("t*").startObject() .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject()); - assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("tags", false).startObject() + assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("tags").startObject() .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject()); // Test: Array of objects (no filtering) expected = newXContentBuilder().startObject().startArray("tags").startObject().field("lastname", "lorem").endObject().startObject() .field("firstname", "ipsum").endObject().endArray().endObject(); - assertXContentBuilder(expected, newXContentBuilder("t*", true).startObject().startArray("tags").startObject() + assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").startObject() .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); - assertXContentBuilder(expected, newXContentBuilder("tags", true).startObject().startArray("tags").startObject() + assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").startObject() .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); - assertXContentBuilder(expected, newXContentBuilder("a", false).startObject().startArray("tags").startObject() + assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").startObject() .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); // Test: Array of objects (with filtering) assertXContentBuilder(newXContentBuilder().startObject().endObject(), - newXContentBuilder("foo", true).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() + newXContentBuilderWithIncludes("foo").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() .startObject().field("firstname", "ipsum").endObject().endArray().endObject()); assertXContentBuilder(newXContentBuilder().startObject().endObject(), - newXContentBuilder("t*", false).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() + newXContentBuilderWithExcludes("t*").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() .startObject().field("firstname", "ipsum").endObject().endArray().endObject()); assertXContentBuilder(newXContentBuilder().startObject().endObject(), - newXContentBuilder("tags", false).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() + newXContentBuilderWithExcludes("tags").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject() .startObject().field("firstname", "ipsum").endObject().endArray().endObject()); // Test: Array of objects (with partial filtering) expected = newXContentBuilder().startObject().startArray("tags").startObject().field("firstname", "ipsum").endObject().endArray() .endObject(); - assertXContentBuilder(expected, newXContentBuilder("t*.firstname", true).startObject().startArray("tags").startObject() + assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*.firstname").startObject().startArray("tags").startObject() .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); - assertXContentBuilder(expected, newXContentBuilder("t*.lastname", false).startObject().startArray("tags").startObject() + assertXContentBuilder(expected, newXContentBuilderWithExcludes("t*.lastname").startObject().startArray("tags").startObject() .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); } + + public void testEmptyObject() throws IOException { + final Function build = builder -> { + try { + return builder.startObject().startObject("foo").endObject().endObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + + XContentBuilder expected = build.apply(newXContentBuilder()); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo"))); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz")))); + + expected = newXContentBuilder().startObject().endObject(); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo"))); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo")))); + } + + public void testSingleFieldObject() throws IOException { + final Function build = builder -> { + try { + return builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + + XContentBuilder expected = build.apply(newXContentBuilder()); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo.bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.baz"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.baz")))); + + expected = newXContentBuilder().startObject().endObject(); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.b*")))); + } + + public void testSingleFieldWithBothExcludesIncludes() throws IOException { + XContentBuilder expected = newXContentBuilder() + .startObject() + .field("pages", 456) + .field("price", 27.99) + .endObject(); + + assertXContentBuilder(expected, sampleWithFilters(singleton("p*"), singleton("properties"))); + } + + public void testObjectsInArrayWithBothExcludesIncludes() throws IOException { + Set includes = Sets.newHashSet("tags", "authors"); + Set excludes = singleton("authors.name"); + + XContentBuilder expected = newXContentBuilder() + .startObject() + .startArray("tags") + .value("elasticsearch") + .value("java") + .endArray() + .startArray("authors") + .startObject() + .field("lastname", "John") + .field("firstname", "Doe") + .endObject() + .startObject() + .field("lastname", "William") + .field("firstname", "Smith") + .endObject() + .endArray() + .endObject(); + + assertXContentBuilder(expected, sampleWithFilters(includes, excludes)); + } + + public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException { + Set includes = Sets.newHashSet("**.language", "properties.weight"); + Set excludes = singleton("**.distributors"); + + XContentBuilder expected = newXContentBuilder() + .startObject() + .startObject("properties") + .field("weight", 0.8d) + .startObject("language") + .startObject("en") + .field("lang", "English") + .field("available", true) + .endObject() + .startObject("fr") + .field("lang", "French") + .field("available", false) + .endObject() + .endObject() + .endObject() + .endObject(); + + assertXContentBuilder(expected, sampleWithFilters(includes, excludes)); + } + + public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException { + Set includes = singleton("**.distributors"); + Set excludes = singleton("**.distributors"); + + XContentBuilder expected = newXContentBuilder().startObject().endObject(); + assertXContentBuilder(expected, sampleWithFilters(includes, excludes)); + } + + public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException { + Set includes = singleton("**.en.*"); + Set excludes = Sets.newHashSet("**.distributors.*.name", "**.street"); + + XContentBuilder expected = newXContentBuilder() + .startObject() + .startObject("properties") + .startObject("language") + .startObject("en") + .field("lang", "English") + .field("available", true) + .startArray("distributors") + .startObject() + .field("name", "The Book Shop") + .startArray("addresses") + .startObject() + .field("city", "London") + .endObject() + .startObject() + .field("city", "Stornoway") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "Sussex Books House") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .endObject(); + + assertXContentBuilder(expected, sampleWithFilters(includes, excludes)); + } + + public void testWithLfAtEnd() throws IOException { + final Function build = builder -> { + try { + return builder.startObject().startObject("foo").field("bar", "baz").endObject().endObject().prettyPrint().lfAtEnd(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + + XContentBuilder expected = build.apply(newXContentBuilder()); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo"))); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz")))); + + expected = newXContentBuilder().startObject().endObject().prettyPrint().lfAtEnd(); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo"))); + assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar"))); + assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo")))); + } } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java index 8dbefedb249..b4d7cb11529 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -25,6 +25,8 @@ import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; + import static org.hamcrest.Matchers.equalTo; public class FilterPathGeneratorFilteringTests extends ESTestCase { @@ -135,7 +137,7 @@ public class FilterPathGeneratorFilteringTests extends ESTestCase { private void assertResult(String input, String filter, boolean inclusive, String expected) throws Exception { try (BytesStreamOutput os = new BytesStreamOutput()) { try (FilteringGeneratorDelegate generator = new FilteringGeneratorDelegate(JSON_FACTORY.createGenerator(os), - new FilterPathBasedFilter(new String[] { filter }, inclusive), true, true)) { + new FilterPathBasedFilter(Collections.singleton(filter), inclusive), true, true)) { try (JsonParser parser = JSON_FACTORY.createParser(replaceQuotes(input))) { while (parser.nextToken() != null) { generator.copyCurrentStructure(parser); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java index 80cc12b5f3b..4eec46d9b27 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java @@ -19,8 +19,12 @@ package org.elasticsearch.common.xcontent.support.filtering; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.ESTestCase; +import java.util.Set; + +import static java.util.Collections.singleton; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -33,7 +37,7 @@ public class FilterPathTests extends ESTestCase { public void testSimpleFilterPath() { final String input = "test"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -52,7 +56,7 @@ public class FilterPathTests extends ESTestCase { public void testFilterPathWithSubField() { final String input = "foo.bar"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -76,7 +80,7 @@ public class FilterPathTests extends ESTestCase { public void testFilterPathWithSubFields() { final String input = "foo.bar.quz"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -103,13 +107,13 @@ public class FilterPathTests extends ESTestCase { } public void testEmptyFilterPath() { - FilterPath[] filterPaths = FilterPath.compile(""); + FilterPath[] filterPaths = FilterPath.compile(singleton("")); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(0)); } public void testNullFilterPath() { - FilterPath[] filterPaths = FilterPath.compile((String) null); + FilterPath[] filterPaths = FilterPath.compile(singleton(null)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(0)); } @@ -117,7 +121,7 @@ public class FilterPathTests extends ESTestCase { public void testFilterPathWithEscapedDots() { String input = "w.0.0.t"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -149,7 +153,7 @@ public class FilterPathTests extends ESTestCase { input = "w\\.0\\.0\\.t"; - filterPaths = FilterPath.compile(input); + filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -167,7 +171,7 @@ public class FilterPathTests extends ESTestCase { input = "w\\.0.0\\.t"; - filterPaths = FilterPath.compile(input); + filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -188,7 +192,7 @@ public class FilterPathTests extends ESTestCase { } public void testSimpleWildcardFilterPath() { - FilterPath[] filterPaths = FilterPath.compile("*"); + FilterPath[] filterPaths = FilterPath.compile(singleton("*")); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -206,7 +210,7 @@ public class FilterPathTests extends ESTestCase { public void testWildcardInNameFilterPath() { String input = "f*o.bar"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -232,7 +236,7 @@ public class FilterPathTests extends ESTestCase { } public void testDoubleWildcardFilterPath() { - FilterPath[] filterPaths = FilterPath.compile("**"); + FilterPath[] filterPaths = FilterPath.compile(singleton("**")); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -250,7 +254,7 @@ public class FilterPathTests extends ESTestCase { public void testStartsWithDoubleWildcardFilterPath() { String input = "**.bar"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -274,7 +278,7 @@ public class FilterPathTests extends ESTestCase { public void testContainsDoubleWildcardFilterPath() { String input = "foo.**.bar"; - FilterPath[] filterPaths = FilterPath.compile(input); + FilterPath[] filterPaths = FilterPath.compile(singleton(input)); assertNotNull(filterPaths); assertThat(filterPaths, arrayWithSize(1)); @@ -302,7 +306,7 @@ public class FilterPathTests extends ESTestCase { } public void testMultipleFilterPaths() { - String[] inputs = {"foo.**.bar.*", "test.dot\\.ted"}; + Set inputs = Sets.newHashSet("foo.**.bar.*", "test.dot\\.ted"); FilterPath[] filterPaths = FilterPath.compile(inputs); assertNotNull(filterPaths); diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index eed162f2c85..ea2e28a92ec 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -276,6 +276,41 @@ curl 'localhost:9200/_segments?pretty&filter_path=indices.**.version' } -------------------------------------------------- +It is also possible to exclude one or more fields by prefixing the filter with the char `-`: + +[source,sh] +-------------------------------------------------- +curl -XGET 'localhost:9200/_count?filter_path=-_shards' +{ + "count" : 1 +} +% +-------------------------------------------------- + +And for more control, both inclusive and exclusive filters can be combined in the same expression. In +this case, the exclusive filters will be applied first and the result will be filtered again using the +inclusive filters: + +[source,sh] +-------------------------------------------------- +curl -XGET 'localhost:9200/_cluster/state?filter_path=metadata.indices.*.state,-metadata.indices.logs-*' +{ + "metadata" : { + "indices" : { + "index-1" : { + "state" : "open" + }, + "index-3" : { + "state" : "open" + }, + "index-2" : { + "state" : "open" + } + } + } +}% +-------------------------------------------------- + Note that elasticsearch sometimes returns directly the raw value of a field, like the `_source` field. If you want to filter `_source` fields, you should consider combining the already existing `_source` parameter (see diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/20_response_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/20_response_filtering.yaml index 4031f405259..432e5d8c207 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/20_response_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/20_response_filtering.yaml @@ -152,3 +152,49 @@ - is_false: nodes.$master.fs.data.0.path - is_true: nodes.$master.fs.data.0.type - is_true: nodes.$master.fs.data.0.total_in_bytes + +--- +"Nodes Stats filtered using both includes and excludes filters": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + # Nodes Stats with "nodes" field but no JVM stats + - do: + nodes.stats: + filter_path: [ "nodes", "-nodes.*.jvm", "-nodes.*.indices" ] + + - is_false: cluster_name + - is_true: nodes + - is_true: nodes.$master.name + - is_true: nodes.$master.os + - is_false: nodes.$master.indices + - is_false: nodes.$master.jvm + + # Nodes Stats with "nodes.*.indices" field and sub-fields but no indices segments + - do: + nodes.stats: + filter_path: "nodes.*.indices,-nodes.*.indices.segments" + + - is_false: cluster_name + - is_true: nodes + - is_false: nodes.$master.name + - is_true: nodes.$master.indices + - is_true: nodes.$master.indices.docs + - is_false: nodes.$master.indices.segments + + # Nodes Stats with "nodes.*.fs.data.t*" fields but no "type" field + - do: + nodes.stats: + filter_path: "nodes.*.fs.data.t*,-**.type" + + - is_false: cluster_name + - is_true: nodes + - is_false: nodes.$master.name + - is_false: nodes.$master.indices + - is_false: nodes.$master.jvm + - is_true: nodes.$master.fs.data + - is_false: nodes.$master.fs.data.0.type + - is_true: nodes.$master.fs.data.0.total_in_bytes diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml index e7d51c5bced..bff9a169604 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml @@ -84,3 +84,64 @@ - is_true: hits.hits.1._index - is_false: hits.hits.1._type - is_true: hits.hits.1._id + +--- +"Search results filtered using both includes and excludes filters": + - do: + bulk: + refresh: true + body: | + {"index": {"_index": "index-1", "_type": "type-1", "_id": "1"}} + {"name": "First document", "properties": {"size": 123, "meta": {"foo": "bar"}}} + {"index": {"_index": "index-1", "_type": "type-1", "_id": "2"}} + {"name": "Second document", "properties": {"size": 465, "meta": {"foo": "bar", "baz": "qux"}}} + + - do: + search: + filter_path: [ "-**._source.properties", "**._source" ] + body: { query: { match_all: {} } } + + - is_false: took + - is_true: hits.hits.0._source + - is_true: hits.hits.0._source.name + - is_false: hits.hits.0._source.properties + - is_true: hits.hits.1._source + - is_true: hits.hits.1._source.name + - is_false: hits.hits.1._source.properties + + - do: + search: + filter_path: [ "**.properties" , "-hits.hits._source.properties.meta" ] + body: { query: { match_all: {} } } + + - is_false: took + - is_true: hits.hits.0._source + - is_false: hits.hits.0._source.name + - is_true: hits.hits.0._source.properties + - is_true: hits.hits.0._source.properties.size + - is_false: hits.hits.0._source.properties.meta + - is_true: hits.hits.1._source + - is_false: hits.hits.1._source.name + - is_true: hits.hits.1._source.properties + - is_true: hits.hits.1._source.properties.size + - is_false: hits.hits.1._source.properties.meta + + - do: + search: + filter_path: "**._source,-**.meta.foo" + body: { query: { match_all: {} } } + + - is_false: took + - is_true: hits.hits.0._source + - is_true: hits.hits.0._source.name + - is_true: hits.hits.0._source.properties + - is_true: hits.hits.0._source.properties.size + - is_false: hits.hits.0._source.properties.meta.foo + + - do: + count: + filter_path: "-*" + body: { query: { match_all: {} } } + + - is_false: count + - is_false: _shards