diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 54826e963cb..00d905aa140 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1255,6 +1255,8 @@ public class SearchIT extends ESRestHighLevelClientTestCase { FieldCapabilitiesResponse response = execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); + assertEquals(new String[] {"index1", "index2"}, response.getIndices()); + // Check the capabilities for the 'rating' field. assertTrue(response.get().containsKey("rating")); Map ratingResponse = response.getField("rating"); diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 034bc92930b..d150e8e5fa6 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -71,6 +71,7 @@ GET _field_caps?fields=rating,title -------------------------------------------------- { "fields": { + "indices": ["index1", "index2", "index3", "index4", "index5"], "rating": { <1> "long": { "searchable": true, @@ -103,9 +104,61 @@ and as a `keyword` in `index3` and `index4`. <3> The field `rating` is not searchable in `index4`. <4> The field `title` is defined as `text` in all indices. +[float] +=== Unmapped fields +By default unmapped fields are ignored. You can include them in the response by +adding a parameter called `include_unmapped` in the request: +[source,js] +-------------------------------------------------- +GET _field_caps?fields=rating,title&include_unmapped +-------------------------------------------------- +// CONSOLE +In which case the response will contain an entry for each field that is present in +some indices but not all: +[source,js] +-------------------------------------------------- +{ + "fields": { + "indices": ["index1", "index2", "index3"], + "rating": { + "long": { + "searchable": true, + "aggregatable": false, + "indices": ["index1", "index2"], + "non_aggregatable_indices": ["index1"] + }, + "keyword": { + "searchable": false, + "aggregatable": true, + "indices": ["index3", "index4"], + "non_searchable_indices": ["index4"] + }, + "unmapped": { <1> + "indices": ["index5"], + "searchable": false, + "aggregatable": false + } + }, + "title": { + "text": { + "indices": ["index1", "index2", "index3", "index4"], + "searchable": true, + "aggregatable": false + }, + "unmapped": { <2> + "indices": ["index5"] + "searchable": false, + "aggregatable": false + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE - +<1> The `rating` field is unmapped` in `index5`. +<2> The `title` field is unmapped` in `index5`. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json index 3ba09ca314b..ebb520fd433 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json @@ -32,6 +32,11 @@ "options" : ["open","closed","none","all"], "default" : "open", "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "include_unmapped": { + "type": "boolean", + "default": false, + "description": "Indicates whether unmapped fields should be included in the response." } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml index fee9933bb85..74fbba14f92 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -14,8 +14,8 @@ setup: type: double geo: type: geo_point - date: - type: date + misc: + type: text object: type: object properties: @@ -299,3 +299,28 @@ setup: - match: {fields.geo.keyword.indices: ["test3"]} - is_false: fields.geo.keyword.non_searchable_indices - is_false: fields.geo.keyword.on_aggregatable_indices + +--- +"Field caps with include_unmapped": + - skip: + version: " - 7.0.99" + reason: include_unmapped has been added in 7.1.0 + + - do: + field_caps: + include_unmapped: true + index: 'test1,test2,test3' + fields: [text, misc] + + - match: {fields.text.text.searchable: true} + - match: {fields.text.text.aggregatable: false} + - is_false: fields.text.text.indices + - is_false: fields.text.text.non_searchable_indices + - is_false: fields.text.text.non_aggregatable_indices + - match: {fields.misc.text.searchable: true} + - match: {fields.misc.text.aggregatable: false} + - match: {fields.misc.text.indices: ["test1"]} + - match: {fields.misc.unmapped.searchable: false} + - match: {fields.misc.unmapped.aggregatable: false} + - match: {fields.misc.unmapped.indices: ["test2", "test3"]} + diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 5cfdba92946..20f525716a2 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.fieldcaps; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -34,6 +35,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * Describes the capabilities of a field optionally merged across multiple indices. @@ -214,30 +217,30 @@ public class FieldCapabilities implements Writeable, ToXContentObject { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - FieldCapabilities that = (FieldCapabilities) o; - - if (isSearchable != that.isSearchable) return false; - if (isAggregatable != that.isAggregatable) return false; - if (!name.equals(that.name)) return false; - if (!type.equals(that.type)) return false; - if (!Arrays.equals(indices, that.indices)) return false; - if (!Arrays.equals(nonSearchableIndices, that.nonSearchableIndices)) return false; - return Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices); + return isSearchable == that.isSearchable && + isAggregatable == that.isAggregatable && + Objects.equals(name, that.name) && + Objects.equals(type, that.type) && + Arrays.equals(indices, that.indices) && + Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) && + Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices); } @Override public int hashCode() { - int result = name.hashCode(); - result = 31 * result + type.hashCode(); - result = 31 * result + (isSearchable ? 1 : 0); - result = 31 * result + (isAggregatable ? 1 : 0); + int result = Objects.hash(name, type, isSearchable, isAggregatable); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); return result; } + @Override + public String toString() { + return Strings.toString(this); + } + static class Builder { private String name; private String type; @@ -260,6 +263,10 @@ public class FieldCapabilities implements Writeable, ToXContentObject { this.isAggregatable &= agg; } + List getIndices() { + return indiceList.stream().map(c -> c.name).collect(Collectors.toList()); + } + FieldCapabilities build(boolean withIndices) { final String[] indices; /* Eclipse can't deal with o -> o.name, maybe because of diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java index fb7e32ed9ab..05cc48174da 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java @@ -29,8 +29,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -public class FieldCapabilitiesIndexRequest - extends SingleShardRequest { +public class FieldCapabilitiesIndexRequest extends SingleShardRequest { private String[] fields; private OriginalIndices originalIndices; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index 1e468624516..8295e1aad25 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Writeable; import java.io.IOException; import java.util.Map; +import java.util.Objects; /** * Response for {@link FieldCapabilitiesIndexRequest} requests. @@ -89,14 +90,13 @@ public class FieldCapabilitiesIndexResponse extends ActionResponse implements Wr public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - FieldCapabilitiesIndexResponse that = (FieldCapabilitiesIndexResponse) o; - - return responseMap.equals(that.responseMap); + return Objects.equals(indexName, that.indexName) && + Objects.equals(responseMap, that.responseMap); } @Override public int hashCode() { - return responseMap.hashCode(); + return Objects.hash(indexName, responseMap); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index e9e77df5f90..e34c55eb99f 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -44,6 +45,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind private String[] indices = Strings.EMPTY_ARRAY; private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); private String[] fields = Strings.EMPTY_ARRAY; + private boolean includeUnmapped = false; // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged private boolean mergeResults = true; @@ -51,8 +53,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind new ObjectParser<>(NAME, FieldCapabilitiesRequest::new); static { - PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields), - FIELDS_FIELD); + PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields), FIELDS_FIELD); } public FieldCapabilitiesRequest() {} @@ -83,6 +84,11 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); mergeResults = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_1_0)) { + includeUnmapped = in.readBoolean(); + } else { + includeUnmapped = false; + } } @Override @@ -92,6 +98,9 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); out.writeBoolean(mergeResults); + if (out.getVersion().onOrAfter(Version.V_7_1_0)) { + out.writeBoolean(includeUnmapped); + } } /** @@ -123,6 +132,11 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind return this; } + public FieldCapabilitiesRequest includeUnmapped(boolean includeUnmapped) { + this.includeUnmapped = includeUnmapped; + return this; + } + @Override public String[] indices() { return indices; @@ -133,12 +147,15 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind return indicesOptions; } + public boolean includeUnmapped() { + return includeUnmapped; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (fields == null || fields.length == 0) { - validationException = - ValidateActions.addValidationError("no fields specified", validationException); + validationException = ValidateActions.addValidationError("no fields specified", validationException); } return validationException; } @@ -152,14 +169,12 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind return Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && Arrays.equals(fields, that.fields) && - Objects.equals(mergeResults, that.mergeResults); + Objects.equals(mergeResults, that.mergeResults) && + includeUnmapped == that.includeUnmapped; } @Override public int hashCode() { - return Objects.hash(Arrays.hashCode(indices), - indicesOptions, - Arrays.hashCode(fields), - mergeResults); + return Objects.hash(Arrays.hashCode(indices), indicesOptions, Arrays.hashCode(fields), mergeResults, includeUnmapped); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 8da925e8fc9..477bf3c733d 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -36,4 +36,9 @@ public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder> responseMap; private List indexResponses; - FieldCapabilitiesResponse(Map> responseMap) { - this(responseMap, Collections.emptyList()); + FieldCapabilitiesResponse(String[] indices, Map> responseMap) { + this(indices, responseMap, Collections.emptyList()); } FieldCapabilitiesResponse(List indexResponses) { - this(Collections.emptyMap(), indexResponses); + this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses); } - private FieldCapabilitiesResponse(Map> responseMap, + private FieldCapabilitiesResponse(String[] indices, Map> responseMap, List indexResponses) { this.responseMap = Objects.requireNonNull(responseMap); this.indexResponses = Objects.requireNonNull(indexResponses); + this.indices = indices; } /** * Used for serialization */ FieldCapabilitiesResponse() { - this(Collections.emptyMap(), Collections.emptyList()); + this(Strings.EMPTY_ARRAY, Collections.emptyMap(), Collections.emptyList()); } + /** + * Get the concrete list of indices that were requested. + */ + public String[] getIndices() { + return indices; + } + + /** * Get the field capabilities map. */ @@ -94,8 +108,12 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - this.responseMap = - in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField); + if (in.getVersion().onOrAfter(Version.V_7_1_0)) { + indices = in.readStringArray(); + } else { + indices = Strings.EMPTY_ARRAY; + } + this.responseMap = in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField); indexResponses = in.readList(FieldCapabilitiesIndexResponse::new); } @@ -106,20 +124,27 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_1_0)) { + out.writeStringArray(indices); + } out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField); out.writeList(indexResponses); } - private static void writeField(StreamOutput out, - Map map) throws IOException { + private static void writeField(StreamOutput out, Map map) throws IOException { out.writeMap(map, StreamOutput::writeString, (valueOut, fc) -> fc.writeTo(valueOut)); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() - .field(FIELDS_FIELD.getPreferredName(), responseMap) - .endObject(); + if (indexResponses.size() > 0) { + throw new IllegalStateException("cannot serialize non-merged response"); + } + builder.startObject(); + builder.field(INDICES_FIELD.getPreferredName(), indices); + builder.field(FIELDS_FIELD.getPreferredName(), responseMap); + builder.endObject(); + return builder; } public static FieldCapabilitiesResponse fromXContent(XContentParser parser) throws IOException { @@ -129,11 +154,14 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("field_capabilities_response", true, - a -> new FieldCapabilitiesResponse( - ((List>>) a[0]).stream() - .collect(Collectors.toMap(Tuple::v1, Tuple::v2)))); + a -> { + List indices = a[0] == null ? Collections.emptyList() : (List) a[0]; + return new FieldCapabilitiesResponse(indices.stream().toArray(String[]::new), + ((List>>) a[1]).stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + }); static { + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> { Map typeToCapabilities = parseTypeToCapabilities(p, n); return new Tuple<>(n, typeToCapabilities); @@ -158,14 +186,21 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; - - return responseMap.equals(that.responseMap); + return Arrays.equals(indices, that.indices) && + Objects.equals(responseMap, that.responseMap) && + Objects.equals(indexResponses, that.indexResponses); } @Override public int hashCode() { - return responseMap.hashCode(); + int result = Objects.hash(responseMap, indexResponses); + result = 31 * result + Arrays.hashCode(indices); + return result; + } + + @Override + public String toString() { + return Strings.toString(this); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 25b075d8d90..dcd52094ff6 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -37,10 +37,13 @@ import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class TransportFieldCapabilitiesAction extends HandledTransportAction { private final ThreadPool threadPool; @@ -75,20 +78,21 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction indexResponses = Collections.synchronizedList(new ArrayList<>()); final Runnable onResponse = () -> { if (completionCounter.countDown()) { if (request.isMergeResults()) { - listener.onResponse(merge(indexResponses)); + listener.onResponse(merge(allIndices, indexResponses, request.includeUnmapped())); } else { listener.onResponse(new FieldCapabilitiesResponse(indexResponses)); } } }; if (totalNumRequest == 0) { - listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyMap())); + listener.onResponse(new FieldCapabilitiesResponse(allIndices, Collections.emptyMap())); } else { ActionListener innerListener = new ActionListener() { @Override @@ -129,35 +133,61 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction indexResponses) { - Map> responseMapBuilder = new HashMap<> (); + private String[] mergeIndiceNames(String[] localIndices, Map remoteIndices) { + Set allIndices = new HashSet<>(); + Arrays.stream(localIndices).forEach(allIndices::add); + for (Map.Entry entry : remoteIndices.entrySet()) { + for (String index : entry.getValue().indices()) { + allIndices.add(RemoteClusterAware.buildRemoteIndexName(entry.getKey(), index)); + } + } + return allIndices.stream().toArray(String[]::new); + } + + private FieldCapabilitiesResponse merge(String[] indices, List indexResponses, + boolean includeUnmapped) { + final Map> responseMapBuilder = new HashMap<> (); for (FieldCapabilitiesIndexResponse response : indexResponses) { innerMerge(responseMapBuilder, response.getIndexName(), response.get()); } - - Map> responseMap = new HashMap<>(); - for (Map.Entry> entry : - responseMapBuilder.entrySet()) { - Map typeMap = new HashMap<>(); - boolean multiTypes = entry.getValue().size() > 1; - for (Map.Entry fieldEntry : - entry.getValue().entrySet()) { + final Map> responseMap = new HashMap<>(); + for (Map.Entry> entry : responseMapBuilder.entrySet()) { + final Map typeMapBuilder = entry.getValue(); + if (includeUnmapped) { + addUnmappedFields(indices, entry.getKey(), typeMapBuilder); + } + boolean multiTypes = typeMapBuilder.size() > 1; + final Map typeMap = new HashMap<>(); + for (Map.Entry fieldEntry : typeMapBuilder.entrySet()) { typeMap.put(fieldEntry.getKey(), fieldEntry.getValue().build(multiTypes)); } - responseMap.put(entry.getKey(), typeMap); + responseMap.put(entry.getKey(), Collections.unmodifiableMap(typeMap)); } - return new FieldCapabilitiesResponse(responseMap); + return new FieldCapabilitiesResponse(indices, Collections.unmodifiableMap(responseMap)); } - private void innerMerge(Map> responseMapBuilder, String indexName, - Map map) { + private void addUnmappedFields(String[] indices, String field, Map typeMap) { + Set unmappedIndices = new HashSet<>(); + Arrays.stream(indices).forEach(unmappedIndices::add); + typeMap.values().stream().forEach((b) -> b.getIndices().stream().forEach(unmappedIndices::remove)); + if (unmappedIndices.isEmpty() == false) { + FieldCapabilities.Builder unmapped = new FieldCapabilities.Builder(field, "unmapped"); + typeMap.put("unmapped", unmapped); + for (String index : unmappedIndices) { + unmapped.add(index, false, false); + } + } + } + + private void innerMerge(Map> responseMapBuilder, + String indexName, Map map) { for (Map.Entry entry : map.entrySet()) { final String field = entry.getKey(); final FieldCapabilities fieldCap = entry.getValue(); Map typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>()); - FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field, - key)); + FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), + key -> new FieldCapabilities.Builder(field, key)); builder.add(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable()); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java index 4b9d37ace91..24e26713ed6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java @@ -57,6 +57,7 @@ public class RestFieldCapabilitiesAction extends BaseRestHandler { fieldRequest.indicesOptions( IndicesOptions.fromRequest(request, fieldRequest.indicesOptions())); + fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false)); return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java index 4e4bd7fdf55..dbab3104d44 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java @@ -50,6 +50,7 @@ public class FieldCapabilitiesRequestTests extends AbstractStreamableTestCase request.setMergeResults(!request.isMergeResults())); + mutators.add(request -> request.includeUnmapped(!request.includeUnmapped())); FieldCapabilitiesRequest mutatedInstance = copyInstance(instance); Consumer mutator = randomFrom(mutators); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 90b730660dd..e75dede451b 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -19,32 +19,18 @@ package org.elasticsearch.action.fieldcaps; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.test.AbstractStreamableTestCase; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Predicate; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLength; -public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTestCase { - - @Override - protected FieldCapabilitiesResponse doParseInstance(XContentParser parser) throws IOException { - return FieldCapabilitiesResponse.fromXContent(parser); - } - +public class FieldCapabilitiesResponseTests extends AbstractStreamableTestCase { @Override protected FieldCapabilitiesResponse createBlankInstance() { return new FieldCapabilitiesResponse(); @@ -52,36 +38,15 @@ public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTe @Override protected FieldCapabilitiesResponse createTestInstance() { - if (randomBoolean()) { - // merged responses - Map> responses = new HashMap<>(); + List responses = new ArrayList<>(); + int numResponse = randomIntBetween(0, 10); - String[] fields = generateRandomStringArray(5, 10, false, true); - assertNotNull(fields); - - for (String field : fields) { - Map typesToCapabilities = new HashMap<>(); - String[] types = generateRandomStringArray(5, 10, false, false); - assertNotNull(types); - - for (String type : types) { - typesToCapabilities.put(type, FieldCapabilitiesTests.randomFieldCaps(field)); - } - responses.put(field, typesToCapabilities); - } - return new FieldCapabilitiesResponse(responses); - } else { - // non-merged responses - List responses = new ArrayList<>(); - int numResponse = randomIntBetween(0, 10); - for (int i = 0; i < numResponse; i++) { - responses.add(createRandomIndexResponse()); - } - return new FieldCapabilitiesResponse(responses); + for (int i = 0; i < numResponse; i++) { + responses.add(createRandomIndexResponse()); } + return new FieldCapabilitiesResponse(responses); } - private FieldCapabilitiesIndexResponse createRandomIndexResponse() { Map responses = new HashMap<>(); @@ -118,78 +83,6 @@ public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTe FieldCapabilitiesTests.randomFieldCaps(toReplace))); break; } - return new FieldCapabilitiesResponse(mutatedResponses); - } - - @Override - protected Predicate getRandomFieldsExcludeFilter() { - // Disallow random fields from being inserted under the 'fields' key, as this - // map only contains field names, and also under 'fields.FIELD_NAME', as these - // maps only contain type names. - return field -> field.matches("fields(\\.\\w+)?"); - } - - public void testToXContent() throws IOException { - FieldCapabilitiesResponse response = createSimpleResponse(); - - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - response.toXContent(builder, ToXContent.EMPTY_PARAMS); - - String generatedResponse = BytesReference.bytes(builder).utf8ToString(); - assertEquals(( - "{" + - " \"fields\": {" + - " \"rating\": { " + - " \"keyword\": {" + - " \"type\": \"keyword\"," + - " \"searchable\": false," + - " \"aggregatable\": true," + - " \"indices\": [\"index3\", \"index4\"]," + - " \"non_searchable_indices\": [\"index4\"] " + - " }," + - " \"long\": {" + - " \"type\": \"long\"," + - " \"searchable\": true," + - " \"aggregatable\": false," + - " \"indices\": [\"index1\", \"index2\"]," + - " \"non_aggregatable_indices\": [\"index1\"] " + - " }" + - " }," + - " \"title\": { " + - " \"text\": {" + - " \"type\": \"text\"," + - " \"searchable\": true," + - " \"aggregatable\": false" + - " }" + - " }" + - " }" + - "}").replaceAll("\\s+", ""), generatedResponse); - } - - public void testEmptyResponse() throws IOException { - FieldCapabilitiesResponse testInstance = new FieldCapabilitiesResponse(); - assertSerialization(testInstance); - } - - private static FieldCapabilitiesResponse createSimpleResponse() { - Map titleCapabilities = new HashMap<>(); - titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false)); - - Map ratingCapabilities = new HashMap<>(); - ratingCapabilities.put("long", new FieldCapabilities("rating", "long", - true, false, - new String[]{"index1", "index2"}, - null, - new String[]{"index1"})); - ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", - false, true, - new String[]{"index3", "index4"}, - new String[]{"index4"}, - null)); - - Map> responses = new HashMap<>(); - responses.put("title", titleCapabilities); - responses.put("rating", ratingCapabilities); - return new FieldCapabilitiesResponse(responses); + return new FieldCapabilitiesResponse(null, mutatedResponses); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index 5dc27f8e6ec..deeae3351ec 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -86,8 +86,8 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase { + + @Override + protected FieldCapabilitiesResponse doParseInstance(XContentParser parser) throws IOException { + return FieldCapabilitiesResponse.fromXContent(parser); + } + + @Override + protected FieldCapabilitiesResponse createBlankInstance() { + return new FieldCapabilitiesResponse(); + } + + @Override + protected FieldCapabilitiesResponse createTestInstance() { + // merged responses + Map> responses = new HashMap<>(); + + String[] fields = generateRandomStringArray(5, 10, false, true); + assertNotNull(fields); + + for (String field : fields) { + Map typesToCapabilities = new HashMap<>(); + String[] types = generateRandomStringArray(5, 10, false, false); + assertNotNull(types); + + for (String type : types) { + typesToCapabilities.put(type, FieldCapabilitiesTests.randomFieldCaps(field)); + } + responses.put(field, typesToCapabilities); + } + int numIndices = randomIntBetween(1, 10); + String[] indices = new String[numIndices]; + for (int i = 0; i < numIndices; i++) { + indices[i] = randomAlphaOfLengthBetween(5, 10); + } + return new FieldCapabilitiesResponse(indices, responses); + } + + @Override + protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse response) { + Map> mutatedResponses = new HashMap<>(response.get()); + + int mutation = response.get().isEmpty() ? 0 : randomIntBetween(0, 2); + + switch (mutation) { + case 0: + String toAdd = randomAlphaOfLength(10); + mutatedResponses.put(toAdd, Collections.singletonMap( + randomAlphaOfLength(10), + FieldCapabilitiesTests.randomFieldCaps(toAdd))); + break; + case 1: + String toRemove = randomFrom(mutatedResponses.keySet()); + mutatedResponses.remove(toRemove); + break; + case 2: + String toReplace = randomFrom(mutatedResponses.keySet()); + mutatedResponses.put(toReplace, Collections.singletonMap( + randomAlphaOfLength(10), + FieldCapabilitiesTests.randomFieldCaps(toReplace))); + break; + } + return new FieldCapabilitiesResponse(null, mutatedResponses); + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // Disallow random fields from being inserted under the 'fields' key, as this + // map only contains field names, and also under 'fields.FIELD_NAME', as these + // maps only contain type names. + return field -> field.matches("fields(\\.\\w+)?"); + } + + public void testToXContent() throws IOException { + FieldCapabilitiesResponse response = createSimpleResponse(); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + String generatedResponse = BytesReference.bytes(builder).utf8ToString(); + assertEquals(( + "{" + + " \"indices\": null," + + " \"fields\": {" + + " \"rating\": { " + + " \"keyword\": {" + + " \"type\": \"keyword\"," + + " \"searchable\": false," + + " \"aggregatable\": true," + + " \"indices\": [\"index3\", \"index4\"]," + + " \"non_searchable_indices\": [\"index4\"] " + + " }," + + " \"long\": {" + + " \"type\": \"long\"," + + " \"searchable\": true," + + " \"aggregatable\": false," + + " \"indices\": [\"index1\", \"index2\"]," + + " \"non_aggregatable_indices\": [\"index1\"] " + + " }" + + " }," + + " \"title\": { " + + " \"text\": {" + + " \"type\": \"text\"," + + " \"searchable\": true," + + " \"aggregatable\": false" + + " }" + + " }" + + " }" + + "}").replaceAll("\\s+", ""), generatedResponse); + } + + public void testEmptyResponse() throws IOException { + FieldCapabilitiesResponse testInstance = new FieldCapabilitiesResponse(); + assertSerialization(testInstance); + } + + private static FieldCapabilitiesResponse createSimpleResponse() { + Map titleCapabilities = new HashMap<>(); + titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false)); + + Map ratingCapabilities = new HashMap<>(); + ratingCapabilities.put("long", new FieldCapabilities("rating", "long", + true, false, + new String[]{"index1", "index2"}, + null, + new String[]{"index1"})); + ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", + false, true, + new String[]{"index3", "index4"}, + new String[]{"index4"}, + null)); + + Map> responses = new HashMap<>(); + responses.put("title", titleCapabilities); + responses.put("rating", ratingCapabilities); + return new FieldCapabilitiesResponse(null, responses); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 07a80a31deb..2608e164972 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -113,7 +113,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase { } private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, Collection expectedFields) { - Map> responseMap = fieldCapabilitiesResponse.get(); + Map> responseMap = new HashMap<>(fieldCapabilitiesResponse.get()); Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields(); for (String field : builtInMetaDataFields) { Map remove = responseMap.remove(field); diff --git a/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 14640b191a2..7adc447a207 100644 --- a/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Map; @@ -60,6 +61,13 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { .field("type", "alias") .field("path", "playlist") .endObject() + .startObject("old_field") + .field("type", "long") + .endObject() + .startObject("new_field") + .field("type", "alias") + .field("path", "old_field") + .endObject() .endObject() .endObject() .endObject(); @@ -75,10 +83,14 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { .startObject("route_length_miles") .field("type", "double") .endObject() + .startObject("new_field") + .field("type", "long") + .endObject() .endObject() .endObject() .endObject(); assertAcked(prepareCreate("new_index").addMapping("_doc", newIndexMapping)); + assertAcked(client().admin().indices().prepareAliases().addAlias("new_index", "current")); } public static class FieldFilterPlugin extends Plugin implements MapperPlugin { @@ -94,9 +106,9 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { } public void testFieldAlias() { - FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "route_length_miles") - .get(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "route_length_miles").get(); + assertIndices(response, "old_index", "new_index"); // Ensure the response has entries for both requested fields. assertTrue(response.get().containsKey("distance")); assertTrue(response.get().containsKey("route_length_miles")); @@ -126,26 +138,73 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { } public void testFieldAliasWithWildcard() { - FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("route*") - .get(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("route*").get(); + assertIndices(response, "old_index", "new_index"); assertEquals(1, response.get().size()); assertTrue(response.get().containsKey("route_length_miles")); } public void testFieldAliasFiltering() { - FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields( - "secret-soundtrack", "route_length_miles") - .get(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("secret-soundtrack", "route_length_miles").get(); + assertIndices(response, "old_index", "new_index"); assertEquals(1, response.get().size()); assertTrue(response.get().containsKey("route_length_miles")); } public void testFieldAliasFilteringWithWildcard() { - FieldCapabilitiesResponse response = client().prepareFieldCaps() - .setFields("distance", "secret*") - .get(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "secret*").get(); + assertIndices(response, "old_index", "new_index"); assertEquals(1, response.get().size()); assertTrue(response.get().containsKey("distance")); } + + public void testWithUnmapped() { + FieldCapabilitiesResponse response = client().prepareFieldCaps() + .setFields("new_field", "old_field") + .setIncludeUnmapped(true) + .get(); + assertIndices(response, "old_index", "new_index"); + + assertEquals(2, response.get().size()); + assertTrue(response.get().containsKey("old_field")); + + Map oldField = response.getField("old_field"); + assertEquals(2, oldField.size()); + + assertTrue(oldField.containsKey("long")); + assertEquals( + new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null), + oldField.get("long")); + + assertTrue(oldField.containsKey("unmapped")); + assertEquals( + new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null), + oldField.get("unmapped")); + + Map newField = response.getField("new_field"); + assertEquals(1, newField.size()); + + assertTrue(newField.containsKey("long")); + assertEquals( + new FieldCapabilities("new_field", "long", true, true), + newField.get("long")); + } + + public void testWithIndexAlias() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").get(); + assertIndices(response, "new_index"); + + FieldCapabilitiesResponse response1 = client().prepareFieldCaps("current", "old_index").setFields("*").get(); + assertIndices(response1, "old_index", "new_index"); + FieldCapabilitiesResponse response2 = client().prepareFieldCaps("current", "old_index", "new_index").setFields("*").get(); + assertEquals(response1, response2); + } + + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { + assertNotNull(response.getIndices()); + Arrays.sort(indices); + Arrays.sort(response.getIndices()); + assertArrayEquals(indices, response.getIndices()); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index b28b379494c..ce235895e0f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -410,7 +410,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase { } private static void assertExpectedFields(FieldCapabilitiesResponse fieldCapabilitiesResponse, String... expectedFields) { - Map> responseMap = fieldCapabilitiesResponse.get(); + Map> responseMap = new HashMap<>(fieldCapabilitiesResponse.get()); Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields(); for (String field : builtInMetaDataFields) { Map remove = responseMap.remove(field);