honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle unmapped fields in _field_caps API (#34071) (#41426) 

Today the `_field_caps` API returns the list of indices where a field
is present only if this field has different types within the requested indices.
However if the request is an index pattern (or an alias, or both...) there
is no way to infer the indices if the response contains only fields that have
the same type in all indices. This commit changes the response to always return
the list of indices in the response. It also adds a way to retrieve unmapped field
in a specific section per field called `unmapped`. This section is created for each field
that is present in some indices but not all if the parameter `include_unmapped` is set to
true in the request (defaults to false).
This commit is contained in:
Jim Ferenczi 2019-04-25 18:13:48 +02:00 committed by GitHub
parent 40aef2b8aa
commit 6184efaff6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 501 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
View File

@ -1255,6 +1255,8 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
FieldCapabilitiesResponse response = execute(request,
highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync);
assertEquals(new String[] {"index1", "index2"}, response.getIndices());
// Check the capabilities for the 'rating' field.
assertTrue(response.get().containsKey("rating"));
Map<String, FieldCapabilities> ratingResponse = response.getField("rating");

55
docs/reference/search/field-caps.asciidoc
View File

@ -71,6 +71,7 @@ GET _field_caps?fields=rating,title
--------------------------------------------------
{
"fields": {
"indices": ["index1", "index2", "index3", "index4", "index5"],
"rating": { <1>
"long": {
"searchable": true,
@ -103,9 +104,61 @@ and as a `keyword` in `index3` and `index4`.
<3> The field `rating` is not searchable in `index4`.
<4> The field `title` is defined as `text` in all indices.
[float]
=== Unmapped fields
By default unmapped fields are ignored. You can include them in the response by
adding a parameter called `include_unmapped` in the request:
[source,js]
--------------------------------------------------
GET _field_caps?fields=rating,title&include_unmapped
--------------------------------------------------
// CONSOLE
In which case the response will contain an entry for each field that is present in
some indices but not all:
[source,js]
--------------------------------------------------
{
"fields": {
"indices": ["index1", "index2", "index3"],
"rating": {
"long": {
"searchable": true,
"aggregatable": false,
"indices": ["index1", "index2"],
"non_aggregatable_indices": ["index1"]
},
"keyword": {
"searchable": false,
"aggregatable": true,
"indices": ["index3", "index4"],
"non_searchable_indices": ["index4"]
},
"unmapped": { <1>
"indices": ["index5"],
"searchable": false,
"aggregatable": false
}
},
"title": {
"text": {
"indices": ["index1", "index2", "index3", "index4"],
"searchable": true,
"aggregatable": false
},
"unmapped": { <2>
"indices": ["index5"]
"searchable": false,
"aggregatable": false
}
}
}
}
--------------------------------------------------
// NOTCONSOLE
<1> The `rating` field is unmapped` in `index5`.
<2> The `title` field is unmapped` in `index5`.

5
rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
View File

@ -32,6 +32,11 @@
"options" : ["open","closed","none","all"],
"default" : "open",
"description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
},
"include_unmapped": {
"type": "boolean",
"default": false,
"description": "Indicates whether unmapped fields should be included in the response."
}
}
},

29
rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml
View File

@ -14,8 +14,8 @@ setup:
type: double
geo:
type: geo_point
date:
type: date
misc:
type: text
object:
type: object
properties:
@ -299,3 +299,28 @@ setup:
- match: {fields.geo.keyword.indices: ["test3"]}
- is_false: fields.geo.keyword.non_searchable_indices
- is_false: fields.geo.keyword.on_aggregatable_indices
---
"Field caps with include_unmapped":
- skip:
version: " - 7.0.99"
reason: include_unmapped has been added in 7.1.0
- do:
field_caps:
include_unmapped: true
index: 'test1,test2,test3'
fields: [text, misc]
- match: {fields.text.text.searchable: true}
- match: {fields.text.text.aggregatable: false}
- is_false: fields.text.text.indices
- is_false: fields.text.text.non_searchable_indices
- is_false: fields.text.text.non_aggregatable_indices
- match: {fields.misc.text.searchable: true}
- match: {fields.misc.text.aggregatable: false}
- match: {fields.misc.text.indices: ["test1"]}
- match: {fields.misc.unmapped.searchable: false}
- match: {fields.misc.unmapped.aggregatable: false}
- match: {fields.misc.unmapped.indices: ["test2", "test3"]}

33
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.action.fieldcaps;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -34,6 +35,8 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* Describes the capabilities of a field optionally merged across multiple indices.
@ -214,30 +217,30 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FieldCapabilities that = (FieldCapabilities) o;
if (isSearchable != that.isSearchable) return false;
if (isAggregatable != that.isAggregatable) return false;
if (!name.equals(that.name)) return false;
if (!type.equals(that.type)) return false;
if (!Arrays.equals(indices, that.indices)) return false;
if (!Arrays.equals(nonSearchableIndices, that.nonSearchableIndices)) return false;
return Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices);
return isSearchable == that.isSearchable &&
isAggregatable == that.isAggregatable &&
Objects.equals(name, that.name) &&
Objects.equals(type, that.type) &&
Arrays.equals(indices, that.indices) &&
Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) &&
Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices);
}
@Override
public int hashCode() {
int result = name.hashCode();
result = 31 * result + type.hashCode();
result = 31 * result + (isSearchable ? 1 : 0);
result = 31 * result + (isAggregatable ? 1 : 0);
int result = Objects.hash(name, type, isSearchable, isAggregatable);
result = 31 * result + Arrays.hashCode(indices);
result = 31 * result + Arrays.hashCode(nonSearchableIndices);
result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
return result;
}
@Override
public String toString() {
return Strings.toString(this);
}
static class Builder {
private String name;
private String type;
@ -260,6 +263,10 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
this.isAggregatable &= agg;
}
List<String> getIndices() {
return indiceList.stream().map(c -> c.name).collect(Collectors.toList());
}
FieldCapabilities build(boolean withIndices) {
final String[] indices;
/* Eclipse can't deal with o -> o.name, maybe because of

3
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
View File

@ -29,8 +29,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import java.io.IOException;
public class FieldCapabilitiesIndexRequest
extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
public class FieldCapabilitiesIndexRequest extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
private String[] fields;
private OriginalIndices originalIndices;

8
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Writeable;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
/**
* Response for {@link FieldCapabilitiesIndexRequest} requests.
@ -89,14 +90,13 @@ public class FieldCapabilitiesIndexResponse extends ActionResponse implements Wr
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FieldCapabilitiesIndexResponse that = (FieldCapabilitiesIndexResponse) o;
return responseMap.equals(that.responseMap);
return Objects.equals(indexName, that.indexName) &&
Objects.equals(responseMap, that.responseMap);
}
@Override
public int hashCode() {
return responseMap.hashCode();
return Objects.hash(indexName, responseMap);
}
}

33
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.action.fieldcaps;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.IndicesRequest;
@ -44,6 +45,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
private String[] indices = Strings.EMPTY_ARRAY;
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private String[] fields = Strings.EMPTY_ARRAY;
private boolean includeUnmapped = false;
// pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
private boolean mergeResults = true;
@ -51,8 +53,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
new ObjectParser<>(NAME, FieldCapabilitiesRequest::new);
static {
PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields),
FIELDS_FIELD);
PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields), FIELDS_FIELD);
}
public FieldCapabilitiesRequest() {}
@ -83,6 +84,11 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
indices = in.readStringArray();
indicesOptions = IndicesOptions.readIndicesOptions(in);
mergeResults = in.readBoolean();
if (in.getVersion().onOrAfter(Version.V_7_1_0)) {
includeUnmapped = in.readBoolean();
} else {
includeUnmapped = false;
}
}
@Override
@ -92,6 +98,9 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
out.writeStringArray(indices);
indicesOptions.writeIndicesOptions(out);
out.writeBoolean(mergeResults);
if (out.getVersion().onOrAfter(Version.V_7_1_0)) {
out.writeBoolean(includeUnmapped);
}
}
/**
@ -123,6 +132,11 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
return this;
}
public FieldCapabilitiesRequest includeUnmapped(boolean includeUnmapped) {
this.includeUnmapped = includeUnmapped;
return this;
}
@Override
public String[] indices() {
return indices;
@ -133,12 +147,15 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
return indicesOptions;
}
public boolean includeUnmapped() {
return includeUnmapped;
}
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (fields == null || fields.length == 0) {
validationException =
ValidateActions.addValidationError("no fields specified", validationException);
validationException = ValidateActions.addValidationError("no fields specified", validationException);
}
return validationException;
}
@ -152,14 +169,12 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
return Arrays.equals(indices, that.indices) &&
Objects.equals(indicesOptions, that.indicesOptions) &&
Arrays.equals(fields, that.fields) &&
Objects.equals(mergeResults, that.mergeResults);
Objects.equals(mergeResults, that.mergeResults) &&
includeUnmapped == that.includeUnmapped;
}
@Override
public int hashCode() {
return Objects.hash(Arrays.hashCode(indices),
indicesOptions,
Arrays.hashCode(fields),
mergeResults);
return Objects.hash(Arrays.hashCode(indices), indicesOptions, Arrays.hashCode(fields), mergeResults, includeUnmapped);
}
}

5
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
View File

@ -36,4 +36,9 @@ public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder<FieldC
request().fields(fields);
return this;
}
public FieldCapabilitiesRequestBuilder setIncludeUnmapped(boolean includeUnmapped) {
request().includeUnmapped(includeUnmapped);
return this;
}
}

73
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
View File

@ -19,8 +19,10 @@
package org.elasticsearch.action.fieldcaps;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -31,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -42,32 +45,43 @@ import java.util.stream.Collectors;
* Response for {@link FieldCapabilitiesRequest} requests.
*/
public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentObject {
private static final ParseField INDICES_FIELD = new ParseField("indices");
private static final ParseField FIELDS_FIELD = new ParseField("fields");
private String[] indices;
private Map<String, Map<String, FieldCapabilities>> responseMap;
private List<FieldCapabilitiesIndexResponse> indexResponses;
FieldCapabilitiesResponse(Map<String, Map<String, FieldCapabilities>> responseMap) {
this(responseMap, Collections.emptyList());
FieldCapabilitiesResponse(String[] indices, Map<String, Map<String, FieldCapabilities>> responseMap) {
this(indices, responseMap, Collections.emptyList());
}
FieldCapabilitiesResponse(List<FieldCapabilitiesIndexResponse> indexResponses) {
this(Collections.emptyMap(), indexResponses);
this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses);
}
private FieldCapabilitiesResponse(Map<String, Map<String, FieldCapabilities>> responseMap,
private FieldCapabilitiesResponse(String[] indices, Map<String, Map<String, FieldCapabilities>> responseMap,
List<FieldCapabilitiesIndexResponse> indexResponses) {
this.responseMap = Objects.requireNonNull(responseMap);
this.indexResponses = Objects.requireNonNull(indexResponses);
this.indices = indices;
}
/**
* Used for serialization
*/
FieldCapabilitiesResponse() {
this(Collections.emptyMap(), Collections.emptyList());
this(Strings.EMPTY_ARRAY, Collections.emptyMap(), Collections.emptyList());
}
/**
* Get the concrete list of indices that were requested.
*/
public String[] getIndices() {
return indices;
}
/**
* Get the field capabilities map.
*/
@ -94,8 +108,12 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
this.responseMap =
in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField);
if (in.getVersion().onOrAfter(Version.V_7_1_0)) {
indices = in.readStringArray();
} else {
indices = Strings.EMPTY_ARRAY;
}
this.responseMap = in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField);
indexResponses = in.readList(FieldCapabilitiesIndexResponse::new);
}
@ -106,20 +124,27 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_7_1_0)) {
out.writeStringArray(indices);
}
out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField);
out.writeList(indexResponses);
}
private static void writeField(StreamOutput out,
Map<String, FieldCapabilities> map) throws IOException {
private static void writeField(StreamOutput out, Map<String, FieldCapabilities> map) throws IOException {
out.writeMap(map, StreamOutput::writeString, (valueOut, fc) -> fc.writeTo(valueOut));
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(FIELDS_FIELD.getPreferredName(), responseMap)
.endObject();
if (indexResponses.size() > 0) {
throw new IllegalStateException("cannot serialize non-merged response");
}
builder.startObject();
builder.field(INDICES_FIELD.getPreferredName(), indices);
builder.field(FIELDS_FIELD.getPreferredName(), responseMap);
builder.endObject();
return builder;
}
public static FieldCapabilitiesResponse fromXContent(XContentParser parser) throws IOException {
@ -129,11 +154,14 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<FieldCapabilitiesResponse, Void> PARSER =
new ConstructingObjectParser<>("field_capabilities_response", true,
a -> new FieldCapabilitiesResponse(
((List<Tuple<String, Map<String, FieldCapabilities>>>) a[0]).stream()
.collect(Collectors.toMap(Tuple::v1, Tuple::v2))));
a -> {
List<String> indices = a[0] == null ? Collections.emptyList() : (List<String>) a[0];
return new FieldCapabilitiesResponse(indices.stream().toArray(String[]::new),
((List<Tuple<String, Map<String, FieldCapabilities>>>) a[1]).stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2)));
});
static {
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD);
PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> {
Map<String, FieldCapabilities> typeToCapabilities = parseTypeToCapabilities(p, n);
return new Tuple<>(n, typeToCapabilities);
@ -158,14 +186,21 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o;
return responseMap.equals(that.responseMap);
return Arrays.equals(indices, that.indices) &&
Objects.equals(responseMap, that.responseMap) &&
Objects.equals(indexResponses, that.indexResponses);
}
@Override
public int hashCode() {
return responseMap.hashCode();
int result = Objects.hash(responseMap, indexResponses);
result = 31 * result + Arrays.hashCode(indices);
return result;
}
@Override
public String toString() {
return Strings.toString(this);
}
}

66
server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
View File

@ -37,10 +37,13 @@ import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class TransportFieldCapabilitiesAction extends HandledTransportAction<FieldCapabilitiesRequest, FieldCapabilitiesResponse> {
private final ThreadPool threadPool;
@ -75,20 +78,21 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
} else {
concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices);
}
final String[] allIndices = mergeIndiceNames(concreteIndices, remoteClusterIndices);
final int totalNumRequest = concreteIndices.length + remoteClusterIndices.size();
final CountDown completionCounter = new CountDown(totalNumRequest);
final List<FieldCapabilitiesIndexResponse> indexResponses = Collections.synchronizedList(new ArrayList<>());
final Runnable onResponse = () -> {
if (completionCounter.countDown()) {
if (request.isMergeResults()) {
listener.onResponse(merge(indexResponses));
listener.onResponse(merge(allIndices, indexResponses, request.includeUnmapped()));
} else {
listener.onResponse(new FieldCapabilitiesResponse(indexResponses));
}
}
};
if (totalNumRequest == 0) {
listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyMap()));
listener.onResponse(new FieldCapabilitiesResponse(allIndices, Collections.emptyMap()));
} else {
ActionListener<FieldCapabilitiesIndexResponse> innerListener = new ActionListener<FieldCapabilitiesIndexResponse>() {
@Override
@ -129,35 +133,61 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
}
}
private FieldCapabilitiesResponse merge(List<FieldCapabilitiesIndexResponse> indexResponses) {
Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder = new HashMap<> ();
private String[] mergeIndiceNames(String[] localIndices, Map<String, OriginalIndices> remoteIndices) {
Set<String> allIndices = new HashSet<>();
Arrays.stream(localIndices).forEach(allIndices::add);
for (Map.Entry<String, OriginalIndices> entry : remoteIndices.entrySet()) {
for (String index : entry.getValue().indices()) {
allIndices.add(RemoteClusterAware.buildRemoteIndexName(entry.getKey(), index));
}
}
return allIndices.stream().toArray(String[]::new);
}
private FieldCapabilitiesResponse merge(String[] indices, List<FieldCapabilitiesIndexResponse> indexResponses,
boolean includeUnmapped) {
final Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder = new HashMap<> ();
for (FieldCapabilitiesIndexResponse response : indexResponses) {
innerMerge(responseMapBuilder, response.getIndexName(), response.get());
}
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry :
responseMapBuilder.entrySet()) {
Map<String, FieldCapabilities> typeMap = new HashMap<>();
boolean multiTypes = entry.getValue().size() > 1;
for (Map.Entry<String, FieldCapabilities.Builder> fieldEntry :
entry.getValue().entrySet()) {
final Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : responseMapBuilder.entrySet()) {
final Map<String, FieldCapabilities.Builder> typeMapBuilder = entry.getValue();
if (includeUnmapped) {
addUnmappedFields(indices, entry.getKey(), typeMapBuilder);
}
boolean multiTypes = typeMapBuilder.size() > 1;
final Map<String, FieldCapabilities> typeMap = new HashMap<>();
for (Map.Entry<String, FieldCapabilities.Builder> fieldEntry : typeMapBuilder.entrySet()) {
typeMap.put(fieldEntry.getKey(), fieldEntry.getValue().build(multiTypes));
}
responseMap.put(entry.getKey(), typeMap);
responseMap.put(entry.getKey(), Collections.unmodifiableMap(typeMap));
}
return new FieldCapabilitiesResponse(responseMap);
return new FieldCapabilitiesResponse(indices, Collections.unmodifiableMap(responseMap));
}
private void innerMerge(Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder, String indexName,
Map<String, FieldCapabilities> map) {
private void addUnmappedFields(String[] indices, String field, Map<String, FieldCapabilities.Builder> typeMap) {
Set<String> unmappedIndices = new HashSet<>();
Arrays.stream(indices).forEach(unmappedIndices::add);
typeMap.values().stream().forEach((b) -> b.getIndices().stream().forEach(unmappedIndices::remove));
if (unmappedIndices.isEmpty() == false) {
FieldCapabilities.Builder unmapped = new FieldCapabilities.Builder(field, "unmapped");
typeMap.put("unmapped", unmapped);
for (String index : unmappedIndices) {
unmapped.add(index, false, false);
}
}
}
private void innerMerge(Map<String, Map<String, FieldCapabilities.Builder>> responseMapBuilder,
String indexName, Map<String, FieldCapabilities> map) {
for (Map.Entry<String, FieldCapabilities> entry : map.entrySet()) {
final String field = entry.getKey();
final FieldCapabilities fieldCap = entry.getValue();
Map<String, FieldCapabilities.Builder> typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>());
FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field,
key));
FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(),
key -> new FieldCapabilities.Builder(field, key));
builder.add(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable());
}
}

1
server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
View File

@ -57,6 +57,7 @@ public class RestFieldCapabilitiesAction extends BaseRestHandler {
fieldRequest.indicesOptions(
IndicesOptions.fromRequest(request, fieldRequest.indicesOptions()));
fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false));
return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel));
}
}

2
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java
View File

@ -50,6 +50,7 @@ public class FieldCapabilitiesRequestTests extends AbstractStreamableTestCase<Fi
if (randomBoolean()) {
request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
}
request.includeUnmapped(randomBoolean());
return request;
}
@ -75,6 +76,7 @@ public class FieldCapabilitiesRequestTests extends AbstractStreamableTestCase<Fi
request.indicesOptions(indicesOptions);
});
mutators.add(request -> request.setMergeResults(!request.isMergeResults()));
mutators.add(request -> request.includeUnmapped(!request.includeUnmapped()));
FieldCapabilitiesRequest mutatedInstance = copyInstance(instance);
Consumer<FieldCapabilitiesRequest> mutator = randomFrom(mutators);

123
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
View File

@ -19,32 +19,18 @@
package org.elasticsearch.action.fieldcaps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.AbstractStreamableXContentTestCase;
import org.elasticsearch.test.AbstractStreamableTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLength;
public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTestCase<FieldCapabilitiesResponse> {
@Override
protected FieldCapabilitiesResponse doParseInstance(XContentParser parser) throws IOException {
return FieldCapabilitiesResponse.fromXContent(parser);
}
public class FieldCapabilitiesResponseTests extends AbstractStreamableTestCase<FieldCapabilitiesResponse> {
@Override
protected FieldCapabilitiesResponse createBlankInstance() {
return new FieldCapabilitiesResponse();
@ -52,36 +38,15 @@ public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTe
@Override
protected FieldCapabilitiesResponse createTestInstance() {
if (randomBoolean()) {
// merged responses
Map<String, Map<String, FieldCapabilities>> responses = new HashMap<>();
List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
int numResponse = randomIntBetween(0, 10);
String[] fields = generateRandomStringArray(5, 10, false, true);
assertNotNull(fields);
for (String field : fields) {
Map<String, FieldCapabilities> typesToCapabilities = new HashMap<>();
String[] types = generateRandomStringArray(5, 10, false, false);
assertNotNull(types);
for (String type : types) {
typesToCapabilities.put(type, FieldCapabilitiesTests.randomFieldCaps(field));
}
responses.put(field, typesToCapabilities);
}
return new FieldCapabilitiesResponse(responses);
} else {
// non-merged responses
List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
int numResponse = randomIntBetween(0, 10);
for (int i = 0; i < numResponse; i++) {
responses.add(createRandomIndexResponse());
}
return new FieldCapabilitiesResponse(responses);
for (int i = 0; i < numResponse; i++) {
responses.add(createRandomIndexResponse());
}
return new FieldCapabilitiesResponse(responses);
}
private FieldCapabilitiesIndexResponse createRandomIndexResponse() {
Map<String, FieldCapabilities> responses = new HashMap<>();
@ -118,78 +83,6 @@ public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTe
FieldCapabilitiesTests.randomFieldCaps(toReplace)));
break;
}
return new FieldCapabilitiesResponse(mutatedResponses);
}
@Override
protected Predicate<String> getRandomFieldsExcludeFilter() {
// Disallow random fields from being inserted under the 'fields' key, as this
// map only contains field names, and also under 'fields.FIELD_NAME', as these
// maps only contain type names.
return field -> field.matches("fields(\\.\\w+)?");
}
public void testToXContent() throws IOException {
FieldCapabilitiesResponse response = createSimpleResponse();
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
String generatedResponse = BytesReference.bytes(builder).utf8ToString();
assertEquals((
"{" +
" \"fields\": {" +
" \"rating\": { " +
" \"keyword\": {" +
" \"type\": \"keyword\"," +
" \"searchable\": false," +
" \"aggregatable\": true," +
" \"indices\": [\"index3\", \"index4\"]," +
" \"non_searchable_indices\": [\"index4\"] " +
" }," +
" \"long\": {" +
" \"type\": \"long\"," +
" \"searchable\": true," +
" \"aggregatable\": false," +
" \"indices\": [\"index1\", \"index2\"]," +
" \"non_aggregatable_indices\": [\"index1\"] " +
" }" +
" }," +
" \"title\": { " +
" \"text\": {" +
" \"type\": \"text\"," +
" \"searchable\": true," +
" \"aggregatable\": false" +
" }" +
" }" +
" }" +
"}").replaceAll("\\s+", ""), generatedResponse);
}
public void testEmptyResponse() throws IOException {
FieldCapabilitiesResponse testInstance = new FieldCapabilitiesResponse();
assertSerialization(testInstance);
}
private static FieldCapabilitiesResponse createSimpleResponse() {
Map<String, FieldCapabilities> titleCapabilities = new HashMap<>();
titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false));
Map<String, FieldCapabilities> ratingCapabilities = new HashMap<>();
ratingCapabilities.put("long", new FieldCapabilities("rating", "long",
true, false,
new String[]{"index1", "index2"},
null,
new String[]{"index1"}));
ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword",
false, true,
new String[]{"index3", "index4"},
new String[]{"index4"},
null));
Map<String, Map<String, FieldCapabilities>> responses = new HashMap<>();
responses.put("title", titleCapabilities);
responses.put("rating", ratingCapabilities);
return new FieldCapabilitiesResponse(responses);
return new FieldCapabilitiesResponse(null, mutatedResponses);
}
}

4
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
View File

@ -86,8 +86,8 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
assertThat(cap2.isAggregatable(), equalTo(false));
assertThat(cap2.indices().length, equalTo(3));
assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"}));
assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"}));
assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"}));
assertThat(cap2.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"}));
assertThat(cap2.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"}));
}
}

173
server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java Normal file
View File

@ -0,0 +1,173 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.fieldcaps;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.AbstractStreamableXContentTestCase;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Predicate;
public class MergedFieldCapabilitiesResponseTests extends AbstractStreamableXContentTestCase<FieldCapabilitiesResponse> {
@Override
protected FieldCapabilitiesResponse doParseInstance(XContentParser parser) throws IOException {
return FieldCapabilitiesResponse.fromXContent(parser);
}
@Override
protected FieldCapabilitiesResponse createBlankInstance() {
return new FieldCapabilitiesResponse();
}
@Override
protected FieldCapabilitiesResponse createTestInstance() {
// merged responses
Map<String, Map<String, FieldCapabilities>> responses = new HashMap<>();
String[] fields = generateRandomStringArray(5, 10, false, true);
assertNotNull(fields);
for (String field : fields) {
Map<String, FieldCapabilities> typesToCapabilities = new HashMap<>();
String[] types = generateRandomStringArray(5, 10, false, false);
assertNotNull(types);
for (String type : types) {
typesToCapabilities.put(type, FieldCapabilitiesTests.randomFieldCaps(field));
}
responses.put(field, typesToCapabilities);
}
int numIndices = randomIntBetween(1, 10);
String[] indices = new String[numIndices];
for (int i = 0; i < numIndices; i++) {
indices[i] = randomAlphaOfLengthBetween(5, 10);
}
return new FieldCapabilitiesResponse(indices, responses);
}
@Override
protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse response) {
Map<String, Map<String, FieldCapabilities>> mutatedResponses = new HashMap<>(response.get());
int mutation = response.get().isEmpty() ? 0 : randomIntBetween(0, 2);
switch (mutation) {
case 0:
String toAdd = randomAlphaOfLength(10);
mutatedResponses.put(toAdd, Collections.singletonMap(
randomAlphaOfLength(10),
FieldCapabilitiesTests.randomFieldCaps(toAdd)));
break;
case 1:
String toRemove = randomFrom(mutatedResponses.keySet());
mutatedResponses.remove(toRemove);
break;
case 2:
String toReplace = randomFrom(mutatedResponses.keySet());
mutatedResponses.put(toReplace, Collections.singletonMap(
randomAlphaOfLength(10),
FieldCapabilitiesTests.randomFieldCaps(toReplace)));
break;
}
return new FieldCapabilitiesResponse(null, mutatedResponses);
}
@Override
protected Predicate<String> getRandomFieldsExcludeFilter() {
// Disallow random fields from being inserted under the 'fields' key, as this
// map only contains field names, and also under 'fields.FIELD_NAME', as these
// maps only contain type names.
return field -> field.matches("fields(\\.\\w+)?");
}
public void testToXContent() throws IOException {
FieldCapabilitiesResponse response = createSimpleResponse();
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
String generatedResponse = BytesReference.bytes(builder).utf8ToString();
assertEquals((
"{" +
" \"indices\": null," +
" \"fields\": {" +
" \"rating\": { " +
" \"keyword\": {" +
" \"type\": \"keyword\"," +
" \"searchable\": false," +
" \"aggregatable\": true," +
" \"indices\": [\"index3\", \"index4\"]," +
" \"non_searchable_indices\": [\"index4\"] " +
" }," +
" \"long\": {" +
" \"type\": \"long\"," +
" \"searchable\": true," +
" \"aggregatable\": false," +
" \"indices\": [\"index1\", \"index2\"]," +
" \"non_aggregatable_indices\": [\"index1\"] " +
" }" +
" }," +
" \"title\": { " +
" \"text\": {" +
" \"type\": \"text\"," +
" \"searchable\": true," +
" \"aggregatable\": false" +
" }" +
" }" +
" }" +
"}").replaceAll("\\s+", ""), generatedResponse);
}
public void testEmptyResponse() throws IOException {
FieldCapabilitiesResponse testInstance = new FieldCapabilitiesResponse();
assertSerialization(testInstance);
}
private static FieldCapabilitiesResponse createSimpleResponse() {
Map<String, FieldCapabilities> titleCapabilities = new HashMap<>();
titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false));
Map<String, FieldCapabilities> ratingCapabilities = new HashMap<>();
ratingCapabilities.put("long", new FieldCapabilities("rating", "long",
true, false,
new String[]{"index1", "index2"},
null,
new String[]{"index1"}));
ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword",
false, true,
new String[]{"index3", "index4"},
new String[]{"index4"},
null));
Map<String, Map<String, FieldCapabilities>> responses = new HashMap<>();
responses.put("title", titleCapabilities);
responses.put("rating", ratingCapabilities);
return new FieldCapabilitiesResponse(null, responses);
}
}

2
server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
View File

@ -113,7 +113,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
}
private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, Collection<String> expectedFields) {
Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>(fieldCapabilitiesResponse.get());
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
for (String field : builtInMetaDataFields) {
Map<String, FieldCapabilities> remove = responseMap.remove(field);

79
server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
@ -60,6 +61,13 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
.field("type", "alias")
.field("path", "playlist")
.endObject()
.startObject("old_field")
.field("type", "long")
.endObject()
.startObject("new_field")
.field("type", "alias")
.field("path", "old_field")
.endObject()
.endObject()
.endObject()
.endObject();
@ -75,10 +83,14 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
.startObject("route_length_miles")
.field("type", "double")
.endObject()
.startObject("new_field")
.field("type", "long")
.endObject()
.endObject()
.endObject()
.endObject();
assertAcked(prepareCreate("new_index").addMapping("_doc", newIndexMapping));
assertAcked(client().admin().indices().prepareAliases().addAlias("new_index", "current"));
}
public static class FieldFilterPlugin extends Plugin implements MapperPlugin {
@ -94,9 +106,9 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
}
public void testFieldAlias() {
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "route_length_miles")
.get();
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "route_length_miles").get();
assertIndices(response, "old_index", "new_index");
// Ensure the response has entries for both requested fields.
assertTrue(response.get().containsKey("distance"));
assertTrue(response.get().containsKey("route_length_miles"));
@ -126,26 +138,73 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
}
public void testFieldAliasWithWildcard() {
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("route*")
.get();
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("route*").get();
assertIndices(response, "old_index", "new_index");
assertEquals(1, response.get().size());
assertTrue(response.get().containsKey("route_length_miles"));
}
public void testFieldAliasFiltering() {
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields(
"secret-soundtrack", "route_length_miles")
.get();
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("secret-soundtrack", "route_length_miles").get();
assertIndices(response, "old_index", "new_index");
assertEquals(1, response.get().size());
assertTrue(response.get().containsKey("route_length_miles"));
}
public void testFieldAliasFilteringWithWildcard() {
FieldCapabilitiesResponse response = client().prepareFieldCaps()
.setFields("distance", "secret*")
.get();
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "secret*").get();
assertIndices(response, "old_index", "new_index");
assertEquals(1, response.get().size());
assertTrue(response.get().containsKey("distance"));
}
public void testWithUnmapped() {
FieldCapabilitiesResponse response = client().prepareFieldCaps()
.setFields("new_field", "old_field")
.setIncludeUnmapped(true)
.get();
assertIndices(response, "old_index", "new_index");
assertEquals(2, response.get().size());
assertTrue(response.get().containsKey("old_field"));
Map<String, FieldCapabilities> oldField = response.getField("old_field");
assertEquals(2, oldField.size());
assertTrue(oldField.containsKey("long"));
assertEquals(
new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null),
oldField.get("long"));
assertTrue(oldField.containsKey("unmapped"));
assertEquals(
new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null),
oldField.get("unmapped"));
Map<String, FieldCapabilities> newField = response.getField("new_field");
assertEquals(1, newField.size());
assertTrue(newField.containsKey("long"));
assertEquals(
new FieldCapabilities("new_field", "long", true, true),
newField.get("long"));
}
public void testWithIndexAlias() {
FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").get();
assertIndices(response, "new_index");
FieldCapabilitiesResponse response1 = client().prepareFieldCaps("current", "old_index").setFields("*").get();
assertIndices(response1, "old_index", "new_index");
FieldCapabilitiesResponse response2 = client().prepareFieldCaps("current", "old_index", "new_index").setFields("*").get();
assertEquals(response1, response2);
}
private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
assertNotNull(response.getIndices());
Arrays.sort(indices);
Arrays.sort(response.getIndices());
assertArrayEquals(indices, response.getIndices());
}
}

2
x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
View File

@ -410,7 +410,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
}
private static void assertExpectedFields(FieldCapabilitiesResponse fieldCapabilitiesResponse, String... expectedFields) {
Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>(fieldCapabilitiesResponse.get());
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
for (String field : builtInMetaDataFields) {
Map<String, FieldCapabilities> remove = responseMap.remove(field);