diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index b83b002fb38..c6346558d63 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.info.BuildParams /* * Licensed to Elasticsearch under one or more contributor @@ -67,6 +68,7 @@ dependencies { testCompile(project(':x-pack:plugin:core')) { exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client' } + testCompile(project(':x-pack:plugin:eql')) restSpec project(':rest-api-spec') } @@ -125,6 +127,10 @@ testClusters.all { setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' setting 'xpack.security.transport.ssl.enabled', 'false' + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.eql_feature_flag_registered', 'true' + } + setting 'xpack.eql.enabled', 'true' // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt' setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks' diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlClient.java new file mode 100644 index 00000000000..87158c23f86 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlClient.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.eql.EqlSearchRequest; +import org.elasticsearch.client.eql.EqlSearchResponse; + +import java.io.IOException; +import java.util.Collections; + +/** + * A wrapper for the {@link RestHighLevelClient} that provides methods for + * accessing the Elastic EQL related functions + *

+ * See the + * EQL APIs on elastic.co for more information. + */ +public final class EqlClient { + + private final RestHighLevelClient restHighLevelClient; + + EqlClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } + + /** + * Executes the eql search query. + *

+ * See + * the docs for more. + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public EqlSearchResponse search(EqlSearchRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + request, + EqlRequestConverters::search, + options, + EqlSearchResponse::fromXContent, + Collections.emptySet() + ); + } + + /** + * Asynchronously executes the eql search query. + *

+ * See + * the docs for more. + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable searchAsync(EqlSearchRequest request, + RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity( + request, + EqlRequestConverters::search, + options, + EqlSearchResponse::fromXContent, + listener, + Collections.emptySet() + ); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlRequestConverters.java new file mode 100644 index 00000000000..77d7ca9e56b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/EqlRequestConverters.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.client.eql.EqlSearchRequest; + +import java.io.IOException; + +import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE; +import static org.elasticsearch.client.RequestConverters.createEntity; + +final class EqlRequestConverters { + + static Request search(EqlSearchRequest eqlSearchRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder() + .addCommaSeparatedPathParts(eqlSearchRequest.indices()) + .addPathPartAsIs("_eql", "search") + .build(); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(); + parameters.withIndicesOptions(eqlSearchRequest.indicesOptions()); + request.setEntity(createEntity(eqlSearchRequest, REQUEST_BODY_CONTENT_TYPE)); + request.addParameters(parameters.asMap()); + return request; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 03aaacf8c76..b464c2166f8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -264,6 +264,7 @@ public class RestHighLevelClient implements Closeable { private final CcrClient ccrClient = new CcrClient(this); private final TransformClient transformClient = new TransformClient(this); private final EnrichClient enrichClient = new EnrichClient(this); + private final EqlClient eqlClient = new EqlClient(this); /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the @@ -492,6 +493,20 @@ public class RestHighLevelClient implements Closeable { return enrichClient; } + /** + * Provides methods for accessing the Elastic EQL APIs that + * are shipped with the Elastic Stack distribution of Elasticsearch. All of + * these APIs will 404 if run against the OSS distribution of Elasticsearch. + *

+ * See the + * EQL APIs on elastic.co for more information. + * + * @return the client wrapper for making Data Frame API calls + */ + public final EqlClient eql() { + return eqlClient; + } + /** * Executes a bulk request using the Bulk API. * See Bulk API on elastic.co diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java new file mode 100644 index 00000000000..a8b342f0a43 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -0,0 +1,216 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.eql; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.searchafter.SearchAfterBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +public class EqlSearchRequest implements Validatable, ToXContentObject { + + private String[] indices; + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, false); + + private QueryBuilder query = null; + private String timestampField = "@timestamp"; + private String eventTypeField = "event_type"; + private String implicitJoinKeyField = "agent.id"; + private int fetchSize = 50; + private SearchAfterBuilder searchAfterBuilder; + private String rule; + + static final String KEY_QUERY = "query"; + static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; + static final String KEY_EVENT_TYPE_FIELD = "event_type_field"; + static final String KEY_IMPLICIT_JOIN_KEY_FIELD = "implicit_join_key_field"; + static final String KEY_SIZE = "size"; + static final String KEY_SEARCH_AFTER = "search_after"; + static final String KEY_RULE = "rule"; + + public EqlSearchRequest(String indices, String rule) { + indices(indices); + rule(rule); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + if (query != null) { + builder.field(KEY_QUERY, query); + } + builder.field(KEY_TIMESTAMP_FIELD, timestampField()); + builder.field(KEY_EVENT_TYPE_FIELD, eventTypeField()); + if (implicitJoinKeyField != null) { + builder.field(KEY_IMPLICIT_JOIN_KEY_FIELD, implicitJoinKeyField()); + } + builder.field(KEY_SIZE, fetchSize()); + + if (searchAfterBuilder != null) { + builder.array(KEY_SEARCH_AFTER, searchAfterBuilder.getSortValues()); + } + + builder.field(KEY_RULE, rule); + builder.endObject(); + return builder; + } + + public EqlSearchRequest indices(String... indices) { + Objects.requireNonNull(indices, "indices must not be null"); + for (String index : indices) { + Objects.requireNonNull(index, "index must not be null"); + } + this.indices = indices; + return this; + } + + public QueryBuilder query() { + return this.query; + } + + public EqlSearchRequest query(QueryBuilder query) { + this.query = query; + return this; + } + + public String timestampField() { + return this.timestampField; + } + + public EqlSearchRequest timestampField(String timestampField) { + Objects.requireNonNull(timestampField, "timestamp field must not be null"); + this.timestampField = timestampField; + return this; + } + + public String eventTypeField() { + return this.eventTypeField; + } + + public EqlSearchRequest eventTypeField(String eventTypeField) { + Objects.requireNonNull(eventTypeField, "event type field must not be null"); + this.eventTypeField = eventTypeField; + return this; + } + + public String implicitJoinKeyField() { + return this.implicitJoinKeyField; + } + + public EqlSearchRequest implicitJoinKeyField(String implicitJoinKeyField) { + Objects.requireNonNull(implicitJoinKeyField, "implicit join key must not be null"); + this.implicitJoinKeyField = implicitJoinKeyField; + return this; + } + + public int fetchSize() { + return this.fetchSize; + } + + public EqlSearchRequest fetchSize(int size) { + this.fetchSize = size; + if (fetchSize <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + return this; + } + + public Object[] searchAfter() { + if (searchAfterBuilder == null) { + return null; + } + return searchAfterBuilder.getSortValues(); + } + + public EqlSearchRequest searchAfter(Object[] values) { + this.searchAfterBuilder = new SearchAfterBuilder().setSortValues(values); + return this; + } + + private EqlSearchRequest setSearchAfter(SearchAfterBuilder builder) { + this.searchAfterBuilder = builder; + return this; + } + + public String rule() { + return this.rule; + } + + public EqlSearchRequest rule(String rule) { + Objects.requireNonNull(rule, "rule must not be null"); + this.rule = rule; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + EqlSearchRequest that = (EqlSearchRequest) o; + return + fetchSize == that.fetchSize && + Arrays.equals(indices, that.indices) && + Objects.equals(indicesOptions, that.indicesOptions) && + Objects.equals(query, that.query) && + Objects.equals(timestampField, that.timestampField) && + Objects.equals(eventTypeField, that.eventTypeField) && + Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) && + Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && + Objects.equals(rule, that.rule); + } + + @Override + public int hashCode() { + return Objects.hash( + Arrays.hashCode(indices), + indicesOptions, + query, + fetchSize, + timestampField, + eventTypeField, + implicitJoinKeyField, + searchAfterBuilder, + rule); + } + + public String[] indices() { + return Arrays.copyOf(this.indices, this.indices.length); + } + + public EqlSearchRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = Objects.requireNonNull(indicesOptions, "indicesOptions must not be null"); + return this; + } + + public IndicesOptions indicesOptions() { + return indicesOptions; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java new file mode 100644 index 00000000000..76d22434273 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java @@ -0,0 +1,332 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.eql; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class EqlSearchResponse { + + private final Hits hits; + private final long tookInMillis; + private final boolean isTimeout; + + private static final class Fields { + static final String TOOK = "took"; + static final String TIMED_OUT = "timed_out"; + static final String HITS = "hits"; + } + + private static final ParseField TOOK = new ParseField(Fields.TOOK); + private static final ParseField TIMED_OUT = new ParseField(Fields.TIMED_OUT); + private static final ParseField HITS = new ParseField(Fields.HITS); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("eql/search_response", true, + args -> { + int i = 0; + Hits hits = (Hits) args[i++]; + Long took = (Long) args[i++]; + Boolean timeout = (Boolean) args[i]; + return new EqlSearchResponse(hits, took, timeout); + }); + + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> Hits.fromXContent(p), HITS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOOK); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), TIMED_OUT); + } + + public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) { + super(); + this.hits = hits == null ? Hits.EMPTY : hits; + this.tookInMillis = tookInMillis; + this.isTimeout = isTimeout; + } + + public static EqlSearchResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public long took() { + return tookInMillis; + } + + public boolean isTimeout() { + return isTimeout; + } + + public Hits hits() { + return hits; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + EqlSearchResponse that = (EqlSearchResponse) o; + return Objects.equals(hits, that.hits) + && Objects.equals(tookInMillis, that.tookInMillis) + && Objects.equals(isTimeout, that.isTimeout); + } + + @Override + public int hashCode() { + return Objects.hash(hits, tookInMillis, isTimeout); + } + + // Sequence + public static class Sequence { + private static final class Fields { + static final String JOIN_KEYS = "join_keys"; + static final String EVENTS = "events"; + } + + private static final ParseField JOIN_KEYS = new ParseField(Fields.JOIN_KEYS); + private static final ParseField EVENTS = new ParseField(Fields.EVENTS); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("eql/search_response_sequence", true, + args -> { + int i = 0; + @SuppressWarnings("unchecked") List joinKeys = (List) args[i++]; + @SuppressWarnings("unchecked") List events = (List) args[i]; + return new EqlSearchResponse.Sequence(joinKeys, events); + }); + + static { + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOIN_KEYS); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), EVENTS); + } + + private final List joinKeys; + private final List events; + + public Sequence(List joinKeys, List events) { + this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys; + this.events = events == null ? Collections.emptyList() : events; + } + + public static Sequence fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Sequence that = (Sequence) o; + return Objects.equals(joinKeys, that.joinKeys) + && Objects.equals(events, that.events); + } + + @Override + public int hashCode() { + return Objects.hash(joinKeys, events); + } + + public List joinKeys() { + return joinKeys; + } + + public List events() { + return events; + } + } + + // Count + public static class Count { + private static final class Fields { + static final String COUNT = "_count"; + static final String KEYS = "_keys"; + static final String PERCENT = "_percent"; + } + + private final int count; + private final List keys; + private final float percent; + + private static final ParseField COUNT = new ParseField(Fields.COUNT); + private static final ParseField KEYS = new ParseField(Fields.KEYS); + private static final ParseField PERCENT = new ParseField(Fields.PERCENT); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("eql/search_response_count", true, + args -> { + int i = 0; + int count = (int) args[i++]; + @SuppressWarnings("unchecked") List joinKeys = (List) args[i++]; + float percent = (float) args[i]; + return new EqlSearchResponse.Count(count, joinKeys, percent); + }); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), COUNT); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), KEYS); + PARSER.declareFloat(ConstructingObjectParser.constructorArg(), PERCENT); + } + + public Count(int count, List keys, float percent) { + this.count = count; + this.keys = keys == null ? Collections.emptyList() : keys; + this.percent = percent; + } + + public static Count fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Count that = (Count) o; + return Objects.equals(count, that.count) + && Objects.equals(keys, that.keys) + && Objects.equals(percent, that.percent); + } + + @Override + public int hashCode() { + return Objects.hash(count, keys, percent); + } + + public int count() { + return count; + } + + public List keys() { + return keys; + } + + public float percent() { + return percent; + } + } + + // Hits + public static class Hits { + public static final Hits EMPTY = new Hits(null, null, null, null); + + private final List events; + private final List sequences; + private final List counts; + private final TotalHits totalHits; + + private static final class Fields { + static final String TOTAL = "total"; + static final String EVENTS = "events"; + static final String SEQUENCES = "sequences"; + static final String COUNTS = "counts"; + } + + public Hits(@Nullable List events, @Nullable List sequences, @Nullable List counts, + @Nullable TotalHits totalHits) { + this.events = events; + this.sequences = sequences; + this.counts = counts; + this.totalHits = totalHits; + } + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("eql/search_response_count", true, + args -> { + int i = 0; + @SuppressWarnings("unchecked") List searchHits = (List) args[i++]; + @SuppressWarnings("unchecked") List sequences = (List) args[i++]; + @SuppressWarnings("unchecked") List counts = (List) args[i++]; + TotalHits totalHits = (TotalHits) args[i]; + return new EqlSearchResponse.Hits(searchHits, sequences, counts, totalHits); + }); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), + new ParseField(Fields.EVENTS)); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), Sequence.PARSER, + new ParseField(Fields.SEQUENCES)); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), Count.PARSER, + new ParseField(Fields.COUNTS)); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHits.parseTotalHitsFragment(p), + new ParseField(Fields.TOTAL)); + } + + public static Hits fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Hits that = (Hits) o; + return Objects.equals(events, that.events) + && Objects.equals(sequences, that.sequences) + && Objects.equals(counts, that.counts) + && Objects.equals(totalHits, that.totalHits); + } + + @Override + public int hashCode() { + return Objects.hash(events, sequences, counts, totalHits); + } + + public List events() { + return this.events; + } + + public List sequences() { + return this.sequences; + } + + public List counts() { + return this.counts; + } + + public TotalHits totalHits() { + return this.totalHits; + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java new file mode 100644 index 00000000000..c887e5459bc --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.client.eql.EqlSearchRequest; +import org.elasticsearch.client.eql.EqlSearchResponse; +import org.junit.Before; + +import static org.hamcrest.Matchers.equalTo; + +public class EqlIT extends ESRestHighLevelClientTestCase { + + @Before + public void setupRemoteClusterConfig() throws Exception { + setupRemoteClusterConfig("local_cluster"); + } + + public void testBasicSearch() throws Exception { + EqlClient eql = highLevelClient().eql(); + // TODO: Add real checks when end-to-end basic functionality is implemented + EqlSearchRequest request = new EqlSearchRequest("test", "test"); + EqlSearchResponse response = execute(request, eql::search, eql::searchAsync); + assertNotNull(response); + assertFalse(response.isTimeout()); + assertNotNull(response.hits()); + assertNull(response.hits().events()); + assertNull(response.hits().counts()); + assertNotNull(response.hits().sequences()); + assertThat(response.hits().sequences().size(), equalTo(2)); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index ea4a4790c26..ce806abe86b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -881,6 +881,7 @@ public class RestHighLevelClientTests extends ESTestCase { apiName.startsWith("ccr.") == false && apiName.startsWith("enrich.") == false && apiName.startsWith("transform.") == false && + apiName.startsWith("eql.") == false && apiName.endsWith("freeze") == false && apiName.endsWith("reload_analyzers") == false && // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java new file mode 100644 index 00000000000..a7ad218a39f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.eql; + +import org.elasticsearch.client.AbstractRequestTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchModule; + +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class EqlSearchRequestTests extends AbstractRequestTestCase { + + @Override + protected EqlSearchRequest createClientTestInstance() { + EqlSearchRequest EqlSearchRequest = new EqlSearchRequest("testindex", randomAlphaOfLength(40)); + if (randomBoolean()) { + EqlSearchRequest.fetchSize(randomIntBetween(1, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + EqlSearchRequest.implicitJoinKeyField(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + EqlSearchRequest.eventTypeField(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + EqlSearchRequest.rule(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + EqlSearchRequest.timestampField(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + EqlSearchRequest.searchAfter(randomArray(1, 4, Object[]::new, () -> randomAlphaOfLength(3))); + } + if (randomBoolean()) { + if (randomBoolean()) { + EqlSearchRequest.query(QueryBuilders.matchAllQuery()); + } else { + EqlSearchRequest.query(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100))); + } + } + return EqlSearchRequest; + } + + @Override + protected org.elasticsearch.xpack.eql.action.EqlSearchRequest doParseToServerInstance(XContentParser parser) throws IOException { + return org.elasticsearch.xpack.eql.action.EqlSearchRequest.fromXContent(parser).indices("testindex"); + } + + @Override + protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchRequest serverInstance, EqlSearchRequest + clientTestInstance) { + assertThat(serverInstance.eventTypeField(), equalTo(clientTestInstance.eventTypeField())); + assertThat(serverInstance.implicitJoinKeyField(), equalTo(clientTestInstance.implicitJoinKeyField())); + assertThat(serverInstance.timestampField(), equalTo(clientTestInstance.timestampField())); + assertThat(serverInstance.query(), equalTo(clientTestInstance.query())); + assertThat(serverInstance.rule(), equalTo(clientTestInstance.rule())); + assertThat(serverInstance.searchAfter(), equalTo(clientTestInstance.searchAfter())); + assertThat(serverInstance.indicesOptions(), equalTo(clientTestInstance.indicesOptions())); + assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices())); + assertThat(serverInstance.fetchSize(), equalTo(clientTestInstance.fetchSize())); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedXContents()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java new file mode 100644 index 00000000000..f25da7d4f91 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java @@ -0,0 +1,166 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.eql; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class EqlSearchResponseTests extends AbstractResponseTestCase { + + static List randomEvents() { + int size = randomIntBetween(1, 10); + List hits = null; + if (randomBoolean()) { + hits = new ArrayList<>(); + for (int i = 0; i < size; i++) { + hits.add(new SearchHit(i, randomAlphaOfLength(10), null, new HashMap<>())); + } + } + if (randomBoolean()) { + return null; + } + return hits; + } + + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomEventsResponse(TotalHits totalHits) { + org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null; + if (randomBoolean()) { + hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(randomEvents(), null, null, totalHits); + } + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } + + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomSequencesResponse(TotalHits totalHits) { + int size = randomIntBetween(1, 10); + List seq = null; + if (randomBoolean()) { + seq = new ArrayList<>(); + for (int i = 0; i < size; i++) { + List joins = null; + if (randomBoolean()) { + joins = Arrays.asList(generateRandomStringArray(6, 11, false)); + } + seq.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(joins, randomEvents())); + } + } + org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null; + if (randomBoolean()) { + hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, seq, null, totalHits); + } + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } + + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomCountResponse(TotalHits totalHits) { + int size = randomIntBetween(1, 10); + List cn = null; + if (randomBoolean()) { + cn = new ArrayList<>(); + for (int i = 0; i < size; i++) { + List keys = null; + if (randomBoolean()) { + keys = Arrays.asList(generateRandomStringArray(6, 11, false)); + } + cn.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Count(randomIntBetween(0, 41), keys, randomFloat())); + } + } + org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null; + if (randomBoolean()) { + hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, null, cn, totalHits); + } + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } + + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomInstance(TotalHits totalHits) { + int type = between(0, 2); + switch (type) { + case 0: + return createRandomEventsResponse(totalHits); + case 1: + return createRandomSequencesResponse(totalHits); + case 2: + return createRandomCountResponse(totalHits); + default: + return null; + } + } + + @Override + protected org.elasticsearch.xpack.eql.action.EqlSearchResponse createServerTestInstance(XContentType xContentType) { + TotalHits totalHits = null; + if (randomBoolean()) { + totalHits = new TotalHits(randomIntBetween(100, 1000), TotalHits.Relation.EQUAL_TO); + } + return createRandomInstance(totalHits); + } + + @Override + protected EqlSearchResponse doParseToClientInstance(XContentParser parser) throws IOException { + return EqlSearchResponse.fromXContent(parser); + } + + @Override + protected void assertInstances( + org.elasticsearch.xpack.eql.action.EqlSearchResponse serverTestInstance, EqlSearchResponse clientInstance) { + assertThat(serverTestInstance.took(), is(clientInstance.took())); + assertThat(serverTestInstance.isTimeout(), is(clientInstance.isTimeout())); + assertThat(serverTestInstance.hits().totalHits(), is(clientInstance.hits().totalHits())); + if (serverTestInstance.hits().counts() == null) { + assertNull(clientInstance.hits().counts()); + } else { + assertThat(serverTestInstance.hits().counts().size(), equalTo(clientInstance.hits().counts().size())); + for (int i = 0; i < serverTestInstance.hits().counts().size(); i++) { + assertThat(serverTestInstance.hits().counts().get(i).count(), is(clientInstance.hits().counts().get(i).count())); + assertThat(serverTestInstance.hits().counts().get(i).keys(), is(clientInstance.hits().counts().get(i).keys())); + assertThat(serverTestInstance.hits().counts().get(i).percent(), is(clientInstance.hits().counts().get(i).percent())); + } + } + if (serverTestInstance.hits().events() == null) { + assertNull(clientInstance.hits().events()); + } else { + assertThat(serverTestInstance.hits().events().size(), equalTo(clientInstance.hits().events().size())); + for (int i = 0; i < serverTestInstance.hits().events().size(); i++) { + assertThat(serverTestInstance.hits().events().get(i), is(clientInstance.hits().events().get(i))); + } + } + if (serverTestInstance.hits().sequences() == null) { + assertNull(clientInstance.hits().sequences()); + } else { + assertThat(serverTestInstance.hits().sequences().size(), equalTo(clientInstance.hits().sequences().size())); + for (int i = 0; i < serverTestInstance.hits().sequences().size(); i++) { + assertThat(serverTestInstance.hits().sequences().get(i).joinKeys(), + is(clientInstance.hits().sequences().get(i).joinKeys())); + assertThat(serverTestInstance.hits().sequences().get(i).events(), is(clientInstance.hits().sequences().get(i).events())); + } + } + } +} diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java index 4c404fb5a0b..628972c4d20 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java @@ -51,9 +51,9 @@ public abstract class CommonEqlRestTestCase extends ESRestTestCase { searchValidationTests.add(new SearchTestConfiguration("{\"rule\": \"" + validRule + "\", \"implicit_join_key_field\": \"\"}", 400, "implicit join key field is null or empty")); searchValidationTests.add(new SearchTestConfiguration("{\"rule\": \"" + validRule + "\", \"size\": 0}", - 400, "size must be more than 0")); + 400, "size must be greater than 0")); searchValidationTests.add(new SearchTestConfiguration("{\"rule\": \"" + validRule + "\", \"size\": -1}", - 400, "size must be more than 0")); + 400, "size must be greater than 0")); searchValidationTests.add(new SearchTestConfiguration("{\"rule\": \"" + validRule + "\", \"search_after\": null}", 400, "search_after doesn't support values of type: VALUE_NULL")); searchValidationTests.add(new SearchTestConfiguration("{\"rule\": \"" + validRule + "\", \"search_after\": []}", diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index b78a398437f..2d5aa5f8c3b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -95,6 +95,10 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re } } + if (indicesOptions == null) { + validationException = addValidationError("indicesOptions is null", validationException); + } + if (rule == null || rule.isEmpty()) { validationException = addValidationError("rule is null or empty", validationException); } @@ -112,7 +116,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re } if (fetchSize <= 0) { - validationException = addValidationError("size must be more than 0", validationException); + validationException = addValidationError("size must be greater than 0", validationException); } return validationException; @@ -276,6 +280,11 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re return indices; } + public EqlSearchRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + @Override public IndicesOptions indicesOptions() { return indicesOptions; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index f06d01f6a3c..0ffcab4ca93 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -260,6 +260,14 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec public int hashCode() { return Objects.hash(joinKeys, events); } + + public List joinKeys() { + return joinKeys; + } + + public List events() { + return events; + } } // Count @@ -345,6 +353,18 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec public int hashCode() { return Objects.hash(count, keys, percent); } + + public int count() { + return count; + } + + public List keys() { + return keys; + } + + public float percent() { + return percent; + } } // Hits @@ -483,5 +503,21 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec public int hashCode() { return Objects.hash(events, sequences, counts, totalHits); } + + public List events() { + return this.events; + } + + public List sequences() { + return this.sequences; + } + + public List counts() { + return this.counts; + } + + public TotalHits totalHits() { + return this.totalHits; + } } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java index 9da71b13a5d..9f615d34f19 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.eql.plugin; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -46,6 +47,7 @@ public class RestEqlSearchAction extends BaseRestHandler { try (XContentParser parser = request.contentOrSourceParamParser()) { eqlRequest = EqlSearchRequest.fromXContent(parser); eqlRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); + eqlRequest.indicesOptions(IndicesOptions.fromRequest(request, eqlRequest.indicesOptions())); } return channel -> client.execute(EqlSearchAction.INSTANCE, eqlRequest, new RestResponseListener(channel) {