honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-27 10:28:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

EQL: Return sequence join keys in the original type (#61268) (#61282)

Browse Source 
(cherry picked from commit d54957d61faa0d502387656e3cace594017b6ea0)
This commit is contained in:
Andrei Stefan 2020-08-18 19:37:15 +03:00 committed by GitHub
parent 78d77ebed7
commit 5de0f19cc3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 148 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java
View File

@ -24,7 +24,9 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.InstantiatingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
@ -146,20 +148,21 @@ public class EqlSearchResponse {
new ConstructingObjectParser<>("eql/search_response_sequence", true,
args -> {
int i = 0;
@SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
@SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
@SuppressWarnings("unchecked") List<SearchHit> events = (List<SearchHit>) args[i];
return new EqlSearchResponse.Sequence(joinKeys, events);
});
static {
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOIN_KEYS);
PARSER.declareFieldArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
JOIN_KEYS, ObjectParser.ValueType.VALUE_ARRAY);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), EVENTS);
}
private final List<String> joinKeys;
private final List<Object> joinKeys;
private final List<SearchHit> events;
public Sequence(List<String> joinKeys, List<SearchHit> events) {
public Sequence(List<Object> joinKeys, List<SearchHit> events) {
this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys;
this.events = events == null ? Collections.emptyList() : events;
}
@ -186,7 +189,7 @@ public class EqlSearchResponse {
return Objects.hash(joinKeys, events);
}
public List<String> joinKeys() {
public List<Object> joinKeys() {
return joinKeys;
}
@ -204,7 +207,7 @@ public class EqlSearchResponse {
}
private final int count;
private final List<String> keys;
private final List<Object> keys;
private final float percent;
private static final ParseField COUNT = new ParseField(Fields.COUNT);
@ -216,18 +219,19 @@ public class EqlSearchResponse {
args -> {
int i = 0;
int count = (int) args[i++];
@SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
@SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
float percent = (float) args[i];
return new EqlSearchResponse.Count(count, joinKeys, percent);
});
static {
PARSER.declareInt(ConstructingObjectParser.constructorArg(), COUNT);
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), KEYS);
PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p), KEYS,
ObjectParser.ValueType.VALUE_ARRAY);
PARSER.declareFloat(ConstructingObjectParser.constructorArg(), PERCENT);
}
public Count(int count, List<String> keys, float percent) {
public Count(int count, List<Object> keys, float percent) {
this.count = count;
this.keys = keys == null ? Collections.emptyList() : keys;
this.percent = percent;
@ -260,7 +264,7 @@ public class EqlSearchResponse {
return count;
}
public List<String> keys() {
public List<Object> keys() {
return keys;
}

21
client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java
View File

@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.function.Supplier;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ -69,11 +70,12 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase<org.elastic
int size = randomIntBetween(1, 10);
List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence> seq = null;
if (randomBoolean()) {
List<Supplier<Object[]>> randoms = getKeysGenerators();
seq = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<String> joins = null;
List<Object> joins = null;
if (randomBoolean()) {
joins = Arrays.asList(generateRandomStringArray(6, 11, false));
joins = Arrays.asList(randomFrom(randoms).get());
}
seq.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(joins, randomEvents()));
}
@ -90,15 +92,26 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase<org.elastic
}
}
private static List<Supplier<Object[]>> getKeysGenerators() {
List<Supplier<Object[]>> randoms = new ArrayList<>();
randoms.add(() -> generateRandomStringArray(6, 11, false));
randoms.add(() -> randomArray(0, 6, Integer[]::new, ()-> randomInt()));
randoms.add(() -> randomArray(0, 6, Long[]::new, ()-> randomLong()));
randoms.add(() -> randomArray(0, 6, Boolean[]::new, ()-> randomBoolean()));
return randoms;
}
public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomCountResponse(TotalHits totalHits) {
int size = randomIntBetween(1, 10);
List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Count> cn = null;
if (randomBoolean()) {
List<Supplier<Object[]>> randoms = getKeysGenerators();
cn = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<String> keys = null;
List<Object> keys = null;
if (randomBoolean()) {
keys = Arrays.asList(generateRandomStringArray(6, 11, false));
keys = Arrays.asList(randomFrom(randoms).get());
}
cn.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Count(randomIntBetween(0, 41), keys, randomFloat()));
}

2
docs/reference/eql/detect-threats-with-eql.asciidoc
View File

@ -328,7 +328,7 @@ The query matches a sequence, indicating the attack likely succeeded.
"sequences": [
{
"join_keys": [
"2012"
2012
],
"events": [
{

4
docs/reference/eql/eql-search-api.asciidoc
View File

@ -405,7 +405,7 @@ a <<eql-sequences,sequence>>.
[%collapsible%open]
=====
`join_keys`::
(array of strings)
(array of values)
Shared field values used to constrain matches in the sequence. These are defined
using the <<eql-sequences,`by` keyword>> in the EQL query syntax.
@ -629,7 +629,7 @@ shared `process.pid` value for each matching event.
"sequences": [
{
"join_keys": [
"2012"
2012
],
"events": [
{

2
docs/reference/eql/eql.asciidoc
View File

@ -320,7 +320,7 @@ contains the shared `process.pid` value for each matching event.
"sequences": [
{
"join_keys": [
"2012"
2012
],
"events": [
{

75
x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml
View File

@ -11,6 +11,26 @@ setup:
- category: process
"@timestamp": 2020-02-03T12:34:56Z
user: SYSTEM
id: 123
valid: false
- index:
_index: eql_test
_id: 2
- event:
- category: process
"@timestamp": 2020-02-04T12:34:56Z
user: SYSTEM
id: 123
valid: true
- index:
_index: eql_test
_id: 3
- event:
- category: process
"@timestamp": 2020-02-05T12:34:56Z
user: SYSTEM
id: 123
valid: true
---
# Testing round-trip and the basic shape of the response
@ -22,9 +42,60 @@ setup:
query: "process where user = 'SYSTEM'"
- match: {timed_out: false}
- match: {hits.total.value: 1}
- match: {hits.total.value: 3}
- match: {hits.total.relation: "eq"}
- match: {hits.events.0._source.user: "SYSTEM"}
- match: {hits.events.0._id: "1"}
- match: {hits.events.1._id: "2"}
- match: {hits.events.2._id: "3"}
---
"Execute EQL sequence with string key.":
- do:
eql.search:
index: eql_test
body:
query: "sequence by user [process where user = 'SYSTEM'] [process where true]"
- match: {timed_out: false}
- match: {hits.total.value: 2}
- match: {hits.total.relation: "eq"}
- match: {hits.sequences.0.join_keys.0: "SYSTEM"}
- match: {hits.sequences.0.events.0._id: "1"}
- match: {hits.sequences.0.events.1._id: "2"}
- match: {hits.sequences.1.join_keys.0: "SYSTEM"}
- match: {hits.sequences.1.events.0._id: "2"}
- match: {hits.sequences.1.events.1._id: "3"}
---
"Execute EQL sequence with numeric key.":
- do:
eql.search:
index: eql_test
body:
query: "sequence by id [process where user = 'SYSTEM'] [process where true]"
- match: {timed_out: false}
- match: {hits.total.value: 2}
- match: {hits.total.relation: "eq"}
- match: {hits.sequences.0.join_keys.0: 123}
- match: {hits.sequences.0.events.0._id: "1"}
- match: {hits.sequences.0.events.1._id: "2"}
- match: {hits.sequences.1.join_keys.0: 123}
- match: {hits.sequences.1.events.0._id: "2"}
- match: {hits.sequences.1.events.1._id: "3"}
---
"Execute EQL sequence with boolean key.":
- do:
eql.search:
index: eql_test
body:
query: "sequence by valid [process where user = 'SYSTEM'] [process where true]"
- match: {timed_out: false}
- match: {hits.total.value: 1}
- match: {hits.total.relation: "eq"}
- match: {hits.sequences.0.join_keys.0: true}
- match: {hits.sequences.0.events.0._id: "2"}
- match: {hits.sequences.0.events.1._id: "3"}
---
"Execute some EQL in async mode":
@ -47,7 +118,7 @@ setup:
- match: {is_running: false}
- match: {is_partial: false}
- match: {timed_out: false}
- match: {hits.total.value: 1}
- match: {hits.total.value: 3}
- match: {hits.total.relation: "eq"}
- match: {hits.events.0._source.user: "SYSTEM"}

34
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
View File

@ -16,10 +16,12 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.InstantiatingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
@ -192,26 +194,28 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
new ConstructingObjectParser<>("eql/search_response_sequence", true,
args -> {
int i = 0;
@SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
@SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
@SuppressWarnings("unchecked") List<SearchHit> events = (List<SearchHit>) args[i];
return new EqlSearchResponse.Sequence(joinKeys, events);
});
static {
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOIN_KEYS);
PARSER.declareFieldArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
JOIN_KEYS, ObjectParser.ValueType.VALUE_ARRAY);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), EVENTS);
}
private final List<String> joinKeys;
private final List<Object> joinKeys;
private final List<SearchHit> events;
public Sequence(List<String> joinKeys, List<SearchHit> events) {
public Sequence(List<Object> joinKeys, List<SearchHit> events) {
this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys;
this.events = events == null ? Collections.emptyList() : events;
}
@SuppressWarnings("unchecked")
public Sequence(StreamInput in) throws IOException {
this.joinKeys = in.readStringList();
this.joinKeys = (List<Object>) in.readGenericValue();
this.events = in.readList(SearchHit::new);
}
@ -221,7 +225,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeStringCollection(joinKeys);
out.writeGenericValue(joinKeys);
out.writeList(events);
}
@ -260,7 +264,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
return Objects.hash(joinKeys, events);
}
public List<String> joinKeys() {
public List<Object> joinKeys() {
return joinKeys;
}
@ -278,7 +282,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
}
private final int count;
private final List<String> keys;
private final List<Object> keys;
private final float percent;
private static final ParseField COUNT = new ParseField(Fields.COUNT);
@ -290,26 +294,28 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
args -> {
int i = 0;
int count = (int) args[i++];
@SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
@SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
float percent = (float) args[i];
return new EqlSearchResponse.Count(count, joinKeys, percent);
});
static {
PARSER.declareInt(constructorArg(), COUNT);
PARSER.declareStringArray(constructorArg(), KEYS);
PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p), KEYS,
ObjectParser.ValueType.VALUE_ARRAY);
PARSER.declareFloat(constructorArg(), PERCENT);
}
public Count(int count, List<String> keys, float percent) {
public Count(int count, List<Object> keys, float percent) {
this.count = count;
this.keys = keys == null ? Collections.emptyList() : keys;
this.percent = percent;
}
@SuppressWarnings("unchecked")
public Count(StreamInput in) throws IOException {
count = in.readVInt();
keys = in.readStringList();
keys = (List<Object>) in.readGenericValue();
percent = in.readFloat();
}
@ -320,7 +326,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(count);
out.writeStringCollection(keys);
out.writeGenericValue(keys);
out.writeFloat(percent);
}
@ -357,7 +363,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
return count;
}
public List<String> keys() {
public List<Object> keys() {
return keys;
}

8
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceKey.java
View File

@ -24,12 +24,8 @@ public class SequenceKey {
this.hashCode = Objects.hash(keys);
}
public List<String> asStringList() {
String[] s = new String[keys.length];
for (int i = 0; i < keys.length; i++) {
s[i] = Objects.toString(keys[i]);
}
return Arrays.asList(s);
public List<Object> asList() {
return Arrays.asList(keys);
}
@Override

2
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java
View File

@ -25,7 +25,7 @@ class SequencePayload extends AbstractPayload {
for (int i = 0; i < sequences.size(); i++) {
Sequence s = sequences.get(i);
List<SearchHit> hits = searchHits.get(i);
values.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asStringList(), hits));
values.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asList(), hits));
}
}

21
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
View File

@ -15,6 +15,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.function.Supplier;
public class EqlSearchResponseTests extends AbstractSerializingTestCase<EqlSearchResponse> {
@ -68,11 +69,12 @@ public class EqlSearchResponseTests extends AbstractSerializingTestCase<EqlSearc
int size = randomIntBetween(1, 10);
List<EqlSearchResponse.Sequence> seq = null;
if (randomBoolean()) {
List<Supplier<Object[]>> randoms = getKeysGenerators();
seq = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<String> joins = null;
List<Object> joins = null;
if (randomBoolean()) {
joins = Arrays.asList(generateRandomStringArray(6, 11, false));
joins = Arrays.asList(randomFrom(randoms).get());
}
seq.add(new EqlSearchResponse.Sequence(joins, randomEvents()));
}
@ -89,15 +91,26 @@ public class EqlSearchResponseTests extends AbstractSerializingTestCase<EqlSearc
}
}
private static List<Supplier<Object[]>> getKeysGenerators() {
List<Supplier<Object[]>> randoms = new ArrayList<>();
randoms.add(() -> generateRandomStringArray(6, 11, false));
randoms.add(() -> randomArray(0, 6, Integer[]::new, ()-> randomInt()));
randoms.add(() -> randomArray(0, 6, Long[]::new, ()-> randomLong()));
randoms.add(() -> randomArray(0, 6, Boolean[]::new, ()-> randomBoolean()));
return randoms;
}
public static EqlSearchResponse createRandomCountResponse(TotalHits totalHits) {
int size = randomIntBetween(1, 10);
List<EqlSearchResponse.Count> cn = null;
if (randomBoolean()) {
List<Supplier<Object[]>> randoms = getKeysGenerators();
cn = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<String> keys = null;
List<Object> keys = null;
if (randomBoolean()) {
keys = Arrays.asList(generateRandomStringArray(6, 11, false));
keys = Arrays.asList(randomFrom(randoms).get());
}
cn.add(new EqlSearchResponse.Count(randomIntBetween(0, 41), keys, randomFloat()));
}