honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: Sequence/Join parsing and model (#54227) 

Add parsing and (logical) domain model for sequence and join

(cherry picked from commit 9e9632d41a39877256c68634ab18e441f4b67fe8)
This commit is contained in:
Costin Leau 2020-04-06 23:15:05 +03:00 committed by Costin Leau
parent e9c3bfc8e5
commit 36121117f0
14 changed files with 907 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
x-pack/plugin/eql/src/main/antlr/EqlBase.g4
View File

@ -32,13 +32,13 @@ sequenceParams
sequence
: SEQUENCE (by=joinKeys sequenceParams? | sequenceParams by=joinKeys?)?
sequenceTerm sequenceTerm+
(UNTIL sequenceTerm)?
(UNTIL until=sequenceTerm)?
;
join
: JOIN (by=joinKeys)?
joinTerm joinTerm+
(UNTIL joinTerm)?
(UNTIL until=joinTerm)?
;
pipe

77
x-pack/plugin/eql/src/main/antlr/EqlBase.tokens Normal file
View File

@ -0,0 +1,77 @@
AND=1
ANY=2
BY=3
FALSE=4
FORK=5
IN=6
JOIN=7
MAXSPAN=8
NOT=9
NULL=10
OF=11
OR=12
SEQUENCE=13
TRUE=14
UNTIL=15
WHERE=16
WITH=17
EQ=18
NEQ=19
LT=20
LTE=21
GT=22
GTE=23
PLUS=24
MINUS=25
ASTERISK=26
SLASH=27
PERCENT=28
DOT=29
COMMA=30
LB=31
RB=32
LP=33
RP=34
PIPE=35
ESCAPED_IDENTIFIER=36
STRING=37
INTEGER_VALUE=38
DECIMAL_VALUE=39
IDENTIFIER=40
LINE_COMMENT=41
BRACKETED_COMMENT=42
WS=43
'and'=1
'any'=2
'by'=3
'false'=4
'fork'=5
'in'=6
'join'=7
'maxspan'=8
'not'=9
'null'=10
'of'=11
'or'=12
'sequence'=13
'true'=14
'until'=15
'where'=16
'with'=17
'!='=19
'<'=20
'<='=21
'>'=22
'>='=23
'+'=24
'-'=25
'*'=26
'/'=27
'%'=28
'.'=29
','=30
'['=31
']'=32
'('=33
')'=34
'|'=35

77
x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens Normal file
View File

@ -0,0 +1,77 @@
AND=1
ANY=2
BY=3
FALSE=4
FORK=5
IN=6
JOIN=7
MAXSPAN=8
NOT=9
NULL=10
OF=11
OR=12
SEQUENCE=13
TRUE=14
UNTIL=15
WHERE=16
WITH=17
EQ=18
NEQ=19
LT=20
LTE=21
GT=22
GTE=23
PLUS=24
MINUS=25
ASTERISK=26
SLASH=27
PERCENT=28
DOT=29
COMMA=30
LB=31
RB=32
LP=33
RP=34
PIPE=35
ESCAPED_IDENTIFIER=36
STRING=37
INTEGER_VALUE=38
DECIMAL_VALUE=39
IDENTIFIER=40
LINE_COMMENT=41
BRACKETED_COMMENT=42
WS=43
'and'=1
'any'=2
'by'=3
'false'=4
'fork'=5
'in'=6
'join'=7
'maxspan'=8
'not'=9
'null'=10
'of'=11
'or'=12
'sequence'=13
'true'=14
'until'=15
'where'=16
'with'=17
'!='=19
'<'=20
'<='=21
'>'=22
'>='=23
'+'=24
'-'=25
'*'=26
'/'=27
'%'=28
'.'=29
','=30
'['=31
']'=32
'('=33
')'=34
'|'=35

6
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java
View File

@ -393,6 +393,7 @@ class EqlBaseParser extends Parser {
public static class SequenceContext extends ParserRuleContext {
public JoinKeysContext by;
public SequenceTermContext until;
public TerminalNode SEQUENCE() { return getToken(EqlBaseParser.SEQUENCE, 0); }
public List<SequenceTermContext> sequenceTerm() {
return getRuleContexts(SequenceTermContext.class);
@ -495,7 +496,7 @@ class EqlBaseParser extends Parser {
setState(94);
match(UNTIL);
setState(95);
sequenceTerm();
((SequenceContext)_localctx).until = sequenceTerm();
}
}
@ -514,6 +515,7 @@ class EqlBaseParser extends Parser {
public static class JoinContext extends ParserRuleContext {
public JoinKeysContext by;
public JoinTermContext until;
public TerminalNode JOIN() { return getToken(EqlBaseParser.JOIN, 0); }
public List<JoinTermContext> joinTerm() {
return getRuleContexts(JoinTermContext.class);
@ -585,7 +587,7 @@ class EqlBaseParser extends Parser {
setState(108);
match(UNTIL);
setState(109);
joinTerm();
((JoinContext)_localctx).until = joinTerm();
}
}

20
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java
View File

@ -133,16 +133,6 @@ public class EqlParser {
this.ruleNames = ruleNames;
}
@Override
public void exitJoin(EqlBaseParser.JoinContext context) {
Token token = context.JOIN().getSymbol();
throw new ParsingException(
"Join is not supported",
null,
token.getLine(),
token.getCharPositionInLine());
}
@Override
public void exitPipe(EqlBaseParser.PipeContext context) {
Token token = context.PIPE().getSymbol();
@ -163,16 +153,6 @@ public class EqlParser {
token.getCharPositionInLine());
}
@Override
public void exitSequence(EqlBaseParser.SequenceContext context) {
Token token = context.SEQUENCE().getSymbol();
throw new ParsingException(
"Sequence is not supported",
null,
token.getLine(),
token.getCharPositionInLine());
}
@Override
public void exitQualifiedName(EqlBaseParser.QualifiedNameContext context) {
if (context.INTEGER_VALUE().size() > 0) {

8
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ArithmeticUnaryContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ComparisonContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.DereferenceContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.FunctionExpressionContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinKeysContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.LogicalBinaryContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.LogicalNotContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.PredicateContext;
@ -46,6 +47,8 @@ import org.elasticsearch.xpack.ql.util.StringUtils;
import java.util.List;
import static java.util.Collections.emptyList;
public class ExpressionBuilder extends IdentifierBuilder {
@ -62,6 +65,11 @@ public class ExpressionBuilder extends IdentifierBuilder {
return expression(ctx.expression());
}
@Override
public List<Expression> visitJoinKeys(JoinKeysContext ctx) {
return ctx != null ? expressions(ctx.expression()) : emptyList();
}
@Override
public Expression visitArithmeticUnary(ArithmeticUnaryContext ctx) {
Expression expr = expression(ctx.valueExpression());

169
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java
View File

@ -5,6 +5,17 @@
*/
package org.elasticsearch.xpack.eql.parser;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.IntegerLiteralContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinTermContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.NumberContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceParamsContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceTermContext;
import org.elasticsearch.xpack.eql.plan.logical.Join;
import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
import org.elasticsearch.xpack.eql.plan.logical.Sequence;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Literal;
import org.elasticsearch.xpack.ql.expression.Order;
@ -17,12 +28,18 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static java.util.Collections.singletonList;
public abstract class LogicalPlanBuilder extends ExpressionBuilder {
private final ParserParams params;
private final UnresolvedRelation RELATION = new UnresolvedRelation(Source.EMPTY, null, "", false, "");
public LogicalPlanBuilder(ParserParams params) {
this.params = params;
@ -44,11 +61,157 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder {
condition = new And(source, eventMatch, condition);
}
Filter filter = new Filter(source, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
// add implicit sorting - when pipes are added, this would better seat there (as a default pipe)
Filter filter = new Filter(source, RELATION, condition);
// add implicit sorting - when pipes are added, this would better sit there (as a default pipe)
Order order = new Order(source, new UnresolvedAttribute(source, params.fieldTimestamp()), Order.OrderDirection.ASC,
Order.NullsPosition.FIRST);
OrderBy orderBy = new OrderBy(source, filter, singletonList(order));
return orderBy;
}
}
@Override
public Join visitJoin(JoinContext ctx) {
List<Expression> parentJoinKeys = visitJoinKeys(ctx.by);
LogicalPlan until;
if (ctx.until != null) {
until = visitJoinTerm(ctx.until, parentJoinKeys);
} else {
// no until declared means the condition never gets executed and thus folds to false
until = new Filter(source(ctx), RELATION, new Literal(source(ctx), Boolean.FALSE, DataTypes.BOOLEAN));
}
int numberOfKeys = -1;
List<LogicalPlan> queries = new ArrayList<>(ctx.joinTerm().size());
for (JoinTermContext joinTermCtx : ctx.joinTerm()) {
KeyedFilter joinTerm = visitJoinTerm(joinTermCtx, parentJoinKeys);
int keySize = joinTerm.keys().size();
if (numberOfKeys < 0) {
numberOfKeys = keySize;
} else {
if (numberOfKeys != keySize) {
Source src = source(joinTermCtx.by != null ? joinTermCtx.by : joinTermCtx);
int expected = numberOfKeys - parentJoinKeys.size();
int found = keySize - parentJoinKeys.size();
throw new ParsingException(src, "Inconsistent number of join keys specified; expected [{}] but found [{}]", expected,
found);
}
queries.add(joinTerm);
}
}
return new Join(source(ctx), queries, until);
}
public KeyedFilter visitJoinTerm(JoinTermContext ctx, List<Expression> joinKeys) {
List<Expression> keys = CollectionUtils.combine(joinKeys, visitJoinKeys(ctx.by));
return new KeyedFilter(source(ctx), visitEventQuery(ctx.subquery().eventQuery()), keys);
}
@Override
public Sequence visitSequence(SequenceContext ctx) {
List<Expression> parentJoinKeys = visitJoinKeys(ctx.by);
TimeValue maxSpan = visitSequenceParams(ctx.sequenceParams());
LogicalPlan until;
if (ctx.until != null) {
until = visitSequenceTerm(ctx.until, parentJoinKeys);
} else {
// no until declared means the condition never gets executed and thus folds to false
until = new Filter(source(ctx), RELATION, new Literal(source(ctx), Boolean.FALSE, DataTypes.BOOLEAN));
}
int numberOfKeys = -1;
List<LogicalPlan> queries = new ArrayList<>(ctx.sequenceTerm().size());
for (SequenceTermContext sequenceTermCtx : ctx.sequenceTerm()) {
KeyedFilter sequenceTerm = visitSequenceTerm(sequenceTermCtx, parentJoinKeys);
int keySize = sequenceTerm.keys().size();
if (numberOfKeys < 0) {
numberOfKeys = keySize;
} else {
if (numberOfKeys != keySize) {
Source src = source(sequenceTermCtx.by != null ? sequenceTermCtx.by : sequenceTermCtx);
int expected = numberOfKeys - parentJoinKeys.size();
int found = keySize - parentJoinKeys.size();
throw new ParsingException(src, "Inconsistent number of join keys specified; expected [{}] but found [{}]", expected,
found);
}
queries.add(sequenceTerm);
}
}
return new Sequence(source(ctx), queries, until, maxSpan);
}
public KeyedFilter visitSequenceTerm(SequenceTermContext ctx, List<Expression> joinKeys) {
if (ctx.FORK() != null) {
throw new ParsingException(source(ctx.FORK()), "sequence fork is unsupported");
}
List<Expression> keys = CollectionUtils.combine(joinKeys, visitJoinKeys(ctx.by));
return new KeyedFilter(source(ctx), visitEventQuery(ctx.subquery().eventQuery()), keys);
}
@Override
public TimeValue visitSequenceParams(SequenceParamsContext ctx) {
if (ctx == null) {
return TimeValue.MINUS_ONE;
}
NumberContext numberCtx = ctx.timeUnit().number();
if (numberCtx instanceof IntegerLiteralContext) {
Number number = (Number) visitIntegerLiteral((IntegerLiteralContext) numberCtx).fold();
long value = number.longValue();
if (value <= 0) {
throw new ParsingException(source(numberCtx), "A positive maxspan value is required; found [{}]", value);
}
String timeString = text(ctx.timeUnit().IDENTIFIER());
TimeUnit timeUnit = TimeUnit.SECONDS;
if (timeString != null) {
switch (timeString) {
case "":
case "s":
case "sec":
case "secs":
case "second":
case "seconds":
timeUnit = TimeUnit.SECONDS;
break;
case "m":
case "min":
case "mins":
case "minute":
case "minutes":
timeUnit = TimeUnit.MINUTES;
break;
case "h":
case "hs":
case "hour":
case "hours":
timeUnit = TimeUnit.HOURS;
break;
case "d":
case "ds":
case "day":
case "days":
timeUnit = TimeUnit.DAYS;
break;
default:
throw new ParsingException(source(ctx.timeUnit().IDENTIFIER()), "Unrecognized time unit [{}]", timeString);
}
}
return new TimeValue(value, timeUnit);
} else {
throw new ParsingException(source(numberCtx), "Decimal time interval [{}] not supported yet", text(numberCtx));
}
}
}

97
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.plan.logical;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Attribute;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import static java.util.Collections.emptyList;
public class Join extends LogicalPlan {
private final List<LogicalPlan> queries;
private final LogicalPlan until;
public Join(Source source, List<LogicalPlan> queries, LogicalPlan until) {
super(source, CollectionUtils.combine(queries, until));
this.queries = queries;
this.until = until;
}
@Override
protected NodeInfo<Join> info() {
return NodeInfo.create(this, Join::new, queries, until);
}
@Override
public Join replaceChildren(List<LogicalPlan> newChildren) {
if (newChildren.size() < 2) {
throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size());
}
int lastIndex = newChildren.size() - 1;
return new Join(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex));
}
@Override
public List<Attribute> output() {
List<Attribute> out = new ArrayList<>();
for (LogicalPlan plan : queries) {
out.addAll(plan.output());
}
if (until != null) {
out.addAll(until.output());
}
return out;
}
@Override
public boolean expressionsResolved() {
return true;
}
public List<LogicalPlan> queries() {
return queries;
}
public LogicalPlan until() {
return until;
}
@Override
public int hashCode() {
return Objects.hash(queries, until);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
Join other = (Join) obj;
return Objects.equals(queries, other.queries)
&& Objects.equals(until, other.until);
}
@Override
public List<Object> nodeProperties() {
return emptyList();
}
}

70
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/KeyedFilter.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.plan.logical;
import org.elasticsearch.xpack.ql.capabilities.Resolvables;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import java.util.List;
import java.util.Objects;
/**
* Filter that has one or multiple associated keys associated with.
* Used inside Join or Sequence.
*/
public class KeyedFilter extends UnaryPlan {
private final List<Expression> keys;
public KeyedFilter(Source source, LogicalPlan child, List<Expression> keys) {
super(source, child);
this.keys = keys;
}
@Override
protected NodeInfo<KeyedFilter> info() {
return NodeInfo.create(this, KeyedFilter::new, child(), keys);
}
@Override
protected KeyedFilter replaceChild(LogicalPlan newChild) {
return new KeyedFilter(source(), newChild, keys);
}
public List<Expression> keys() {
return keys;
}
@Override
public boolean expressionsResolved() {
return Resolvables.resolved(keys);
}
@Override
public int hashCode() {
return Objects.hash(keys, child());
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
KeyedFilter other = (KeyedFilter) obj;
return Objects.equals(keys, other.keys)
&& Objects.equals(child(), other.child());
}
}

72
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.plan.logical;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import java.util.List;
import java.util.Objects;
import static java.util.Collections.singletonList;
public class Sequence extends Join {
private final TimeValue maxSpan;
public Sequence(Source source, List<LogicalPlan> queries, LogicalPlan until, TimeValue maxSpan) {
super(source, queries, until);
this.maxSpan = maxSpan;
}
@Override
protected NodeInfo<Join> info() {
return NodeInfo.create(this, Sequence::new, queries(), until(), maxSpan);
}
@Override
public Join replaceChildren(List<LogicalPlan> newChildren) {
if (newChildren.size() < 2) {
throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size());
}
int lastIndex = newChildren.size() - 1;
return new Sequence(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), maxSpan);
}
public TimeValue maxSpan() {
return maxSpan;
}
@Override
public int hashCode() {
return Objects.hash(maxSpan, queries(), until());
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
Sequence other = (Sequence) obj;
return Objects.equals(maxSpan, other.maxSpan)
&& Objects.equals(queries(), other.queries())
&& Objects.equals(until(), other.until());
}
@Override
public List<Object> nodeProperties() {
return singletonList(maxSpan);
}
}

12
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
View File

@ -99,18 +99,6 @@ public class VerifierTests extends ESTestCase {
" and child of [file where file_name=\"svchost.exe\" and opcode=0]"));
}
public void testSequencesUnsupported() {
assertEquals("1:1: Sequence is not supported", errorParsing("sequence\n" +
" [process where serial_event_id = 1]\n" +
" [process where serial_event_id = 2]"));
}
public void testJoinUnsupported() {
assertEquals("1:1: Join is not supported", errorParsing("join by user_name\n" +
" [process where opcode in (1,3) and process_name=\"smss.exe\"]\n" +
" [process where opcode in (1,3) and process_name == \"python.exe\"]"));
}
// Some functions fail with "Unsupported" message at the parse stage
public void testArrayFunctionsUnsupported() {
assertEquals("1:16: Unknown function [arrayContains]",

70
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java
View File

@ -6,7 +6,11 @@
package org.elasticsearch.xpack.eql.parser;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.eql.plan.logical.Join;
import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
import org.elasticsearch.xpack.eql.plan.logical.Sequence;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Order;
import org.elasticsearch.xpack.ql.expression.Order.NullsPosition;
@ -18,6 +22,9 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
import org.elasticsearch.xpack.ql.tree.Source;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static java.util.Collections.singletonList;
public class LogicalPlanTests extends ESTestCase {
@ -58,4 +65,65 @@ public class LogicalPlanTests extends ESTestCase {
LogicalPlan expected = new OrderBy(Source.EMPTY, filter, singletonList(order));
assertEquals(expected, fullQuery);
}
}
public void testJoinPlan() {
LogicalPlan plan = parser.createStatement(
"join by pid " +
" [process where true] " +
" [network where true] " +
" [registry where true] " +
" [file where true] " +
" " +
"until [process where event_subtype_full == \"termination_event\"]");
assertEquals(Join.class, plan.getClass());
Join join = (Join) plan;
assertEquals(KeyedFilter.class, join.until().getClass());
KeyedFilter f = (KeyedFilter) join.until();
Expression key = f.keys().get(0);
assertEquals(UnresolvedAttribute.class, key.getClass());
assertEquals("pid", ((UnresolvedAttribute) key).name());
List<LogicalPlan> queries = join.queries();
assertEquals(4, queries.size());
LogicalPlan subPlan = queries.get(0);
assertEquals(KeyedFilter.class, subPlan.getClass());
KeyedFilter kf = (KeyedFilter) subPlan;
List<Expression> keys = kf.keys();
key = keys.get(0);
assertEquals(UnresolvedAttribute.class, key.getClass());
assertEquals("pid", ((UnresolvedAttribute) key).name());
}
public void testSequencePlan() {
LogicalPlan plan = parser.createStatement(
"sequence by pid with maxspan=2s " +
" [process where process_name == \"*\" ] " +
" [file where file_path == \"*\"]");
assertEquals(Sequence.class, plan.getClass());
Sequence seq = (Sequence) plan;
assertEquals(Filter.class, seq.until().getClass());
Filter f = (Filter) seq.until();
assertEquals(false, f.condition().fold());
List<LogicalPlan> queries = seq.queries();
assertEquals(1, queries.size());
LogicalPlan subPlan = queries.get(0);
assertEquals(KeyedFilter.class, subPlan.getClass());
KeyedFilter kf = (KeyedFilter) subPlan;
List<Expression> keys = kf.keys();
Expression key = keys.get(0);
assertEquals(UnresolvedAttribute.class, key.getClass());
assertEquals("pid", ((UnresolvedAttribute) key).name());
TimeValue maxSpan = seq.maxSpan();
assertEquals(new TimeValue(2, TimeUnit.SECONDS), maxSpan);
}
}

235
x-pack/plugin/eql/src/test/resources/queries-supported.eql
View File

@ -330,3 +330,238 @@ something where `@timestamp` == "2020-01-01 00:00:00";
something where `some escaped identifier` == "blah";
something where `some escaped identifier` == "blah";
something where `some.escaped.identifier` == "blah";
//
// Joins and Sequences
//
//
// Joins
//
// docs
join by source_ip, destination_ip
[network where destination_port == 3389] // RDP
[network where destination_port == 135] // RPC
[network where destination_port == 445] // SMB
;
join by pid
[process where true]
[network where true]
[registry where true]
[file where true]
until [process where event_subtype_full == "termination_event"]
;
///
join
[process where process_name == "*"]
[file where file_path == "*"]
;
join by pid
[process where name == "*"]
[file where path == "*"]
until [process where opcode == 2]
;
join
[process where process_name == "*"] by process_path
[file where file_path == "*"] by image_path
;
join by user_name
[process where opcode in (1,3) and process_name="smss.exe"]
[process where opcode in (1,3) and process_name == "python.exe"]
;
join by unique_pid
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"]
;
join by unique_pid
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"]
until [file where opcode == 2];
join
[file where opcode=0 and file_name="svchost.exe"] by unique_pid
[process where opcode == 1] by unique_ppid
;
join by unique_pid
[process where opcode in (1,3) and process_name="python.exe"]
[file where file_name == "*.exe"];
join by user_name
[process where opcode in (1,3) and process_name="python.exe"]
[process where opcode in (1,3) and process_name == "smss.exe"]
;
join
[process where opcode in (1,3) and process_name="python.exe"]
[process where opcode in (1,3) and process_name == "smss.exe"]
;
//
// Sequences
//
// docs
sequence by user_name
[process where process_name == "whoami"]
[process where process_name == "hostname"]
[process where process_name == "ifconfig"]
;
sequence with maxspan=30s
[network where destination_port==3389 and event_subtype_full="*_accept_event*"]
[security where event_id in (4624, 4625) and logon_type == 10]
;
sequence with maxspan=30s
[network where destination_port==3389 and event_subtype_full="*_accept_event"] by source_address
[security where event_id in (4624, 4625) and logon_type == 10] by ip_address
;
sequence with maxspan=5m
[file where file_name == "*.exe"] by user_name, file_path
[process where true] by user_name, process_path
;
sequence by user_name with maxspan=5m
[file where file_name == "*.exe"] by file_path
[process where true] by process_path
;
//
sequence
[process where name == "*"]
[file where path == "*"]
until [process where opcode == 2]
;
sequence by pid
[process where name == "*"]
[file where path == "*"]
until [process where opcode == 2]
;
sequence
[process where process_name == "*"] by process_path
[file where file_path == "*"] by image_path
;
sequence by pid
[process where process_name == "*"]
[file where file_path == "*"]
;
sequence by field1
[file where true] by f1
[process where true] by f1
;
sequence by a,b,c,d
[file where true] by f1,f2
[process where true] by f1,f2
;
sequence
[file where 1] by f1,f2
[process where 1] by f1,f2
until [process where 1] by f1,f2
;
sequence by f
[file where true] by a,b
[process where true] by c,d
until [process where 1] by e,f
;
sequence
[process where serial_event_id = 1]
[process where serial_event_id = 2]
;
sequence
[process where serial_event_id < 5]
[process where serial_event_id = 5]
;
sequence
[process where serial_event_id=1] by unique_pid
[process where true] by unique_ppid;
sequence
[process where serial_event_id<3] by unique_pid
[process where true] by unique_ppid
;
sequence
[process where serial_event_id < 5]
[process where serial_event_id < 5]
;
sequence
[file where opcode=0 and file_name="svchost.exe"] by unique_pid
[process where opcode == 1] by unique_ppid
;
sequence
[file where file_name="lsass.exe"] by file_path,process_path
[process where true] by process_path,parent_process_path
;
sequence by user_name
[file where file_name="lsass.exe"] by file_path, process_path
[process where true] by process_path, parent_process_path
;
sequence by pid
[file where file_name="lsass.exe"] by file_path,process_path
[process where true] by process_path,parent_process_path
;
sequence by user_name
[file where opcode=0] by pid,file_path
[file where opcode=2] by pid,file_path
until [process where opcode=2] by ppid,process_path
;
sequence by unique_pid
[process where opcode=1 and process_name == 'msbuild.exe']
[network where true]
;
sequence by pid with maxspan=200
[process where process_name == "*" ]
[file where file_path == "*"]
;
sequence by pid with maxspan=2s
[process where process_name == "*" ]
[file where file_path == "*"]
;
sequence by pid with maxspan=2sec
[process where process_name == "*" ]
[file where file_path == "*"]
;
sequence by pid with maxspan=2seconds
[process where process_name == "*" ]
[file where file_path == "*"]
;

217
x-pack/plugin/eql/src/test/resources/queries-unsupported.eql
View File

@ -34,92 +34,8 @@ network where total_out_bytes > 100000000
| tail 5
;
//
// Sequences
//
sequence by user_name
[process where process_name == "whoami"]
[process where process_name == "hostname"]
[process where process_name == "ifconfig"]
;
sequence with maxspan=30s
[network where destination_port==3389 and event_subtype_full="*_accept_event*"]
[security where event_id in (4624, 4625) and logon_type == 10]
;
sequence with maxspan=30s
[network where destination_port==3389 and event_subtype_full="*_accept_event"] by source_address
[security where event_id in (4624, 4625) and logon_type == 10] by ip_address
;
sequence with maxspan=5m
[file where file_name == "*.exe"] by user_name, file_path
[process where true] by user_name, process_path
;
sequence by user_name with maxspan=5m
[file where file_name == "*.exe"] by file_path
[process where true] by process_path
;
//
// Joins
//
join by source_ip, destination_ip
[network where destination_port == 3389] // RDP
[network where destination_port == 135] // RPC
[network where destination_port == 445] // SMB
;
join by pid
[process where true]
[network where true]
[registry where true]
[file where true]
until [process where event_subtype_full == "termination_event"]
;
process where descendant of [process where process_name == "lsass.exe"] and process_name == "cmd.exe";
join [process where process_name == "*"] [file where file_path == "*"
];
join by pid [process where name == "*"] [file where path == "*"] until [process where opcode == 2];
sequence [process where name == "*"] [file where path == "*"] until [process where opcode == 2];
sequence by pid [process where name == "*"] [file where path == "*"] until [process where opcode == 2];
join [process where process_name == "*"] by process_path [file where file_path == "*"] by image_path;
sequence [process where process_name == "*"] by process_path [file where file_path == "*"] by image_path;
sequence by pid [process where process_name == "*"] [file where file_path == "*"];
sequence by pid with maxspan=200 [process where process_name == "*" ] [file where file_path == "*"];
sequence by pid with maxspan=2s [process where process_name == "*" ] [file where file_path == "*"];
sequence by pid with maxspan=2sec [process where process_name == "*" ] [file where file_path == "*"];
sequence by pid with maxspan=2seconds [process where process_name == "*" ] [file where file_path == "*"];
sequence with maxspan=2.5m [process where x == x] by pid [file where file_path == "*"] by ppid;
sequence by pid with maxspan=2.0h [process where process_name == "*"] [file where file_path == "*"];
sequence by pid with maxspan=2.0h [process where process_name == "*"] [file where file_path == "*"];
sequence by pid with maxspan=1.0075d [process where process_name == "*"] [file where file_path == "*"];
dns where pid == 100 | head 100 | tail 50 | unique pid;
network where pid == 100 | unique command_line | count;
@ -159,14 +75,6 @@ file where event of [registry where true];
file where descendant of [registry where true];
sequence by field1 [file where true] by f1 [process where true] by f1;
sequence by a,b,c,d [file where true] by f1,f2 [process where true] by f1,f2;
sequence [file where 1] by f1,f2 [process where 1] by f1,f2 until [process where 1] by f1,f2;
sequence by f [file where true] by a,b [process where true] by c,d until [process where 1] by e,f;
//sequence by unique_pid [process where true] [file where true] fork;
sequence by unique_pid [process where true] [file where true] fork=true;
@ -186,7 +94,25 @@ sequence by unique_pid [process where true] [file where true] fork [network wher
sequence by unique_pid [process where true] [file where true] fork=true;
sequence with maxspan=2.5m
[process where x == x] by pid
[file where file_path == "*"] by ppid
;
sequence by pid with maxspan=2.0h
[process where process_name == "*"]
[file where file_path == "*"]
;
sequence by pid with maxspan=2.0h
[process where process_name == "*"]
[file where file_path == "*"]
;
sequence by pid with maxspan=1.0075d
[process where process_name == "*"]
[file where file_path == "*"]
;
@ -317,35 +243,6 @@ process where true
| sort md5, event_subtype_full, null_field, process_name
| sort serial_event_id;
sequence
[process where serial_event_id = 1]
[process where serial_event_id = 2]
;
sequence
[process where serial_event_id < 5]
[process where serial_event_id = 5]
;
sequence
[process where serial_event_id=1] by unique_pid
[process where true] by unique_ppid;
sequence
[process where serial_event_id<3] by unique_pid
[process where true] by unique_ppid
;
sequence
[process where serial_event_id<3] by unique_pid * 2
[process where true] by unique_ppid * 2
;
sequence
[process where serial_event_id<3] by unique_pid * 2, length(unique_pid), string(unique_pid)
[process where true] by unique_ppid * 2, length(unique_ppid), string(unique_ppid)
;
sequence
[file where event_subtype_full == "file_create_event"] by file_path
[process where opcode == 1] by process_path
@ -394,16 +291,6 @@ sequence with maxspan=0.5s
| head 4
| tail 2;
sequence
[process where serial_event_id < 5]
[process where serial_event_id < 5]
;
sequence
[file where opcode=0 and file_name="svchost.exe"] by unique_pid
[process where opcode == 1] by unique_ppid
;
sequence
[file where opcode=0] by unique_pid
[file where opcode=0] by unique_pid
@ -430,55 +317,24 @@ join
[file where opcode=0 and file_name="*.exe"] by unique_pid
[file where opcode=2 and file_name="*.exe"] by unique_pid
until [process where opcode=1] by unique_ppid
| head 1;
join by user_name
[process where opcode in (1,3) and process_name="smss.exe"]
[process where opcode in (1,3) and process_name == "python.exe"]
| head 1
;
join by unique_pid
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"];
join by string(unique_pid)
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"];
join by unique_pid
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"]
until [file where opcode == 2];
| head 1
;
join by string(unique_pid), unique_pid, unique_pid * 2
[process where opcode=1]
[file where opcode=0 and file_name="svchost.exe"]
[file where opcode == 0 and file_name == "lsass.exe"]
until [file where opcode == 2];
join
[file where opcode=0 and file_name="svchost.exe"] by unique_pid
[process where opcode == 1] by unique_ppid
until [file where opcode == 2]
: tail 1
;
join by unique_pid
[process where opcode in (1,3) and process_name="python.exe"]
[file where file_name == "*.exe"];
join by user_name
[process where opcode in (1,3) and process_name="python.exe"]
[process where opcode in (1,3) and process_name == "smss.exe"]
;
join
[process where opcode in (1,3) and process_name="python.exe"]
[process where opcode in (1,3) and process_name == "smss.exe"]
;
any where true
| unique event_type_full;
@ -522,23 +378,18 @@ file where event of [process where process_name = "python.exe" ]
process where event of [process where process_name = "python.exe" ];
sequence
[file where file_name="lsass.exe"] by file_path,process_path
[process where true] by process_path,parent_process_path
[process where serial_event_id<3] by unique_pid * 2
[process where true] by unique_ppid * 2
| tail 1
;
sequence by user_name
[file where file_name="lsass.exe"] by file_path, process_path
[process where true] by process_path, parent_process_path
;
sequence by pid
[file where file_name="lsass.exe"] by file_path,process_path
[process where true] by process_path,parent_process_path
sequence
[process where serial_event_id<3] by unique_pid * 2, length(unique_pid), string(unique_pid)
[process where true] by unique_ppid * 2, length(unique_ppid), string(unique_ppid)
| tail 1
;
sequence by user_name
@ -548,12 +399,6 @@ sequence by user_name
[file where opcode=2] by file_path
| tail 1;
sequence by user_name
[file where opcode=0] by pid,file_path
[file where opcode=2] by pid,file_path
until [process where opcode=2] by ppid,process_path
;
sequence by user_name
[file where opcode=0] by pid,file_path
[file where opcode=2] by pid,file_path
@ -601,8 +446,6 @@ process where process_name != original_file_name
sequence by unique_pid [process where opcode=1 and process_name == 'msbuild.exe'] [network where true];
process where fake_field != "*"
| head 4;