diff --git a/x-pack/plugin/eql/src/main/antlr/EqlBase.g4 b/x-pack/plugin/eql/src/main/antlr/EqlBase.g4 index dc53354a7d1..ef52fd1bf3d 100644 --- a/x-pack/plugin/eql/src/main/antlr/EqlBase.g4 +++ b/x-pack/plugin/eql/src/main/antlr/EqlBase.g4 @@ -32,13 +32,13 @@ sequenceParams sequence : SEQUENCE (by=joinKeys sequenceParams? | sequenceParams by=joinKeys?)? sequenceTerm sequenceTerm+ - (UNTIL sequenceTerm)? + (UNTIL until=sequenceTerm)? ; join : JOIN (by=joinKeys)? joinTerm joinTerm+ - (UNTIL joinTerm)? + (UNTIL until=joinTerm)? ; pipe diff --git a/x-pack/plugin/eql/src/main/antlr/EqlBase.tokens b/x-pack/plugin/eql/src/main/antlr/EqlBase.tokens new file mode 100644 index 00000000000..c5e847ffadb --- /dev/null +++ b/x-pack/plugin/eql/src/main/antlr/EqlBase.tokens @@ -0,0 +1,77 @@ +AND=1 +ANY=2 +BY=3 +FALSE=4 +FORK=5 +IN=6 +JOIN=7 +MAXSPAN=8 +NOT=9 +NULL=10 +OF=11 +OR=12 +SEQUENCE=13 +TRUE=14 +UNTIL=15 +WHERE=16 +WITH=17 +EQ=18 +NEQ=19 +LT=20 +LTE=21 +GT=22 +GTE=23 +PLUS=24 +MINUS=25 +ASTERISK=26 +SLASH=27 +PERCENT=28 +DOT=29 +COMMA=30 +LB=31 +RB=32 +LP=33 +RP=34 +PIPE=35 +ESCAPED_IDENTIFIER=36 +STRING=37 +INTEGER_VALUE=38 +DECIMAL_VALUE=39 +IDENTIFIER=40 +LINE_COMMENT=41 +BRACKETED_COMMENT=42 +WS=43 +'and'=1 +'any'=2 +'by'=3 +'false'=4 +'fork'=5 +'in'=6 +'join'=7 +'maxspan'=8 +'not'=9 +'null'=10 +'of'=11 +'or'=12 +'sequence'=13 +'true'=14 +'until'=15 +'where'=16 +'with'=17 +'!='=19 +'<'=20 +'<='=21 +'>'=22 +'>='=23 +'+'=24 +'-'=25 +'*'=26 +'/'=27 +'%'=28 +'.'=29 +','=30 +'['=31 +']'=32 +'('=33 +')'=34 +'|'=35 diff --git a/x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens b/x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens new file mode 100644 index 00000000000..c5e847ffadb --- /dev/null +++ b/x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens @@ -0,0 +1,77 @@ +AND=1 +ANY=2 +BY=3 +FALSE=4 +FORK=5 +IN=6 +JOIN=7 +MAXSPAN=8 +NOT=9 +NULL=10 +OF=11 +OR=12 +SEQUENCE=13 +TRUE=14 +UNTIL=15 +WHERE=16 +WITH=17 +EQ=18 +NEQ=19 +LT=20 +LTE=21 +GT=22 +GTE=23 +PLUS=24 +MINUS=25 +ASTERISK=26 +SLASH=27 +PERCENT=28 +DOT=29 +COMMA=30 +LB=31 +RB=32 +LP=33 +RP=34 +PIPE=35 +ESCAPED_IDENTIFIER=36 +STRING=37 +INTEGER_VALUE=38 +DECIMAL_VALUE=39 +IDENTIFIER=40 +LINE_COMMENT=41 +BRACKETED_COMMENT=42 +WS=43 +'and'=1 +'any'=2 +'by'=3 +'false'=4 +'fork'=5 +'in'=6 +'join'=7 +'maxspan'=8 +'not'=9 +'null'=10 +'of'=11 +'or'=12 +'sequence'=13 +'true'=14 +'until'=15 +'where'=16 +'with'=17 +'!='=19 +'<'=20 +'<='=21 +'>'=22 +'>='=23 +'+'=24 +'-'=25 +'*'=26 +'/'=27 +'%'=28 +'.'=29 +','=30 +'['=31 +']'=32 +'('=33 +')'=34 +'|'=35 diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java index 73539cfb33f..624cda6dc9c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java @@ -393,6 +393,7 @@ class EqlBaseParser extends Parser { public static class SequenceContext extends ParserRuleContext { public JoinKeysContext by; + public SequenceTermContext until; public TerminalNode SEQUENCE() { return getToken(EqlBaseParser.SEQUENCE, 0); } public List sequenceTerm() { return getRuleContexts(SequenceTermContext.class); @@ -495,7 +496,7 @@ class EqlBaseParser extends Parser { setState(94); match(UNTIL); setState(95); - sequenceTerm(); + ((SequenceContext)_localctx).until = sequenceTerm(); } } @@ -514,6 +515,7 @@ class EqlBaseParser extends Parser { public static class JoinContext extends ParserRuleContext { public JoinKeysContext by; + public JoinTermContext until; public TerminalNode JOIN() { return getToken(EqlBaseParser.JOIN, 0); } public List joinTerm() { return getRuleContexts(JoinTermContext.class); @@ -585,7 +587,7 @@ class EqlBaseParser extends Parser { setState(108); match(UNTIL); setState(109); - joinTerm(); + ((JoinContext)_localctx).until = joinTerm(); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java index 81f5eeecfb3..17324b42d0e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java @@ -133,16 +133,6 @@ public class EqlParser { this.ruleNames = ruleNames; } - @Override - public void exitJoin(EqlBaseParser.JoinContext context) { - Token token = context.JOIN().getSymbol(); - throw new ParsingException( - "Join is not supported", - null, - token.getLine(), - token.getCharPositionInLine()); - } - @Override public void exitPipe(EqlBaseParser.PipeContext context) { Token token = context.PIPE().getSymbol(); @@ -163,16 +153,6 @@ public class EqlParser { token.getCharPositionInLine()); } - @Override - public void exitSequence(EqlBaseParser.SequenceContext context) { - Token token = context.SEQUENCE().getSymbol(); - throw new ParsingException( - "Sequence is not supported", - null, - token.getLine(), - token.getCharPositionInLine()); - } - @Override public void exitQualifiedName(EqlBaseParser.QualifiedNameContext context) { if (context.INTEGER_VALUE().size() > 0) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index e5e344f5acd..9303f6fbcdb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ArithmeticUnaryContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ComparisonContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.DereferenceContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.FunctionExpressionContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinKeysContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.LogicalBinaryContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.LogicalNotContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.PredicateContext; @@ -46,6 +47,8 @@ import org.elasticsearch.xpack.ql.util.StringUtils; import java.util.List; +import static java.util.Collections.emptyList; + public class ExpressionBuilder extends IdentifierBuilder { @@ -62,6 +65,11 @@ public class ExpressionBuilder extends IdentifierBuilder { return expression(ctx.expression()); } + @Override + public List visitJoinKeys(JoinKeysContext ctx) { + return ctx != null ? expressions(ctx.expression()) : emptyList(); + } + @Override public Expression visitArithmeticUnary(ArithmeticUnaryContext ctx) { Expression expr = expression(ctx.valueExpression()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java index 1a42b7cbb21..0aba0f208d2 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java @@ -5,6 +5,17 @@ */ package org.elasticsearch.xpack.eql.parser; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.IntegerLiteralContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinTermContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.NumberContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceParamsContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceTermContext; +import org.elasticsearch.xpack.eql.plan.logical.Join; +import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; +import org.elasticsearch.xpack.eql.plan.logical.Sequence; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.Order; @@ -17,12 +28,18 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import static java.util.Collections.singletonList; public abstract class LogicalPlanBuilder extends ExpressionBuilder { private final ParserParams params; + private final UnresolvedRelation RELATION = new UnresolvedRelation(Source.EMPTY, null, "", false, ""); public LogicalPlanBuilder(ParserParams params) { this.params = params; @@ -44,11 +61,157 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { condition = new And(source, eventMatch, condition); } - Filter filter = new Filter(source, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition); - // add implicit sorting - when pipes are added, this would better seat there (as a default pipe) + Filter filter = new Filter(source, RELATION, condition); + // add implicit sorting - when pipes are added, this would better sit there (as a default pipe) Order order = new Order(source, new UnresolvedAttribute(source, params.fieldTimestamp()), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); OrderBy orderBy = new OrderBy(source, filter, singletonList(order)); return orderBy; } -} + + @Override + public Join visitJoin(JoinContext ctx) { + List parentJoinKeys = visitJoinKeys(ctx.by); + + LogicalPlan until; + + if (ctx.until != null) { + until = visitJoinTerm(ctx.until, parentJoinKeys); + } else { + // no until declared means the condition never gets executed and thus folds to false + until = new Filter(source(ctx), RELATION, new Literal(source(ctx), Boolean.FALSE, DataTypes.BOOLEAN)); + } + + int numberOfKeys = -1; + List queries = new ArrayList<>(ctx.joinTerm().size()); + + for (JoinTermContext joinTermCtx : ctx.joinTerm()) { + KeyedFilter joinTerm = visitJoinTerm(joinTermCtx, parentJoinKeys); + int keySize = joinTerm.keys().size(); + if (numberOfKeys < 0) { + numberOfKeys = keySize; + } else { + if (numberOfKeys != keySize) { + Source src = source(joinTermCtx.by != null ? joinTermCtx.by : joinTermCtx); + int expected = numberOfKeys - parentJoinKeys.size(); + int found = keySize - parentJoinKeys.size(); + throw new ParsingException(src, "Inconsistent number of join keys specified; expected [{}] but found [{}]", expected, + found); + } + queries.add(joinTerm); + } + } + + return new Join(source(ctx), queries, until); + } + + public KeyedFilter visitJoinTerm(JoinTermContext ctx, List joinKeys) { + List keys = CollectionUtils.combine(joinKeys, visitJoinKeys(ctx.by)); + return new KeyedFilter(source(ctx), visitEventQuery(ctx.subquery().eventQuery()), keys); + } + + @Override + public Sequence visitSequence(SequenceContext ctx) { + List parentJoinKeys = visitJoinKeys(ctx.by); + + TimeValue maxSpan = visitSequenceParams(ctx.sequenceParams()); + + LogicalPlan until; + + if (ctx.until != null) { + until = visitSequenceTerm(ctx.until, parentJoinKeys); + } else { + // no until declared means the condition never gets executed and thus folds to false + until = new Filter(source(ctx), RELATION, new Literal(source(ctx), Boolean.FALSE, DataTypes.BOOLEAN)); + } + + int numberOfKeys = -1; + List queries = new ArrayList<>(ctx.sequenceTerm().size()); + + for (SequenceTermContext sequenceTermCtx : ctx.sequenceTerm()) { + KeyedFilter sequenceTerm = visitSequenceTerm(sequenceTermCtx, parentJoinKeys); + int keySize = sequenceTerm.keys().size(); + if (numberOfKeys < 0) { + numberOfKeys = keySize; + } else { + if (numberOfKeys != keySize) { + Source src = source(sequenceTermCtx.by != null ? sequenceTermCtx.by : sequenceTermCtx); + int expected = numberOfKeys - parentJoinKeys.size(); + int found = keySize - parentJoinKeys.size(); + throw new ParsingException(src, "Inconsistent number of join keys specified; expected [{}] but found [{}]", expected, + found); + } + queries.add(sequenceTerm); + } + } + + return new Sequence(source(ctx), queries, until, maxSpan); + } + + public KeyedFilter visitSequenceTerm(SequenceTermContext ctx, List joinKeys) { + if (ctx.FORK() != null) { + throw new ParsingException(source(ctx.FORK()), "sequence fork is unsupported"); + } + + List keys = CollectionUtils.combine(joinKeys, visitJoinKeys(ctx.by)); + return new KeyedFilter(source(ctx), visitEventQuery(ctx.subquery().eventQuery()), keys); + } + + @Override + public TimeValue visitSequenceParams(SequenceParamsContext ctx) { + if (ctx == null) { + return TimeValue.MINUS_ONE; + } + + NumberContext numberCtx = ctx.timeUnit().number(); + if (numberCtx instanceof IntegerLiteralContext) { + Number number = (Number) visitIntegerLiteral((IntegerLiteralContext) numberCtx).fold(); + long value = number.longValue(); + + if (value <= 0) { + throw new ParsingException(source(numberCtx), "A positive maxspan value is required; found [{}]", value); + } + + String timeString = text(ctx.timeUnit().IDENTIFIER()); + TimeUnit timeUnit = TimeUnit.SECONDS; + if (timeString != null) { + switch (timeString) { + case "": + case "s": + case "sec": + case "secs": + case "second": + case "seconds": + timeUnit = TimeUnit.SECONDS; + break; + case "m": + case "min": + case "mins": + case "minute": + case "minutes": + timeUnit = TimeUnit.MINUTES; + break; + case "h": + case "hs": + case "hour": + case "hours": + timeUnit = TimeUnit.HOURS; + break; + case "d": + case "ds": + case "day": + case "days": + timeUnit = TimeUnit.DAYS; + break; + default: + throw new ParsingException(source(ctx.timeUnit().IDENTIFIER()), "Unrecognized time unit [{}]", timeString); + } + } + + return new TimeValue(value, timeUnit); + + } else { + throw new ParsingException(source(numberCtx), "Decimal time interval [{}] not supported yet", text(numberCtx)); + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java new file mode 100644 index 00000000000..eb746d8c7f8 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static java.util.Collections.emptyList; + +public class Join extends LogicalPlan { + + private final List queries; + private final LogicalPlan until; + + public Join(Source source, List queries, LogicalPlan until) { + super(source, CollectionUtils.combine(queries, until)); + this.queries = queries; + this.until = until; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Join::new, queries, until); + } + + @Override + public Join replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size()); + } + int lastIndex = newChildren.size() - 1; + return new Join(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex)); + } + + @Override + public List output() { + List out = new ArrayList<>(); + for (LogicalPlan plan : queries) { + out.addAll(plan.output()); + } + if (until != null) { + out.addAll(until.output()); + } + + return out; + } + + @Override + public boolean expressionsResolved() { + return true; + } + + public List queries() { + return queries; + } + + public LogicalPlan until() { + return until; + } + + @Override + public int hashCode() { + return Objects.hash(queries, until); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + Join other = (Join) obj; + + return Objects.equals(queries, other.queries) + && Objects.equals(until, other.until); + } + + @Override + public List nodeProperties() { + return emptyList(); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/KeyedFilter.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/KeyedFilter.java new file mode 100644 index 00000000000..cd082b09a67 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/KeyedFilter.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.xpack.ql.capabilities.Resolvables; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.Objects; + +/** + * Filter that has one or multiple associated keys associated with. + * Used inside Join or Sequence. + */ +public class KeyedFilter extends UnaryPlan { + + private final List keys; + + public KeyedFilter(Source source, LogicalPlan child, List keys) { + super(source, child); + this.keys = keys; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, KeyedFilter::new, child(), keys); + } + + @Override + protected KeyedFilter replaceChild(LogicalPlan newChild) { + return new KeyedFilter(source(), newChild, keys); + } + + public List keys() { + return keys; + } + + @Override + public boolean expressionsResolved() { + return Resolvables.resolved(keys); + } + + @Override + public int hashCode() { + return Objects.hash(keys, child()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KeyedFilter other = (KeyedFilter) obj; + + return Objects.equals(keys, other.keys) + && Objects.equals(child(), other.child()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java new file mode 100644 index 00000000000..cf68f808008 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; +import java.util.Objects; + +import static java.util.Collections.singletonList; + +public class Sequence extends Join { + + private final TimeValue maxSpan; + + public Sequence(Source source, List queries, LogicalPlan until, TimeValue maxSpan) { + super(source, queries, until); + this.maxSpan = maxSpan; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sequence::new, queries(), until(), maxSpan); + } + + @Override + public Join replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size()); + } + int lastIndex = newChildren.size() - 1; + return new Sequence(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), maxSpan); + } + + public TimeValue maxSpan() { + return maxSpan; + } + + @Override + public int hashCode() { + return Objects.hash(maxSpan, queries(), until()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + Sequence other = (Sequence) obj; + + return Objects.equals(maxSpan, other.maxSpan) + && Objects.equals(queries(), other.queries()) + && Objects.equals(until(), other.until()); + } + + @Override + public List nodeProperties() { + return singletonList(maxSpan); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 4c97191d6f8..99d83b7adea 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -99,18 +99,6 @@ public class VerifierTests extends ESTestCase { " and child of [file where file_name=\"svchost.exe\" and opcode=0]")); } - public void testSequencesUnsupported() { - assertEquals("1:1: Sequence is not supported", errorParsing("sequence\n" + - " [process where serial_event_id = 1]\n" + - " [process where serial_event_id = 2]")); - } - - public void testJoinUnsupported() { - assertEquals("1:1: Join is not supported", errorParsing("join by user_name\n" + - " [process where opcode in (1,3) and process_name=\"smss.exe\"]\n" + - " [process where opcode in (1,3) and process_name == \"python.exe\"]")); - } - // Some functions fail with "Unsupported" message at the parse stage public void testArrayFunctionsUnsupported() { assertEquals("1:16: Unknown function [arrayContains]", diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index 2b36efc8f15..4ef443aa64c 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -6,7 +6,11 @@ package org.elasticsearch.xpack.eql.parser; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.eql.plan.logical.Join; +import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; +import org.elasticsearch.xpack.eql.plan.logical.Sequence; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Order; import org.elasticsearch.xpack.ql.expression.Order.NullsPosition; @@ -18,6 +22,9 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.tree.Source; +import java.util.List; +import java.util.concurrent.TimeUnit; + import static java.util.Collections.singletonList; public class LogicalPlanTests extends ESTestCase { @@ -58,4 +65,65 @@ public class LogicalPlanTests extends ESTestCase { LogicalPlan expected = new OrderBy(Source.EMPTY, filter, singletonList(order)); assertEquals(expected, fullQuery); } -} + + + public void testJoinPlan() { + LogicalPlan plan = parser.createStatement( + "join by pid " + + " [process where true] " + + " [network where true] " + + " [registry where true] " + + " [file where true] " + + " " + + "until [process where event_subtype_full == \"termination_event\"]"); + + assertEquals(Join.class, plan.getClass()); + Join join = (Join) plan; + assertEquals(KeyedFilter.class, join.until().getClass()); + KeyedFilter f = (KeyedFilter) join.until(); + Expression key = f.keys().get(0); + assertEquals(UnresolvedAttribute.class, key.getClass()); + assertEquals("pid", ((UnresolvedAttribute) key).name()); + + List queries = join.queries(); + assertEquals(4, queries.size()); + LogicalPlan subPlan = queries.get(0); + assertEquals(KeyedFilter.class, subPlan.getClass()); + KeyedFilter kf = (KeyedFilter) subPlan; + + List keys = kf.keys(); + key = keys.get(0); + assertEquals(UnresolvedAttribute.class, key.getClass()); + assertEquals("pid", ((UnresolvedAttribute) key).name()); + + } + + + public void testSequencePlan() { + LogicalPlan plan = parser.createStatement( + "sequence by pid with maxspan=2s " + + " [process where process_name == \"*\" ] " + + " [file where file_path == \"*\"]"); + + assertEquals(Sequence.class, plan.getClass()); + Sequence seq = (Sequence) plan; + assertEquals(Filter.class, seq.until().getClass()); + Filter f = (Filter) seq.until(); + assertEquals(false, f.condition().fold()); + + List queries = seq.queries(); + assertEquals(1, queries.size()); + LogicalPlan subPlan = queries.get(0); + assertEquals(KeyedFilter.class, subPlan.getClass()); + KeyedFilter kf = (KeyedFilter) subPlan; + + List keys = kf.keys(); + Expression key = keys.get(0); + assertEquals(UnresolvedAttribute.class, key.getClass()); + assertEquals("pid", ((UnresolvedAttribute) key).name()); + + TimeValue maxSpan = seq.maxSpan(); + assertEquals(new TimeValue(2, TimeUnit.SECONDS), maxSpan); + + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/resources/queries-supported.eql b/x-pack/plugin/eql/src/test/resources/queries-supported.eql index 8326d558ba4..41d8c0f4a74 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-supported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-supported.eql @@ -330,3 +330,238 @@ something where `@timestamp` == "2020-01-01 00:00:00"; something where `some escaped identifier` == "blah"; something where `some escaped identifier` == "blah"; something where `some.escaped.identifier` == "blah"; + + +// +// Joins and Sequences +// + + +// +// Joins +// + +// docs + +join by source_ip, destination_ip + [network where destination_port == 3389] // RDP + [network where destination_port == 135] // RPC + [network where destination_port == 445] // SMB +; + +join by pid + [process where true] + [network where true] + [registry where true] + [file where true] + +until [process where event_subtype_full == "termination_event"] +; + +/// + +join + [process where process_name == "*"] + [file where file_path == "*"] +; + +join by pid + [process where name == "*"] + [file where path == "*"] +until [process where opcode == 2] +; + +join + [process where process_name == "*"] by process_path + [file where file_path == "*"] by image_path +; + +join by user_name + [process where opcode in (1,3) and process_name="smss.exe"] + [process where opcode in (1,3) and process_name == "python.exe"] +; + +join by unique_pid + [process where opcode=1] + [file where opcode=0 and file_name="svchost.exe"] + [file where opcode == 0 and file_name == "lsass.exe"] +; + +join by unique_pid + [process where opcode=1] + [file where opcode=0 and file_name="svchost.exe"] + [file where opcode == 0 and file_name == "lsass.exe"] +until [file where opcode == 2]; + +join + [file where opcode=0 and file_name="svchost.exe"] by unique_pid + [process where opcode == 1] by unique_ppid +; + +join by unique_pid + [process where opcode in (1,3) and process_name="python.exe"] + [file where file_name == "*.exe"]; + +join by user_name + [process where opcode in (1,3) and process_name="python.exe"] + [process where opcode in (1,3) and process_name == "smss.exe"] +; + +join + [process where opcode in (1,3) and process_name="python.exe"] + [process where opcode in (1,3) and process_name == "smss.exe"] +; + + +// +// Sequences +// + +// docs +sequence by user_name + [process where process_name == "whoami"] + [process where process_name == "hostname"] + [process where process_name == "ifconfig"] +; + +sequence with maxspan=30s + [network where destination_port==3389 and event_subtype_full="*_accept_event*"] + [security where event_id in (4624, 4625) and logon_type == 10] +; + +sequence with maxspan=30s + [network where destination_port==3389 and event_subtype_full="*_accept_event"] by source_address + [security where event_id in (4624, 4625) and logon_type == 10] by ip_address +; + +sequence with maxspan=5m + [file where file_name == "*.exe"] by user_name, file_path + [process where true] by user_name, process_path +; + +sequence by user_name with maxspan=5m + [file where file_name == "*.exe"] by file_path + [process where true] by process_path +; + +// + +sequence + [process where name == "*"] + [file where path == "*"] +until [process where opcode == 2] +; + +sequence by pid + [process where name == "*"] + [file where path == "*"] +until [process where opcode == 2] +; + + +sequence + [process where process_name == "*"] by process_path + [file where file_path == "*"] by image_path +; + +sequence by pid + [process where process_name == "*"] + [file where file_path == "*"] +; + +sequence by field1 + [file where true] by f1 + [process where true] by f1 +; + +sequence by a,b,c,d + [file where true] by f1,f2 + [process where true] by f1,f2 +; + +sequence + [file where 1] by f1,f2 + [process where 1] by f1,f2 +until [process where 1] by f1,f2 +; + +sequence by f + [file where true] by a,b + [process where true] by c,d +until [process where 1] by e,f +; + +sequence + [process where serial_event_id = 1] + [process where serial_event_id = 2] +; + +sequence + [process where serial_event_id < 5] + [process where serial_event_id = 5] +; + +sequence + [process where serial_event_id=1] by unique_pid + [process where true] by unique_ppid; + +sequence + [process where serial_event_id<3] by unique_pid + [process where true] by unique_ppid +; + +sequence + [process where serial_event_id < 5] + [process where serial_event_id < 5] +; + +sequence + [file where opcode=0 and file_name="svchost.exe"] by unique_pid + [process where opcode == 1] by unique_ppid +; + +sequence + [file where file_name="lsass.exe"] by file_path,process_path + [process where true] by process_path,parent_process_path +; + +sequence by user_name + [file where file_name="lsass.exe"] by file_path, process_path + [process where true] by process_path, parent_process_path +; + +sequence by pid + [file where file_name="lsass.exe"] by file_path,process_path + [process where true] by process_path,parent_process_path +; + +sequence by user_name + [file where opcode=0] by pid,file_path + [file where opcode=2] by pid,file_path +until [process where opcode=2] by ppid,process_path +; + +sequence by unique_pid + [process where opcode=1 and process_name == 'msbuild.exe'] + [network where true] +; + +sequence by pid with maxspan=200 + [process where process_name == "*" ] + [file where file_path == "*"] +; + +sequence by pid with maxspan=2s + [process where process_name == "*" ] + [file where file_path == "*"] +; + +sequence by pid with maxspan=2sec + [process where process_name == "*" ] + [file where file_path == "*"] +; + +sequence by pid with maxspan=2seconds + [process where process_name == "*" ] + [file where file_path == "*"] +; \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql index e9c20febfed..46bbd6d02ee 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql @@ -34,92 +34,8 @@ network where total_out_bytes > 100000000 | tail 5 ; -// -// Sequences -// - -sequence by user_name - [process where process_name == "whoami"] - [process where process_name == "hostname"] - [process where process_name == "ifconfig"] -; - -sequence with maxspan=30s - [network where destination_port==3389 and event_subtype_full="*_accept_event*"] - [security where event_id in (4624, 4625) and logon_type == 10] -; - -sequence with maxspan=30s - [network where destination_port==3389 and event_subtype_full="*_accept_event"] by source_address - [security where event_id in (4624, 4625) and logon_type == 10] by ip_address -; - -sequence with maxspan=5m - [file where file_name == "*.exe"] by user_name, file_path - [process where true] by user_name, process_path -; - -sequence by user_name with maxspan=5m - [file where file_name == "*.exe"] by file_path - [process where true] by process_path -; - -// -// Joins -// - -join by source_ip, destination_ip - [network where destination_port == 3389] // RDP - [network where destination_port == 135] // RPC - [network where destination_port == 445] // SMB -; - -join by pid - [process where true] - [network where true] - [registry where true] - [file where true] - -until [process where event_subtype_full == "termination_event"] -; - - - - - process where descendant of [process where process_name == "lsass.exe"] and process_name == "cmd.exe"; - join [process where process_name == "*"] [file where file_path == "*" - ]; - - join by pid [process where name == "*"] [file where path == "*"] until [process where opcode == 2]; - -sequence [process where name == "*"] [file where path == "*"] until [process where opcode == 2]; - -sequence by pid [process where name == "*"] [file where path == "*"] until [process where opcode == 2]; - - join [process where process_name == "*"] by process_path [file where file_path == "*"] by image_path; - -sequence [process where process_name == "*"] by process_path [file where file_path == "*"] by image_path; - -sequence by pid [process where process_name == "*"] [file where file_path == "*"]; - -sequence by pid with maxspan=200 [process where process_name == "*" ] [file where file_path == "*"]; - -sequence by pid with maxspan=2s [process where process_name == "*" ] [file where file_path == "*"]; - -sequence by pid with maxspan=2sec [process where process_name == "*" ] [file where file_path == "*"]; - -sequence by pid with maxspan=2seconds [process where process_name == "*" ] [file where file_path == "*"]; - -sequence with maxspan=2.5m [process where x == x] by pid [file where file_path == "*"] by ppid; - -sequence by pid with maxspan=2.0h [process where process_name == "*"] [file where file_path == "*"]; - -sequence by pid with maxspan=2.0h [process where process_name == "*"] [file where file_path == "*"]; - -sequence by pid with maxspan=1.0075d [process where process_name == "*"] [file where file_path == "*"]; - dns where pid == 100 | head 100 | tail 50 | unique pid; network where pid == 100 | unique command_line | count; @@ -159,14 +75,6 @@ file where event of [registry where true]; file where descendant of [registry where true]; -sequence by field1 [file where true] by f1 [process where true] by f1; - -sequence by a,b,c,d [file where true] by f1,f2 [process where true] by f1,f2; - -sequence [file where 1] by f1,f2 [process where 1] by f1,f2 until [process where 1] by f1,f2; - -sequence by f [file where true] by a,b [process where true] by c,d until [process where 1] by e,f; - //sequence by unique_pid [process where true] [file where true] fork; sequence by unique_pid [process where true] [file where true] fork=true; @@ -186,7 +94,25 @@ sequence by unique_pid [process where true] [file where true] fork [network wher sequence by unique_pid [process where true] [file where true] fork=true; +sequence with maxspan=2.5m + [process where x == x] by pid + [file where file_path == "*"] by ppid +; +sequence by pid with maxspan=2.0h + [process where process_name == "*"] + [file where file_path == "*"] +; + +sequence by pid with maxspan=2.0h + [process where process_name == "*"] + [file where file_path == "*"] +; + +sequence by pid with maxspan=1.0075d + [process where process_name == "*"] + [file where file_path == "*"] +; @@ -317,35 +243,6 @@ process where true | sort md5, event_subtype_full, null_field, process_name | sort serial_event_id; -sequence - [process where serial_event_id = 1] - [process where serial_event_id = 2] -; - -sequence - [process where serial_event_id < 5] - [process where serial_event_id = 5] -; - -sequence - [process where serial_event_id=1] by unique_pid - [process where true] by unique_ppid; - -sequence - [process where serial_event_id<3] by unique_pid - [process where true] by unique_ppid -; - -sequence - [process where serial_event_id<3] by unique_pid * 2 - [process where true] by unique_ppid * 2 -; - -sequence - [process where serial_event_id<3] by unique_pid * 2, length(unique_pid), string(unique_pid) - [process where true] by unique_ppid * 2, length(unique_ppid), string(unique_ppid) -; - sequence [file where event_subtype_full == "file_create_event"] by file_path [process where opcode == 1] by process_path @@ -394,16 +291,6 @@ sequence with maxspan=0.5s | head 4 | tail 2; -sequence - [process where serial_event_id < 5] - [process where serial_event_id < 5] -; - -sequence - [file where opcode=0 and file_name="svchost.exe"] by unique_pid - [process where opcode == 1] by unique_ppid -; - sequence [file where opcode=0] by unique_pid [file where opcode=0] by unique_pid @@ -430,55 +317,24 @@ join [file where opcode=0 and file_name="*.exe"] by unique_pid [file where opcode=2 and file_name="*.exe"] by unique_pid until [process where opcode=1] by unique_ppid -| head 1; - -join by user_name - [process where opcode in (1,3) and process_name="smss.exe"] - [process where opcode in (1,3) and process_name == "python.exe"] +| head 1 ; -join by unique_pid - [process where opcode=1] - [file where opcode=0 and file_name="svchost.exe"] - [file where opcode == 0 and file_name == "lsass.exe"]; - join by string(unique_pid) - [process where opcode=1] - [file where opcode=0 and file_name="svchost.exe"] - [file where opcode == 0 and file_name == "lsass.exe"]; - -join by unique_pid [process where opcode=1] [file where opcode=0 and file_name="svchost.exe"] [file where opcode == 0 and file_name == "lsass.exe"] -until [file where opcode == 2]; +| head 1 +; join by string(unique_pid), unique_pid, unique_pid * 2 [process where opcode=1] [file where opcode=0 and file_name="svchost.exe"] [file where opcode == 0 and file_name == "lsass.exe"] -until [file where opcode == 2]; - -join - [file where opcode=0 and file_name="svchost.exe"] by unique_pid - [process where opcode == 1] by unique_ppid +until [file where opcode == 2] +: tail 1 ; -join by unique_pid - [process where opcode in (1,3) and process_name="python.exe"] - [file where file_name == "*.exe"]; - -join by user_name - [process where opcode in (1,3) and process_name="python.exe"] - [process where opcode in (1,3) and process_name == "smss.exe"] -; - -join - [process where opcode in (1,3) and process_name="python.exe"] - [process where opcode in (1,3) and process_name == "smss.exe"] -; - - any where true | unique event_type_full; @@ -522,23 +378,18 @@ file where event of [process where process_name = "python.exe" ] - - process where event of [process where process_name = "python.exe" ]; sequence - [file where file_name="lsass.exe"] by file_path,process_path - [process where true] by process_path,parent_process_path + [process where serial_event_id<3] by unique_pid * 2 + [process where true] by unique_ppid * 2 +| tail 1 ; -sequence by user_name - [file where file_name="lsass.exe"] by file_path, process_path - [process where true] by process_path, parent_process_path -; - -sequence by pid - [file where file_name="lsass.exe"] by file_path,process_path - [process where true] by process_path,parent_process_path +sequence + [process where serial_event_id<3] by unique_pid * 2, length(unique_pid), string(unique_pid) + [process where true] by unique_ppid * 2, length(unique_ppid), string(unique_ppid) +| tail 1 ; sequence by user_name @@ -548,12 +399,6 @@ sequence by user_name [file where opcode=2] by file_path | tail 1; -sequence by user_name - [file where opcode=0] by pid,file_path - [file where opcode=2] by pid,file_path -until [process where opcode=2] by ppid,process_path -; - sequence by user_name [file where opcode=0] by pid,file_path [file where opcode=2] by pid,file_path @@ -601,8 +446,6 @@ process where process_name != original_file_name -sequence by unique_pid [process where opcode=1 and process_name == 'msbuild.exe'] [network where true]; - process where fake_field != "*" | head 4;