diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml index 038a1c3efc5..64e0b7adb79 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml @@ -82,6 +82,12 @@ query = ''' process where match(command_line, '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') ''' +[[queries]] +expected_event_ids = [50, 98] +query = ''' +process where match(substring(command_line, 5), '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') +''' + [[queries]] # Basic test for modulo function query = ''' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java index 6fefd865995..dab176ba442 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList; public class Analyzer extends RuleExecutor { @@ -42,8 +42,8 @@ public class Analyzer extends RuleExecutor { Batch resolution = new Batch("Resolution", new ResolveRefs(), new ResolveFunctions()); - - return asList(resolution); + + return singletonList(resolution); } public LogicalPlan analyze(LogicalPlan plan) { @@ -123,4 +123,4 @@ public class Analyzer extends RuleExecutor { }); } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Match.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Match.java index 9721b031fa4..e79c4a0ff6b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Match.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Match.java @@ -36,9 +36,13 @@ public class Match extends BaseSurrogateFunction { private final List patterns; public Match(Source source, Expression field, List patterns) { - super(source, CollectionUtils.combine(singletonList(field), patterns)); - this.field = field; - this.patterns = patterns; + this(source, CollectionUtils.combine(singletonList(field), patterns)); + } + + private Match(Source source, List children) { + super(source, children); + this.field = children().get(0); + this.patterns = children().subList(1, children().size()); } @Override @@ -51,7 +55,7 @@ public class Match extends BaseSurrogateFunction { if (newChildren.size() < 2) { throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); } - return new Match(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); + return new Match(source(), newChildren); } @Override diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 4f1343afd86..0753063103f 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -257,6 +257,14 @@ process where match(command_line, "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\ "regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+" ; +matchFunctionScalar +process where match(substring(command_line, 5), "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(InternalEqlScriptUtils.substring( +InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", +"params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"}} +; + wildcardFunctionSingleArgument process where wildcard(process_path, "*\\red_ttp\\wininit.*") ; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java index a894a86524a..fad8020f680 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java @@ -121,19 +121,17 @@ public final class ExpressionTranslators { if (e.field() instanceof FieldAttribute) { targetFieldName = handler.nameOf(((FieldAttribute) e.field()).exactAttribute()); + if (e instanceof Like) { + LikePattern p = ((Like) e).pattern(); + q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard()); + } + + if (e instanceof RLike) { + String pattern = ((RLike) e).pattern().asJavaRegex(); + q = new RegexQuery(e.source(), targetFieldName, pattern); + } } else { - throw new QlIllegalArgumentException("Scalar function [{}] not allowed (yet) as argument for " + e.sourceText(), - Expressions.name(e.field())); - } - - if (e instanceof Like) { - LikePattern p = ((Like) e).pattern(); - q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard()); - } - - if (e instanceof RLike) { - String pattern = ((RLike) e).pattern().asJavaRegex(); - q = new RegexQuery(e.source(), targetFieldName, pattern); + q = new ScriptQuery(e.source(), e.asScript()); } return wrapIfNested(q, e.field()); diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec index a5d2b185029..dde36cf040c 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec @@ -12,6 +12,16 @@ SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name RLIKE 'S.* Simmel ; +whereFieldWithRLikeWithScalarsMatch +SELECT LTRIM(concat(' ', first_name)) lt FROM test_emp WHERE LTRIM(concat(' ', first_name)) RLIKE '.*z' ORDER BY emp_no; + + lt +--------------- +Erez +Parviz +; + + whereFieldWithNotRLikeMatch SELECT last_name, first_name FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT RLIKE 'Ma.*' ORDER BY first_name LIMIT 5; diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec index 0fe38c48cb0..42185c773a4 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec @@ -51,6 +51,8 @@ whereFieldWithNotEqualsOnString SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND gender <> 'M'; whereFieldWithLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name LIKE 'K%'; +whereFieldWithLikeAndScalarsMatch +SELECT RTRIM(CONCAT(last_name, ' ')) AS l FROM "test_emp" WHERE RTRIM(CONCAT(last_name, ' ')) LIKE '%k%' ORDER BY emp_no; whereFieldWithNotLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT LIKE 'Ma%'; whereFieldWithInlineLikeMatch diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 8b7f6cc877f..d752492167a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -577,24 +577,34 @@ public class QueryTranslatorTests extends ESTestCase { assertEquals("some.string.typical", qsq.field()); } - public void testLikeConstructsNotSupported() { + public void testLikeWithScalars() { LogicalPlan p = plan("SELECT LTRIM(keyword) lt FROM test WHERE LTRIM(keyword) like '%a%'"); assertTrue(p instanceof Project); p = ((Project) p).child(); assertTrue(p instanceof Filter); Expression condition = ((Filter) p).condition(); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> translate(condition)); - assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) like '%a%'", ex.getMessage()); + QueryTranslation qt = translate(condition); + assertTrue(qt.query instanceof ScriptQuery); + ScriptQuery sc = (ScriptQuery) qt.query; + assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(" + + "InternalSqlScriptUtils.ltrim(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", + sc.script().toString()); + assertEquals("[{v=keyword}, {v=^.*a.*$}]", sc.script().params().toString()); } - public void testRLikeConstructsNotSupported() { + public void testRLikeWithScalars() { LogicalPlan p = plan("SELECT LTRIM(keyword) lt FROM test WHERE LTRIM(keyword) RLIKE '.*a.*'"); assertTrue(p instanceof Project); p = ((Project) p).child(); assertTrue(p instanceof Filter); Expression condition = ((Filter) p).condition(); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> translate(condition)); - assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) RLIKE '.*a.*'", ex.getMessage()); + QueryTranslation qt = translate(condition); + assertTrue(qt.query instanceof ScriptQuery); + ScriptQuery sc = (ScriptQuery) qt.query; + assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(" + + "InternalSqlScriptUtils.ltrim(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", + sc.script().toString()); + assertEquals("[{v=keyword}, {v=.*a.*}]", sc.script().params().toString()); } public void testDifferentLikeAndNotLikePatterns() {