SQL/EQL: Add support for scalars within LIKE/RLIKE (#56495) (#56674)

- Add support for scalar functions on the field of SQL's LIKE/RLIKE
- Add support for scalar functions on the field of EQL's match/matchLite

Closes: #55058
(cherry picked from commit 51c14e2dbb7fb29004a23369c449d425b3ac8fe2)

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -82,6 +82,12 @@ query = '''
 process where match(command_line, '.*?net[1]?  localgroup.*?', '.*? myappserver.py .*?')
 '''
 
+[[queries]]
+expected_event_ids  = [50, 98]
+query = '''
+process where match(substring(command_line, 5), '.*?net[1]?  localgroup.*?', '.*? myappserver.py .*?')
+'''
+
 [[queries]]
 # Basic test for modulo function
 query = '''

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

@@ -22,7 +22,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList;
 
 public class Analyzer extends RuleExecutor<LogicalPlan> {
@@ -43,7 +43,7 @@ public class Analyzer extends RuleExecutor<LogicalPlan> {
                 new ResolveRefs(),
                 new ResolveFunctions());
 
-        return asList(resolution);
+        return singletonList(resolution);
     }
 
     public LogicalPlan analyze(LogicalPlan plan) {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Match.java

@@ -36,9 +36,13 @@ public class Match extends BaseSurrogateFunction {
     private final List<Expression> patterns;
 
     public Match(Source source, Expression field, List<Expression> patterns) {
-        super(source, CollectionUtils.combine(singletonList(field), patterns));
+        this(source, CollectionUtils.combine(singletonList(field), patterns));
-        this.field = field;
+    }
-        this.patterns = patterns;
+
+    private Match(Source source, List<Expression> children) {
+        super(source, children);
+        this.field = children().get(0);
+        this.patterns = children().subList(1, children().size());
     }
 
     @Override
@@ -51,7 +55,7 @@ public class Match extends BaseSurrogateFunction {
         if (newChildren.size() < 2) {
             throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
         }
-        return new Match(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
+        return new Match(source(), newChildren);
     }
 
     @Override

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -257,6 +257,14 @@ process where match(command_line, "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\
 "regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"
 ;
 
+matchFunctionScalar
+process where match(substring(command_line, 5), "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+")
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(InternalEqlScriptUtils.substring(
+InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
+"params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"}}
+;
+
 wildcardFunctionSingleArgument
 process where wildcard(process_path, "*\\red_ttp\\wininit.*")
 ;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java

@@ -121,11 +121,6 @@ public final class ExpressionTranslators {
 
             if (e.field() instanceof FieldAttribute) {
                 targetFieldName = handler.nameOf(((FieldAttribute) e.field()).exactAttribute());
-            } else {
-                throw new QlIllegalArgumentException("Scalar function [{}] not allowed (yet) as argument for " + e.sourceText(),
-                        Expressions.name(e.field()));
-            }
-
                 if (e instanceof Like) {
                     LikePattern p = ((Like) e).pattern();
                     q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard());
@@ -135,6 +130,9 @@ public final class ExpressionTranslators {
                     String pattern = ((RLike) e).pattern().asJavaRegex();
                     q = new RegexQuery(e.source(), targetFieldName, pattern);
                 }
+            } else {
+                q = new ScriptQuery(e.source(), e.asScript());
+            }
 
             return wrapIfNested(q, e.field());
         }

x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec

@@ -12,6 +12,16 @@ SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name RLIKE 'S.*
 Simmel
 ;
 
+whereFieldWithRLikeWithScalarsMatch
+SELECT LTRIM(concat('         ', first_name)) lt FROM test_emp WHERE LTRIM(concat('         ', first_name)) RLIKE '.*z' ORDER BY emp_no;
+
+      lt
+---------------
+Erez
+Parviz
+;
+
+
 whereFieldWithNotRLikeMatch
 SELECT last_name, first_name FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT RLIKE 'Ma.*' ORDER BY first_name LIMIT 5;
 

x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec

@@ -51,6 +51,8 @@ whereFieldWithNotEqualsOnString
 SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND gender <> 'M';
 whereFieldWithLikeMatch
 SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name LIKE 'K%';
+whereFieldWithLikeAndScalarsMatch
+SELECT RTRIM(CONCAT(last_name, '     ')) AS l FROM "test_emp" WHERE RTRIM(CONCAT(last_name, '     ')) LIKE '%k%' ORDER BY emp_no;
 whereFieldWithNotLikeMatch
 SELECT last_name l FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT LIKE 'Ma%';
 whereFieldWithInlineLikeMatch

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java

@@ -577,24 +577,34 @@ public class QueryTranslatorTests extends ESTestCase {
         assertEquals("some.string.typical", qsq.field());
     }
 
-    public void testLikeConstructsNotSupported() {
+    public void testLikeWithScalars() {
         LogicalPlan p = plan("SELECT LTRIM(keyword) lt FROM test WHERE LTRIM(keyword) like '%a%'");
         assertTrue(p instanceof Project);
         p = ((Project) p).child();
         assertTrue(p instanceof Filter);
         Expression condition = ((Filter) p).condition();
-        QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> translate(condition));
+        QueryTranslation qt = translate(condition);
-        assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) like '%a%'", ex.getMessage());
+        assertTrue(qt.query instanceof ScriptQuery);
+        ScriptQuery sc = (ScriptQuery) qt.query;
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(" +
+                "InternalSqlScriptUtils.ltrim(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+            sc.script().toString());
+        assertEquals("[{v=keyword}, {v=^.*a.*$}]", sc.script().params().toString());
     }
 
-    public void testRLikeConstructsNotSupported() {
+    public void testRLikeWithScalars() {
         LogicalPlan p = plan("SELECT LTRIM(keyword) lt FROM test WHERE LTRIM(keyword) RLIKE '.*a.*'");
         assertTrue(p instanceof Project);
         p = ((Project) p).child();
         assertTrue(p instanceof Filter);
         Expression condition = ((Filter) p).condition();
-        QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> translate(condition));
+        QueryTranslation qt = translate(condition);
-        assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) RLIKE '.*a.*'", ex.getMessage());
+        assertTrue(qt.query instanceof ScriptQuery);
+        ScriptQuery sc = (ScriptQuery) qt.query;
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(" +
+                "InternalSqlScriptUtils.ltrim(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+            sc.script().toString());
+        assertEquals("[{v=keyword}, {v=.*a.*}]", sc.script().params().toString());
     }
 
     public void testDifferentLikeAndNotLikePatterns() {