From 977302e46c0be754206f5ae524340e578720fd3d Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 31 Mar 2020 18:06:03 +0300 Subject: [PATCH] EQL: startsWith and endsWith functions implementation (#54504) * EQL: startsWith function implementation (#54400) (cherry picked from commit 666719fcfc40f6fc0535609577791369123320ab) * EQL: endsWith function implementation (#54442) (cherry picked from commit 554a4c8ef04b67eed107d29b57185e9af25d9d4f) --- .../resources/test_queries_unsupported.toml | 38 ------ .../function/EqlFunctionRegistry.java | 4 + .../function/scalar/string/EndsWith.java | 120 ++++++++++++++++++ .../scalar/string/EndsWithFunctionPipe.java | 104 +++++++++++++++ .../string/EndsWithFunctionProcessor.java | 95 ++++++++++++++ .../function/scalar/string/StartsWith.java | 120 ++++++++++++++++++ .../scalar/string/StartsWithFunctionPipe.java | 104 +++++++++++++++ .../string/StartsWithFunctionProcessor.java | 95 ++++++++++++++ .../whitelist/InternalEqlScriptUtils.java | 10 ++ .../xpack/eql/plugin/eql_whitelist.txt | 2 + .../xpack/eql/analysis/VerifierTests.java | 4 - .../scalar/string/EndsWithProcessorTests.java | 54 ++++++++ .../string/StartsWithProcessorTests.java | 54 ++++++++ .../eql/planner/QueryFolderFailTests.java | 16 +++ .../src/test/resources/queryfolder_tests.txt | 14 ++ .../expression/function/FunctionRegistry.java | 4 +- 16 files changed, 794 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f9a869f9e25..f45e477e320 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -786,44 +786,6 @@ process where original_file_name == process_name expected_event_ids = [97, 98, 75273, 75303] description = "check that case insensitive comparisons are performed for fields." -[[queries]] -query = ''' -file where opcode=0 and startsWith(file_name, 'exploRER.') -''' -expected_event_ids = [88, 92] -description = "check built-in string functions" - -[[queries]] -query = ''' -file where opcode=0 and startsWith(file_name, 'expLORER.exe') -''' -expected_event_ids = [88, 92] -description = "check built-in string functions" - -[[queries]] -query = ''' -file where opcode=0 and endsWith(file_name, 'loREr.exe')''' -expected_event_ids = [88] -description = "check built-in string functions" - -[[queries]] -query = ''' -file where opcode=0 and startsWith(file_name, 'explORER.EXE')''' -expected_event_ids = [88, 92] -description = "check built-in string functions" - -[[queries]] -query = ''' -file where opcode=0 and startsWith('explorer.exeaaaaaaaa', file_name)''' -expected_event_ids = [88] -description = "check built-in string functions" - -[[queries]] -query = ''' -file where opcode=0 and serial_event_id = 88 and startsWith('explorer.exeaAAAA', 'EXPLORER.exe')''' -expected_event_ids = [88] -description = "check built-in string functions" - [[queries]] query = ''' file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 4d7b319cc66..ec297a9ad2b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,7 +6,9 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -24,7 +26,9 @@ public class EqlFunctionRegistry extends FunctionRegistry { // Scalar functions // String new FunctionDefinition[] { + def(EndsWith.class, EndsWith::new, "endswith"), def(Length.class, Length::new, "length"), + def(StartsWith.class, StartsWith::new, "startswith"), def(Substring.class, Substring::new, "substring") } }; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java new file mode 100644 index 00000000000..4eec4c39426 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor.doProcess; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; + +/** + * Function that checks if first parameter ends with the second parameter. Both parameters should be strings + * and the function returns a boolean value. The function is case insensitive. + */ +public class EndsWith extends ScalarFunction { + + private final Expression source; + private final Expression pattern; + + public EndsWith(Source source, Expression src, Expression pattern) { + super(source, Arrays.asList(src, pattern)); + this.source = src; + this.pattern = pattern; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution sourceResolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST); + if (sourceResolution.unresolved()) { + return sourceResolution; + } + + return isStringAndExact(pattern, sourceText(), ParamOrdinal.SECOND); + } + + @Override + protected Pipe makePipe() { + return new EndsWithFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(pattern)); + } + + @Override + public boolean foldable() { + return source.foldable() && pattern.foldable(); + } + + @Override + public Object fold() { + return doProcess(source.fold(), pattern.fold()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, EndsWith::new, source, pattern); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate sourceScript = asScript(source); + ScriptTemplate patternScript = asScript(pattern); + + return asScriptFrom(sourceScript, patternScript); + } + + protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate patternScript) { + return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s)"), + "endsWith", + sourceScript.template(), + patternScript.template()), + paramsBuilder() + .script(sourceScript.params()) + .script(patternScript.params()) + .build(), dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript(Scripts.DOC_VALUE), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 2) { + throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]"); + } + + return new EndsWith(source(), newChildren.get(0), newChildren.get(1)); + } + +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionPipe.java new file mode 100644 index 00000000000..d69de471cee --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionPipe.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class EndsWithFunctionPipe extends Pipe { + + private final Pipe source; + private final Pipe pattern; + + public EndsWithFunctionPipe(Source source, Expression expression, Pipe src, Pipe pattern) { + super(source, expression, Arrays.asList(src, pattern)); + this.source = src; + this.pattern = pattern; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() != 2) { + throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0), newChildren.get(1)); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + Pipe newPattern = pattern.resolveAttributes(resolver); + if (newSource == source && newPattern == pattern) { + return this; + } + return replaceChildren(newSource, newPattern); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + return source.supportedByAggsOnlyQuery() && pattern.supportedByAggsOnlyQuery(); + } + + @Override + public boolean resolved() { + return source.resolved() && pattern.resolved(); + } + + protected Pipe replaceChildren(Pipe newSource, Pipe newPattern) { + return new EndsWithFunctionPipe(source(), expression(), newSource, newPattern); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + pattern.collectFields(sourceBuilder); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, EndsWithFunctionPipe::new, expression(), source, pattern); + } + + @Override + public EndsWithFunctionProcessor asProcessor() { + return new EndsWithFunctionProcessor(source.asProcessor(), pattern.asProcessor()); + } + + public Pipe src() { + return source; + } + + public Pipe pattern() { + return pattern; + } + + @Override + public int hashCode() { + return Objects.hash(source, pattern); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + EndsWithFunctionPipe other = (EndsWithFunctionPipe) obj; + return Objects.equals(source, other.source) + && Objects.equals(pattern, other.pattern); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionProcessor.java new file mode 100644 index 00000000000..b9896c35ced --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionProcessor.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class EndsWithFunctionProcessor implements Processor { + + public static final String NAME = "senw"; + + private final Processor source; + private final Processor pattern; + + public EndsWithFunctionProcessor(Processor source, Processor pattern) { + this.source = source; + this.pattern = pattern; + } + + public EndsWithFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + pattern = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + out.writeNamedWriteable(pattern); + } + + @Override + public Object process(Object input) { + return doProcess(source.process(input), pattern.process(input)); + } + + public static Object doProcess(Object source, Object pattern) { + if (source == null) { + return null; + } + if (source instanceof String == false && source instanceof Character == false) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source); + } + if (pattern == null) { + return null; + } + if (pattern instanceof String == false && pattern instanceof Character == false) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", pattern); + } + + return source.toString().toLowerCase(Locale.ROOT).endsWith(pattern.toString().toLowerCase(Locale.ROOT)); + } + + protected Processor source() { + return source; + } + + protected Processor pattern() { + return pattern; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + EndsWithFunctionProcessor other = (EndsWithFunctionProcessor) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(pattern(), other.pattern()); + } + + @Override + public int hashCode() { + return Objects.hash(source(), pattern()); + } + + + @Override + public String getWriteableName() { + return NAME; + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java new file mode 100644 index 00000000000..85c48da03d6 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor.doProcess; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; + +/** + * Function that checks if first parameter starts with the second parameter. Both parameters should be strings + * and the function returns a boolean value. The function is case insensitive. + */ +public class StartsWith extends ScalarFunction { + + private final Expression source; + private final Expression pattern; + + public StartsWith(Source source, Expression src, Expression pattern) { + super(source, Arrays.asList(src, pattern)); + this.source = src; + this.pattern = pattern; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution sourceResolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST); + if (sourceResolution.unresolved()) { + return sourceResolution; + } + + return isStringAndExact(pattern, sourceText(), ParamOrdinal.SECOND); + } + + @Override + protected Pipe makePipe() { + return new StartsWithFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(pattern)); + } + + @Override + public boolean foldable() { + return source.foldable() && pattern.foldable(); + } + + @Override + public Object fold() { + return doProcess(source.fold(), pattern.fold()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StartsWith::new, source, pattern); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate sourceScript = asScript(source); + ScriptTemplate patternScript = asScript(pattern); + + return asScriptFrom(sourceScript, patternScript); + } + + protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate patternScript) { + return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s)"), + "startsWith", + sourceScript.template(), + patternScript.template()), + paramsBuilder() + .script(sourceScript.params()) + .script(patternScript.params()) + .build(), dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript(Scripts.DOC_VALUE), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 2) { + throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]"); + } + + return new StartsWith(source(), newChildren.get(0), newChildren.get(1)); + } + +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java new file mode 100644 index 00000000000..e0aab5b472a --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class StartsWithFunctionPipe extends Pipe { + + private final Pipe source; + private final Pipe pattern; + + public StartsWithFunctionPipe(Source source, Expression expression, Pipe src, Pipe pattern) { + super(source, expression, Arrays.asList(src, pattern)); + this.source = src; + this.pattern = pattern; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() != 2) { + throw new IllegalArgumentException("expected [2] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0), newChildren.get(1)); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + Pipe newPattern = pattern.resolveAttributes(resolver); + if (newSource == source && newPattern == pattern) { + return this; + } + return replaceChildren(newSource, newPattern); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + return source.supportedByAggsOnlyQuery() && pattern.supportedByAggsOnlyQuery(); + } + + @Override + public boolean resolved() { + return source.resolved() && pattern.resolved(); + } + + protected Pipe replaceChildren(Pipe newSource, Pipe newPattern) { + return new StartsWithFunctionPipe(source(), expression(), newSource, newPattern); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + pattern.collectFields(sourceBuilder); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StartsWithFunctionPipe::new, expression(), source, pattern); + } + + @Override + public StartsWithFunctionProcessor asProcessor() { + return new StartsWithFunctionProcessor(source.asProcessor(), pattern.asProcessor()); + } + + public Pipe src() { + return source; + } + + public Pipe pattern() { + return pattern; + } + + @Override + public int hashCode() { + return Objects.hash(source, pattern); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + StartsWithFunctionPipe other = (StartsWithFunctionPipe) obj; + return Objects.equals(source, other.source) + && Objects.equals(pattern, other.pattern); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java new file mode 100644 index 00000000000..8792abab8b8 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class StartsWithFunctionProcessor implements Processor { + + public static final String NAME = "sstw"; + + private final Processor source; + private final Processor pattern; + + public StartsWithFunctionProcessor(Processor source, Processor pattern) { + this.source = source; + this.pattern = pattern; + } + + public StartsWithFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + pattern = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + out.writeNamedWriteable(pattern); + } + + @Override + public Object process(Object input) { + return doProcess(source.process(input), pattern.process(input)); + } + + public static Object doProcess(Object source, Object pattern) { + if (source == null) { + return null; + } + if (source instanceof String == false && source instanceof Character == false) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source); + } + if (pattern == null) { + return null; + } + if (pattern instanceof String == false && pattern instanceof Character == false) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", pattern); + } + + return source.toString().toLowerCase(Locale.ROOT).startsWith(pattern.toString().toLowerCase(Locale.ROOT)); + } + + protected Processor source() { + return source; + } + + protected Processor pattern() { + return pattern; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + StartsWithFunctionProcessor other = (StartsWithFunctionProcessor) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(pattern(), other.pattern()); + } + + @Override + public int hashCode() { + return Objects.hash(source(), pattern()); + } + + + @Override + public String getWriteableName() { + return NAME; + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java index a6b950074f9..67ff257f9b4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -6,7 +6,9 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor; import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils; @@ -19,10 +21,18 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils { InternalEqlScriptUtils() {} + public static Boolean endsWith(String s, String pattern) { + return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern); + } + public static Integer length(String s) { return (Integer) LengthFunctionProcessor.doProcess(s); } + public static Boolean startsWith(String s, String pattern) { + return (Boolean) StartsWithFunctionProcessor.doProcess(s, pattern); + } + public static String substring(String s, Number start, Number end) { return (String) SubstringFunctionProcessor.doProcess(s, start, end); } diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index fc61e1fe2b5..8c6e8c7d1f0 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -55,6 +55,8 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE # # ASCII Functions # + Boolean endsWith(String, String) Integer length(String) + Boolean startsWith(String, String) String substring(String, Number, Number) } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 03125afc05b..4c97191d6f8 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -131,10 +131,6 @@ public class VerifierTests extends ESTestCase { // Test the known EQL functions that are not supported public void testFunctionVerificationUnknown() { - assertEquals("1:25: Unknown function [endsWith]", - error("file where opcode=0 and endsWith(file_name, 'loREr.exe')")); - assertEquals("1:25: Unknown function [startsWith]", - error("file where opcode=0 and startsWith(file_name, 'explORER.EXE')")); assertEquals("1:25: Unknown function [stringContains]", error("file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name)")); assertEquals("1:25: Unknown function [indexOf]", diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithProcessorTests.java new file mode 100644 index 00000000000..0f6b4c15a16 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithProcessorTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.LiteralTests; + +import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l; +import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.hamcrest.Matchers.startsWith; + +public class EndsWithProcessorTests extends ESTestCase { + + public void testStartsWithFunctionWithValidInput() { + assertEquals(true, new EndsWith(EMPTY, l("foobarbar"), l("r")).makePipe().asProcessor().process(null)); + assertEquals(false, new EndsWith(EMPTY, l("foobar"), l("foo")).makePipe().asProcessor().process(null)); + assertEquals(false, new EndsWith(EMPTY, l("foo"), l("foobar")).makePipe().asProcessor().process(null)); + assertEquals(true, new EndsWith(EMPTY, l("foobar"), l("")).makePipe().asProcessor().process(null)); + assertEquals(true, new EndsWith(EMPTY, l("foo"), l("foo")).makePipe().asProcessor().process(null)); + assertEquals(true, new EndsWith(EMPTY, l("foo"), l("oO")).makePipe().asProcessor().process(null)); + assertEquals(true, new EndsWith(EMPTY, l("foo"), l("FOo")).makePipe().asProcessor().process(null)); + assertEquals(true, new EndsWith(EMPTY, l('f'), l('f')).makePipe().asProcessor().process(null)); + assertEquals(false, new EndsWith(EMPTY, l(""), l("bar")).makePipe().asProcessor().process(null)); + assertEquals(null, new EndsWith(EMPTY, l(null), l("bar")).makePipe().asProcessor().process(null)); + assertEquals(null, new EndsWith(EMPTY, l("foo"), l(null)).makePipe().asProcessor().process(null)); + assertEquals(null, new EndsWith(EMPTY, l(null), l(null)).makePipe().asProcessor().process(null)); + } + + public void testStartsWithFunctionInputsValidation() { + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new EndsWith(EMPTY, l(5), l("foo")).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [5]", siae.getMessage()); + siae = expectThrows(QlIllegalArgumentException.class, + () -> new EndsWith(EMPTY, l("bar"), l(false)).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [false]", siae.getMessage()); + } + + public void testStartsWithFunctionWithRandomInvalidDataType() { + Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral()); + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new EndsWith(EMPTY, literal, l("foo")).makePipe().asProcessor().process(null)); + assertThat(siae.getMessage(), startsWith("A string/char is required; received")); + siae = expectThrows(QlIllegalArgumentException.class, + () -> new EndsWith(EMPTY, l("foo"), literal).makePipe().asProcessor().process(null)); + assertThat(siae.getMessage(), startsWith("A string/char is required; received")); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java new file mode 100644 index 00000000000..6c9e6a8277f --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.LiteralTests; + +import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l; +import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.hamcrest.Matchers.startsWith; + +public class StartsWithProcessorTests extends ESTestCase { + + public void testStartsWithFunctionWithValidInput() { + assertEquals(true, new StartsWith(EMPTY, l("foobarbar"), l("f")).makePipe().asProcessor().process(null)); + assertEquals(false, new StartsWith(EMPTY, l("foobar"), l("bar")).makePipe().asProcessor().process(null)); + assertEquals(false, new StartsWith(EMPTY, l("foo"), l("foobar")).makePipe().asProcessor().process(null)); + assertEquals(true, new StartsWith(EMPTY, l("foobar"), l("")).makePipe().asProcessor().process(null)); + assertEquals(true, new StartsWith(EMPTY, l("foo"), l("foo")).makePipe().asProcessor().process(null)); + assertEquals(true, new StartsWith(EMPTY, l("foo"), l("FO")).makePipe().asProcessor().process(null)); + assertEquals(true, new StartsWith(EMPTY, l("foo"), l("FOo")).makePipe().asProcessor().process(null)); + assertEquals(true, new StartsWith(EMPTY, l('f'), l('f')).makePipe().asProcessor().process(null)); + assertEquals(false, new StartsWith(EMPTY, l(""), l("bar")).makePipe().asProcessor().process(null)); + assertEquals(null, new StartsWith(EMPTY, l(null), l("bar")).makePipe().asProcessor().process(null)); + assertEquals(null, new StartsWith(EMPTY, l("foo"), l(null)).makePipe().asProcessor().process(null)); + assertEquals(null, new StartsWith(EMPTY, l(null), l(null)).makePipe().asProcessor().process(null)); + } + + public void testStartsWithFunctionInputsValidation() { + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new StartsWith(EMPTY, l(5), l("foo")).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [5]", siae.getMessage()); + siae = expectThrows(QlIllegalArgumentException.class, + () -> new StartsWith(EMPTY, l("bar"), l(false)).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [false]", siae.getMessage()); + } + + public void testStartsWithFunctionWithRandomInvalidDataType() { + Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral()); + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new StartsWith(EMPTY, literal, l("foo")).makePipe().asProcessor().process(null)); + assertThat(siae.getMessage(), startsWith("A string/char is required; received")); + siae = expectThrows(QlIllegalArgumentException.class, + () -> new StartsWith(EMPTY, l("foo"), literal).makePipe().asProcessor().process(null)); + assertThat(siae.getMessage(), startsWith("A string/char is required; received")); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 9029a31715e..77209b7c67c 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -32,4 +32,20 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field " + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } + + public void testEndsWithFunctionWithInexact() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where endsWith(plain_text, \"foo\") == true")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:15: [endsWith(plain_text, \"foo\")] cannot operate on first argument field of data type " + + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + } + + public void testStartsWithFunctionWithInexact() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where startsWith(plain_text, \"foo\") == true")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type " + + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + } } diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index a509b41415f..5b8e15b201d 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -61,6 +61,13 @@ process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) {"terms":{"opcode":[0,1,2,3] +endsWithFunction +process where endsWith(user_name, 'c') +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.endsWith( +InternalQlScriptUtils.docValue(doc,params.v0),params.v1))", +"params":{"v0":"user_name","v1":"c"} + + lengthFunctionWithExactSubField process where length(file_name) > 0 "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( @@ -82,6 +89,13 @@ InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),par "params":{"v0":"constant_keyword","v1":5} +startsWithFunction +process where startsWith(user_name, 'A') +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.startsWith( +InternalQlScriptUtils.docValue(doc,params.v0),params.v1))", +"params":{"v0":"user_name","v1":"A"} + + substringFunction process where substring(file_name, -4) == '.exe' "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 6b90a59289f..0d1a36edb05 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -91,8 +91,8 @@ public class FunctionRegistry { } public String resolveAlias(String alias) { - String upperCase = normalize(alias); - return aliases.getOrDefault(upperCase, upperCase); + String normalized = normalize(alias); + return aliases.getOrDefault(normalized, normalized); } public boolean functionExists(String functionName) {