EQL: indexOf function implementation (#54543) (#54989)

(cherry picked from commit a4b1d6e52d9ba22d541dd86d69861b1efee83604)
This commit is contained in:
Andrei Stefan 2020-04-09 02:41:01 +03:00 committed by GitHub
parent 1f17df13c1
commit 85f129a50a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 450 additions and 20 deletions

View File

@ -807,12 +807,6 @@ file where opcode=0 and indexOf(file_name, 'explorer.') and indexOf(file_name, '
expected_event_ids = []
description = "check built-in string functions"
[[queries]]
query = '''
file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2'''
expected_event_ids = [88, 92]
description = "check built-in string functions"
[[queries]]
query = '''
file where opcode=0 and indexOf(file_name, 'plorer.', 2)'''
@ -831,18 +825,6 @@ file where opcode=0 and indexOf(file_name, 'thing that never happened')'''
expected_event_ids = []
description = "check built-in string functions"
[[queries]]
query = '''
file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2'''
expected_event_ids = [88, 92]
description = "check substring ranges"
[[queries]]
query = '''
file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0'''
expected_event_ids = [88, 92]
description = "check substring ranges"
[[queries]]
query = '''
process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id'''

View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.eql.expression.function;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
@ -33,6 +34,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
def(Between.class, Between::new, 2, "between"),
def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
def(EndsWith.class, EndsWith::new, "endswith"),
def(IndexOf.class, IndexOf::new, "indexof"),
def(Length.class, Length::new, "length"),
def(StartsWith.class, StartsWith::new, "startswith"),
def(StringContains.class, StringContains::new, "stringcontains"),

View File

@ -0,0 +1,131 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.Literal;
import org.elasticsearch.xpack.ql.expression.function.OptionalArgument;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import static java.lang.String.format;
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor.doProcess;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
/**
* Find the first position (zero-indexed) of a string where a substring is found.
* If the optional parameter start is provided, then this will find the first occurrence at or after the start position.
*/
public class IndexOf extends ScalarFunction implements OptionalArgument {
private final Expression source, substring, start;
public IndexOf(Source source, Expression src, Expression substring, Expression start) {
super(source, Arrays.asList(src, substring, start != null ? start : new Literal(source, null, DataTypes.NULL)));
this.source = src;
this.substring = substring;
this.start = arguments().get(2);
}
@Override
protected TypeResolution resolveType() {
if (!childrenResolved()) {
return new TypeResolution("Unresolved children");
}
TypeResolution resolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST);
if (resolution.unresolved()) {
return resolution;
}
resolution = isStringAndExact(substring, sourceText(), ParamOrdinal.SECOND);
if (resolution.unresolved()) {
return resolution;
}
return isInteger(start, sourceText(), ParamOrdinal.THIRD);
}
@Override
protected Pipe makePipe() {
return new IndexOfFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(substring), Expressions.pipe(start));
}
@Override
public boolean foldable() {
return source.foldable() && substring.foldable() && start.foldable();
}
@Override
public Object fold() {
return doProcess(source.fold(), substring.fold(), start.fold());
}
@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, IndexOf::new, source, substring, start);
}
@Override
public ScriptTemplate asScript() {
ScriptTemplate sourceScript = asScript(source);
ScriptTemplate substringScript = asScript(substring);
ScriptTemplate startScript = asScript(start);
return asScriptFrom(sourceScript, substringScript, startScript);
}
protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate substringScript, ScriptTemplate startScript) {
return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s,%s)"),
"indexOf",
sourceScript.template(),
substringScript.template(),
startScript.template()),
paramsBuilder()
.script(sourceScript.params())
.script(substringScript.params())
.script(startScript.params())
.build(), dataType());
}
@Override
public ScriptTemplate scriptWithField(FieldAttribute field) {
return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
paramsBuilder().variable(field.exactAttribute().name()).build(),
dataType());
}
@Override
public DataType dataType() {
return DataTypes.INTEGER;
}
@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() != 3) {
throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
}
return new IndexOf(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
}
}

View File

@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class IndexOfFunctionPipe extends Pipe {
private final Pipe source, substring, start;
public IndexOfFunctionPipe(Source source, Expression expression, Pipe src, Pipe substring, Pipe start) {
super(source, expression, Arrays.asList(src, substring, start));
this.source = src;
this.substring = substring;
this.start = start;
}
@Override
public final Pipe replaceChildren(List<Pipe> newChildren) {
if (newChildren.size() != 3) {
throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
}
return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2));
}
@Override
public final Pipe resolveAttributes(AttributeResolver resolver) {
Pipe newSource = source.resolveAttributes(resolver);
Pipe newSubstring = substring.resolveAttributes(resolver);
Pipe newStart = start.resolveAttributes(resolver);
if (newSource == source && newSubstring == substring && newStart == start) {
return this;
}
return replaceChildren(newSource, newSubstring, newStart);
}
@Override
public boolean supportedByAggsOnlyQuery() {
return source.supportedByAggsOnlyQuery() && substring.supportedByAggsOnlyQuery() && start.supportedByAggsOnlyQuery();
}
@Override
public boolean resolved() {
return source.resolved() && substring.resolved() && start.resolved();
}
protected Pipe replaceChildren(Pipe newSource, Pipe newSubstring, Pipe newStart) {
return new IndexOfFunctionPipe(source(), expression(), newSource, newSubstring, newStart);
}
@Override
public final void collectFields(QlSourceBuilder sourceBuilder) {
source.collectFields(sourceBuilder);
substring.collectFields(sourceBuilder);
start.collectFields(sourceBuilder);
}
@Override
protected NodeInfo<IndexOfFunctionPipe> info() {
return NodeInfo.create(this, IndexOfFunctionPipe::new, expression(), source, substring, start);
}
@Override
public IndexOfFunctionProcessor asProcessor() {
return new IndexOfFunctionProcessor(source.asProcessor(), substring.asProcessor(), start.asProcessor());
}
public Pipe src() {
return source;
}
public Pipe substring() {
return substring;
}
public Pipe start() {
return start;
}
@Override
public int hashCode() {
return Objects.hash(source, substring, start);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
IndexOfFunctionPipe other = (IndexOfFunctionPipe) obj;
return Objects.equals(source, other.source)
&& Objects.equals(substring, other.substring)
&& Objects.equals(start, other.start);
}
}

View File

@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import java.io.IOException;
import java.util.Locale;
import java.util.Objects;
public class IndexOfFunctionProcessor implements Processor {
public static final String NAME = "siof";
private final Processor source;
private final Processor substring;
private final Processor start;
public IndexOfFunctionProcessor(Processor source, Processor substring, Processor start) {
this.source = source;
this.substring = substring;
this.start = start;
}
public IndexOfFunctionProcessor(StreamInput in) throws IOException {
source = in.readNamedWriteable(Processor.class);
substring = in.readNamedWriteable(Processor.class);
start = in.readNamedWriteable(Processor.class);
}
@Override
public final void writeTo(StreamOutput out) throws IOException {
out.writeNamedWriteable(source);
out.writeNamedWriteable(substring);
out.writeNamedWriteable(start);
}
@Override
public Object process(Object input) {
return doProcess(source.process(input), substring.process(input), start.process(input));
}
public static Object doProcess(Object source, Object substring, Object start) {
if (source == null) {
return null;
}
if (source instanceof String == false && source instanceof Character == false) {
throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source);
}
if (substring == null) {
return null;
}
if (substring instanceof String == false && substring instanceof Character == false) {
throw new EqlIllegalArgumentException("A string/char is required; received [{}]", substring);
}
if (start != null && start instanceof Number == false) {
throw new EqlIllegalArgumentException("A number is required; received [{}]", start);
}
int startIndex = start == null ? 0 : ((Number) start).intValue();
int result = source.toString().toLowerCase(Locale.ROOT).indexOf(substring.toString().toLowerCase(Locale.ROOT), startIndex);
return result < 0 ? null : result;
}
protected Processor source() {
return source;
}
protected Processor substring() {
return substring;
}
protected Processor start() {
return start;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
IndexOfFunctionProcessor other = (IndexOfFunctionProcessor) obj;
return Objects.equals(source(), other.source())
&& Objects.equals(substring(), other.substring())
&& Objects.equals(start(), other.start());
}
@Override
public int hashCode() {
return Objects.hash(source(), substring(), start());
}
@Override
public String getWriteableName() {
return NAME;
}
}

View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
@ -31,6 +32,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern);
}
public static Integer indexOf(String s, String substring, Number start) {
return (Integer) IndexOfFunctionProcessor.doProcess(s, substring, start);
}
public static Integer length(String s) {
return (Integer) LengthFunctionProcessor.doProcess(s);
}

View File

@ -57,6 +57,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
#
String between(String, String, String, Boolean, Boolean)
Boolean endsWith(String, String)
Integer indexOf(String, String, Number)
Integer length(String)
Boolean startsWith(String, String)
Boolean stringContains(String, String)

View File

@ -119,8 +119,6 @@ public class VerifierTests extends ESTestCase {
// Test the known EQL functions that are not supported
public void testFunctionVerificationUnknown() {
assertEquals("1:25: Unknown function [indexOf]",
error("file where opcode=0 and indexOf(file_name, 'plore') == 2"));
assertEquals("1:15: Unknown function [add]",
error("process where add(serial_event_id, 0) == 1"));
assertEquals("1:15: Unknown function [subtract], did you mean [substring]?",

View File

@ -0,0 +1,68 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Literal;
import org.elasticsearch.xpack.ql.expression.LiteralTests;
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
import static org.hamcrest.Matchers.startsWith;
public class IndexOfProcessorTests extends ESTestCase {
public void testStartsWithFunctionWithValidInput() {
assertEquals(5, new IndexOf(EMPTY, l("foobarbar"), l("r"), l(null)).makePipe().asProcessor().process(null));
assertEquals(0, new IndexOf(EMPTY, l("foobar"), l("foo"), l(null)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l("foo"), l("foobar"), l(null)).makePipe().asProcessor().process(null));
assertEquals(0, new IndexOf(EMPTY, l("foo"), l("foo"), l(null)).makePipe().asProcessor().process(null));
assertEquals(1, new IndexOf(EMPTY, l("foo"), l("oO"), l(null)).makePipe().asProcessor().process(null));
assertEquals(0, new IndexOf(EMPTY, l("foo"), l("FOo"), l(null)).makePipe().asProcessor().process(null));
assertEquals(0, new IndexOf(EMPTY, l('f'), l('f'), l(null)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l(""), l("bar"), l(1)).makePipe().asProcessor().process(null));
assertEquals(5, new IndexOf(EMPTY, l("foobarbar"), l("R"), l(5)).makePipe().asProcessor().process(null));
assertEquals(2, new IndexOf(EMPTY, l("foobar"), l("O"), l(2)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l("foobar"), l("O"), l(3)).makePipe().asProcessor().process(null));
assertEquals(6, new IndexOf(EMPTY, l("foobarbaz"), l("ba"), l(4)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l(null), l("bar"), l(2)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l(null), l("bar"), l(2)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l("foo"), l(null), l(3)).makePipe().asProcessor().process(null));
assertEquals(null, new IndexOf(EMPTY, l(null), l(null), l(4)).makePipe().asProcessor().process(null));
assertEquals(0, new IndexOf(EMPTY, l("bar"), l("bar"), l(null)).makePipe().asProcessor().process(null));
}
public void testStartsWithFunctionInputsValidation() {
QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, l(5), l("foo"), l(null)).makePipe().asProcessor().process(null));
assertEquals("A string/char is required; received [5]", siae.getMessage());
siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, l("bar"), l(false), l(2)).makePipe().asProcessor().process(null));
assertEquals("A string/char is required; received [false]", siae.getMessage());
siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, l("bar"), l("a"), l("1")).makePipe().asProcessor().process(null));
assertEquals("A number is required; received [1]", siae.getMessage());
}
public void testStartsWithFunctionWithRandomInvalidDataType() {
Literal stringLiteral = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral());
QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, stringLiteral, l("foo"), l(1)).makePipe().asProcessor().process(null));
assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, l("foo"), stringLiteral, l(2)).makePipe().asProcessor().process(null));
assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
Literal numericLiteral = randomValueOtherThanMany(v -> v.dataType().isNumeric(), () -> LiteralTests.randomLiteral());
siae = expectThrows(QlIllegalArgumentException.class,
() -> new IndexOf(EMPTY, l("foo"), l("o"), numericLiteral).makePipe().asProcessor().process(null));
assertThat(siae.getMessage(), startsWith("A number is required; received"));
}
}

View File

@ -132,6 +132,20 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testIndexOfFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where indexOf(plain_text, \"foo\") == 1"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [indexOf(plain_text, \"foo\")] cannot operate on first argument field of data type "
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
e = expectThrows(VerificationException.class,
() -> plan("process where indexOf(\"bla\", plain_text) == 1"));
msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [indexOf(\"bla\", plain_text)] cannot operate on second argument field of data type "
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testStringContainsWrongParams() {
assertEquals("1:16: error building [stringcontains]: expects exactly two arguments",
errorParsing("process where stringContains()"));

View File

@ -113,6 +113,14 @@ InternalQlScriptUtils.docValue(doc,params.v0),params.v1))"
"params":{"v0":"process_name","v1":"foo"}
;
indexOfFunction
process where indexOf(user_name, 'A', 2) > 0
;
"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(
InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
"params":{"v0":"user_name","v1":"A","v2":2,"v3":0}
;
substringFunction
process where substring(file_name, -4) == '.exe'
;