diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java index 7029c469d2f..463ec35e88f 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java @@ -31,19 +31,25 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase { public static List readScriptSpec() throws Exception { Parser parser = specParser(); List tests = new ArrayList<>(); - tests.addAll(readScriptSpec("/select.csv-spec", parser)); - tests.addAll(readScriptSpec("/command.csv-spec", parser)); - tests.addAll(readScriptSpec("/fulltext.csv-spec", parser)); tests.addAll(readScriptSpec("/agg.csv-spec", parser)); + tests.addAll(readScriptSpec("/alias.csv-spec", parser)); + tests.addAll(readScriptSpec("/arithmetic.csv-spec", parser)); tests.addAll(readScriptSpec("/columns.csv-spec", parser)); + tests.addAll(readScriptSpec("/command.csv-spec", parser)); + //tests.addAll(readScriptSpec("/command-sys.csv-spec", parser)); tests.addAll(readScriptSpec("/date.csv-spec", parser)); tests.addAll(readScriptSpec("/datetime.csv-spec", parser)); - tests.addAll(readScriptSpec("/alias.csv-spec", parser)); + tests.addAll(readScriptSpec("/datetime-interval.csv-spec", parser)); + tests.addAll(readScriptSpec("/field-alias.csv-spec", parser)); + tests.addAll(readScriptSpec("/filter.csv-spec", parser)); + tests.addAll(readScriptSpec("/fulltext.csv-spec", parser)); + tests.addAll(readScriptSpec("/functions.csv-spec", parser)); + //tests.addAll(readScriptSpec("/ip.csv-spec", parser)); + tests.addAll(readScriptSpec("/math.csv-spec", parser)); tests.addAll(readScriptSpec("/null.csv-spec", parser)); tests.addAll(readScriptSpec("/nested.csv-spec", parser)); - tests.addAll(readScriptSpec("/functions.csv-spec", parser)); - tests.addAll(readScriptSpec("/math.csv-spec", parser)); - tests.addAll(readScriptSpec("/field-alias.csv-spec", parser)); + tests.addAll(readScriptSpec("/select.csv-spec", parser)); + return tests; } diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec index 97c917ceeb8..41762c37d56 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec @@ -178,10 +178,7 @@ aggCountOnColumnAndMultipleHaving SELECT gender g, COUNT(gender) c FROM "test_emp" GROUP BY g HAVING c > 10 AND c < 70 ORDER BY gender ; aggCountOnColumnAndMultipleHavingEquals SELECT gender g, COUNT(gender) c FROM "test_emp" GROUP BY g HAVING c > 10 AND c = 63 ORDER BY gender ; -// -// Count(column) = Column(*) which is a bug -// https://github.com/elastic/elasticsearch/issues/34549 -// + aggCountOnColumnAndMultipleHavingWithLimit SELECT gender g, COUNT(gender) c FROM "test_emp" GROUP BY g HAVING c > 10 AND c < 70 ORDER BY gender LIMIT 1; aggCountOnColumnAndHavingBetween-Ignore diff --git a/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec index 4d8a9fc3fc2..e117866b5e9 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec @@ -5,9 +5,14 @@ // the standard behavior here is to return the constant for each element // the weird thing is that an actual query needs to be ran arithmeticWithFrom -SELECT 5 - 2 x FROM test_emp; +SELECT 5 - 2 x FROM test_emp LIMIT 5; -x + x:i +--------------- +3 +3 +3 +3 3 ; diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec new file mode 100644 index 00000000000..e60460a63e5 --- /dev/null +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec @@ -0,0 +1,119 @@ +// +// Filter +// + +whereFieldWithRLikeMatch +SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name RLIKE 'S.*'; + + l:s +--------------- +Simmel +; + +whereFieldWithNotRLikeMatch +SELECT last_name, first_name FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT RLIKE 'Ma.*' ORDER BY first_name LIMIT 5; + + last_name:s | first_name:s +---------------+--------------- +Preusig |Anneke +Genin |Berni +Simmel |Bezalel +Koblick |Chirstian +Bouloucos |Cristinel +; + +whereFieldWithRLikeMatchNot +SELECT last_name AS L, emp_no FROM "test_emp" WHERE NOT (emp_no < 10003 AND L NOT RLIKE 'K.*') ORDER BY emp_no LIMIT 5; + + L:s | emp_no:i +---------------+--------------- +Bamford |10003 +Koblick |10004 +Maliniak |10005 +Preusig |10006 +Zielinski |10007 +; + +whereFieldOnMatchWithAndAndOr +SELECT last_name l, gender g FROM "test_emp" WHERE (last_name RLIKE 'K.*' OR gender = 'F') AND emp_no < 10050 ORDER BY last_name; + + l:s | g:s +---------------+--------------- +Casley |F +Kalloufi |M +Koblick |M +Lenart |F +Meriste |F +Montemayor |F +Peac |F +Pettey |F +Preusig |F +Reistad |F +Reistad |F +Simmel |F +Stamatiou |F +Tramer |F +Zielinski |F +; + +whereFieldWithRLikeAndGroupByOrderBy +SELECT last_name l, gender g, COUNT(*) c, MAX(salary) AS sal FROM "test_emp" WHERE emp_no < 10050 AND (last_name RLIKE 'B.*' OR gender = 'F') GROUP BY g, l ORDER BY sal; + + l:s | g:s | c:l | sal:i +---------------+---------------+---------------+--------------- +Berztiss |M |1 |28336 +Stamatiou |F |1 |30404 +Brender |M |1 |36051 +Meriste |F |1 |37112 +Tramer |F |1 |37853 +Casley |F |1 |39728 +Montemayor |F |1 |47896 +Bridgland |null |1 |48942 +Simmel |F |1 |56371 +Lenart |F |1 |56415 +Bouloucos |null |1 |58715 +Preusig |F |1 |60335 +Bamford |M |1 |61805 +Pettey |F |1 |64675 +Peac |F |1 |66174 +Reistad |F |2 |73851 +Zielinski |F |1 |74572 +; + +whereFieldWithRLikeAndNotRLike +SELECT COUNT(*), last_name AS f FROM test_emp WHERE last_name RLIKE '.*o.*' AND last_name NOT RLIKE '.*f.*' GROUP BY f HAVING COUNT(*) > 1; + + COUNT(*):l | f:s +---------------+--------------- +2 |Lortz +; + +whereInlineRLike +SELECT emp_no FROM test_emp WHERE 'aaabbb' RLIKE 'aa+b+' AND 'aaabbb' NOT RLIKE 'a++c+' AND emp_no < 10080 ORDER BY emp_no DESC LIMIT 5; + + emp_no:i +--------------- +10079 +10078 +10077 +10076 +10075 +; + +whereInlineRLikeAndCount_1 +SELECT COUNT(*), TRUNCATE(emp_no, -2) t FROM test_emp WHERE 'aaabbb' RLIKE '.....?.?' AND 'aaabbb' NOT RLIKE 'aa?bb?' GROUP BY TRUNCATE(emp_no, -2) ORDER BY t ASC; + + COUNT(*):l | t:i +---------------+--------------- +99 |10000 +1 |10100 +; + +whereInlineRLikeAndCount_2 +SELECT COUNT(*), TRUNCATE(emp_no, -2) t FROM test_emp WHERE 'aaabbb' RLIKE 'a{2,}b{2,}' AND 'aaabbb' NOT RLIKE 'a{4,6}b{4,6}' GROUP BY TRUNCATE(emp_no, -2) ORDER BY t ASC; + + COUNT(*):l | t:i +---------------+--------------- +99 |10000 +1 |10100 +; diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec index eafc09c053e..d1564726979 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec @@ -51,6 +51,8 @@ whereFieldWithLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name LIKE 'K%'; whereFieldWithNotLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT LIKE 'Ma%'; +whereFieldWithInlineLikeMatch +SELECT emp_no FROM "test_emp" WHERE 'aaabbb' LIKE 'aa%b%' AND 'aaabbb' NOT LIKE 'a%%c%' AND emp_no < 10080 ORDER BY emp_no DESC LIMIT 5; whereFieldWithOrderNot SELECT last_name l FROM "test_emp" WHERE NOT emp_no < 10003 ORDER BY emp_no LIMIT 5; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 8935910df2b..65a9410941b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.sql.expression.function.aggregate.Count; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.ArithmeticOperation; -import org.elasticsearch.xpack.sql.expression.predicate.regex.Like; +import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.sql.plan.TableIdentifier; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; import org.elasticsearch.xpack.sql.plan.logical.EsRelation; @@ -852,8 +852,8 @@ public class Analyzer extends RuleExecutor { // TODO: we should move to always compare the functions directly // Special check for COUNT: an already seen COUNT function will be returned only if its DISTINCT property // matches the one from the unresolved function to be checked. - // Same for LIKE: the equals function also compares the pattern of LIKE - if (seenFunction instanceof Count || seenFunction instanceof Like) { + // Same for LIKE/RLIKE: the equals function also compares the pattern of LIKE/RLIKE + if (seenFunction instanceof Count || seenFunction instanceof RegexMatch) { if (seenFunction.equals(f)){ return seenFunction; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java index 75f43662233..e67f4943445 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java @@ -6,41 +6,35 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.tree.NodeInfo; import org.elasticsearch.xpack.sql.tree.Source; -import java.util.Objects; - -public class Like extends RegexMatch { - - private final LikePattern pattern; +public class Like extends RegexMatch { public Like(Source source, Expression left, LikePattern pattern) { - super(source, left, pattern.asJavaRegex()); - this.pattern = pattern; - } - - public LikePattern pattern() { - return pattern; + super(source, left, pattern); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Like::new, field(), pattern); + return NodeInfo.create(this, Like::new, field(), pattern()); } @Override protected Like replaceChild(Expression newLeft) { - return new Like(source(), newLeft, pattern); + return new Like(source(), newLeft, pattern()); } @Override - public boolean equals(Object obj) { - return super.equals(obj) && Objects.equals(((Like) obj).pattern(), pattern()); + public Boolean fold() { + Object val = field().fold(); + return RegexOperation.match(val, pattern().asJavaRegex()); } @Override - public int hashCode() { - return Objects.hash(super.hashCode(), pattern()); + protected Processor makeProcessor() { + return new RegexProcessor(pattern().asJavaRegex()); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java index b925bd769ea..187eda3fdec 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java @@ -6,29 +6,35 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; -public class RLike extends RegexMatch { +public class RLike extends RegexMatch { - private final String pattern; - - public RLike(Source source, Expression left, String pattern) { - super(source, left, pattern); - this.pattern = pattern; - } - - public String pattern() { - return pattern; + public RLike(Source source, Expression value, String pattern) { + super(source, value, pattern); } @Override protected NodeInfo info() { - return NodeInfo.create(this, RLike::new, field(), pattern); + return NodeInfo.create(this, RLike::new, field(), pattern()); } @Override protected RLike replaceChild(Expression newChild) { - return new RLike(source(), newChild, pattern); + return new RLike(source(), newChild, pattern()); + } + + @Override + public Boolean fold() { + Object val = field().fold(); + return RegexOperation.match(val, pattern()); + } + + @Override + protected Processor makeProcessor() { + return new RegexProcessor(pattern()); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java index b3c09c67fc6..82c6d570f39 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java @@ -10,21 +10,25 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.Nullability; import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; -import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; -import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataType; +import java.util.Objects; + import static org.elasticsearch.xpack.sql.expression.TypeResolutions.isStringAndExact; -public abstract class RegexMatch extends UnaryScalarFunction { +public abstract class RegexMatch extends UnaryScalarFunction { - private final String pattern; - - protected RegexMatch(Source source, Expression value, String pattern) { + private final T pattern; + + protected RegexMatch(Source source, Expression value, T pattern) { super(source, value); this.pattern = pattern; } + + public T pattern() { + return pattern; + } @Override public DataType dataType() { @@ -33,7 +37,7 @@ public abstract class RegexMatch extends UnaryScalarFunction { @Override public Nullability nullable() { - if (pattern == null) { + if (pattern() == null) { return Nullability.TRUE; } return field().nullable(); @@ -49,15 +53,14 @@ public abstract class RegexMatch extends UnaryScalarFunction { // right() is not directly foldable in any context but Like can fold it. return field().foldable(); } - + @Override - public Boolean fold() { - Object val = field().fold(); - return RegexOperation.match(val, pattern); + public boolean equals(Object obj) { + return super.equals(obj) && Objects.equals(((RegexMatch) obj).pattern(), pattern()); } @Override - protected Processor makeProcessor() { - return new RegexProcessor(pattern); + public int hashCode() { + return Objects.hash(super.hashCode(), pattern()); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index 9d86c572c2b..1fdd27d9b0b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -472,6 +472,7 @@ final class QueryTranslator { // TODO: need to optimize on ngram // TODO: see whether escaping is needed + @SuppressWarnings("rawtypes") static class Likes extends ExpressionTranslator { @Override diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 2c31fff1fb9..a39b5466bc1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -228,6 +228,41 @@ public class QueryTranslatorTests extends ESTestCase { assertEquals(1, rqsq.fields().size()); assertEquals("keyword", rqsq.fields().keySet().iterator().next()); } + + public void testRLikePatterns() { + String[] patterns = new String[] {"(...)+", "abab(ab)?", "(ab){1,2}", "(ab){3}", "aabb|bbaa", "a+b+|b+a+", "aa(cc|bb)", + "a{4,6}b{4,6}", ".{3}.{3}", "aaa*bbb*", "a+.+", "a.c.e", "[^abc\\-]"}; + for (int i = 0; i < 5; i++) { + assertDifferentRLikeAndNotRLikePatterns(randomFrom(patterns), randomFrom(patterns)); + } + } + + private void assertDifferentRLikeAndNotRLikePatterns(String firstPattern, String secondPattern) { + LogicalPlan p = plan("SELECT keyword k FROM test WHERE k RLIKE '" + firstPattern + "' AND k NOT RLIKE '" + secondPattern + "'"); + assertTrue(p instanceof Project); + p = ((Project) p).child(); + assertTrue(p instanceof Filter); + + Expression condition = ((Filter) p).condition(); + QueryTranslation qt = QueryTranslator.toQuery(condition, false); + assertEquals(BoolQuery.class, qt.query.getClass()); + BoolQuery bq = ((BoolQuery) qt.query); + assertTrue(bq.isAnd()); + assertTrue(bq.left() instanceof QueryStringQuery); + assertTrue(bq.right() instanceof NotQuery); + + NotQuery nq = (NotQuery) bq.right(); + assertTrue(nq.child() instanceof QueryStringQuery); + QueryStringQuery lqsq = (QueryStringQuery) bq.left(); + QueryStringQuery rqsq = (QueryStringQuery) nq.child(); + + assertEquals("/" + firstPattern + "/", lqsq.query()); + assertEquals(1, lqsq.fields().size()); + assertEquals("keyword", lqsq.fields().keySet().iterator().next()); + assertEquals("/" + secondPattern + "/", rqsq.query()); + assertEquals(1, rqsq.fields().size()); + assertEquals("keyword", rqsq.fields().keySet().iterator().next()); + } public void testTranslateNotExpression_WhereClause_Painless() { LogicalPlan p = plan("SELECT * FROM test WHERE NOT(POSITION('x', keyword) = 0)");