diff --git a/docs/build.gradle b/docs/build.gradle index 73289c8fc41..530b6a7a455 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -213,9 +213,15 @@ setups['library'] = ''' book: properties: name: - type: keyword + type: text + fields: + keyword: + type: keyword author: - type: keyword + type: text + fields: + keyword: + type: keyword release_date: type: date page_count: @@ -232,6 +238,12 @@ setups['library'] = ''' {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} {"index":{"_id": "Dune"}} {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} + {"index":{"_id": "Dune Messiah"}} + {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} + {"index":{"_id": "Children of Dune"}} + {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} + {"index":{"_id": "God Emperor of Dune"}} + {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} {"index":{"_id": "Consider Phlebas"}} {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} {"index":{"_id": "Pandora's Star"}} diff --git a/docs/en/sql/endpoints/sql-rest.asciidoc b/docs/en/sql/endpoints/sql-rest.asciidoc index f6fcd00c41e..ffde8934203 100644 --- a/docs/en/sql/endpoints/sql-rest.asciidoc +++ b/docs/en/sql/endpoints/sql-rest.asciidoc @@ -28,6 +28,7 @@ Frank Herbert |Dune |604 |-144720000000 Alastair Reynolds|Revelation Space |585 |953078400000 James S.A. Corey |Leviathan Wakes |561 |1306972800000 -------------------------------------------------- +// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] // TESTRESPONSE[_cat] You can also choose to get results in a structured format by adding the `format` parameter. Currently supported formats: @@ -58,8 +59,8 @@ Which returns: -------------------------------------------------- { "columns": [ - {"name": "author", "type": "keyword"}, - {"name": "name", "type": "keyword"}, + {"name": "author", "type": "text"}, + {"name": "name", "type": "text"}, {"name": "page_count", "type": "short"}, {"name": "release_date", "type": "date"} ], @@ -100,8 +101,8 @@ Which looks like: ["Dan Simmons", "Hyperion", 482, 612144000000], ["Iain M. Banks", "Consider Phlebas", 471, 546134400000], ["Neal Stephenson", "Snow Crash", 470, 707356800000], - ["Robert A. Heinlein", "Starship Troopers", 335, -318297600000], - ["George Orwell", "1984", 328, 486432000000] + ["Frank Herbert", "God Emperor of Dune", 454, 359856000000], + ["Frank Herbert", "Children of Dune", 408, 198892800000] ], "cursor" : "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f///w8=" } @@ -174,6 +175,7 @@ Which returns: ---------------+------------------------------------+---------------+--------------- Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |308534400000 -------------------------------------------------- +// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] // TESTRESPONSE[_cat] [[sql-rest-fields]] diff --git a/docs/en/sql/endpoints/sql-translate.asciidoc b/docs/en/sql/endpoints/sql-translate.asciidoc index 19c5befb392..5c00fe29c3c 100644 --- a/docs/en/sql/endpoints/sql-translate.asciidoc +++ b/docs/en/sql/endpoints/sql-translate.asciidoc @@ -23,11 +23,16 @@ Which returns: { "size" : 10, "docvalue_fields" : [ - "author", - "name", "page_count", "release_date" ], + "_source": { + "includes": [ + "author", + "name" + ], + "excludes": [] + }, "sort" : [ { "page_count" : { diff --git a/docs/en/sql/language/sql-syntax.asciidoc b/docs/en/sql/language/sql-syntax.asciidoc index dcee4ca6d55..9311a0bd6ca 100644 --- a/docs/en/sql/language/sql-syntax.asciidoc +++ b/docs/en/sql/language/sql-syntax.asciidoc @@ -9,3 +9,92 @@ Each entry might get its own file and code snippet -------------------------------------------------- include-tagged::{sql-spec}/select.sql-spec[wildcardWithOrder] -------------------------------------------------- + + +[[sql-spec-syntax-order-by]] +==== ORDER BY + +Elasticsearch supports `ORDER BY` for consistent ordering. You add +any field in the index that has <> or +`SCORE()` to sort by `_score`. By default SQL sorts on what it +considers to be the most efficient way to get the results. + +So sorting by a field looks like: + +[source,js] +-------------------------------------------------- +POST /_xpack/sql +{ + "query": "SELECT * FROM library ORDER BY page_count DESC LIMIT 5" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:library] + +which results in something like: + +[source,text] +-------------------------------------------------- + author | name | page_count | release_date +-----------------+--------------------+---------------+--------------- +Peter F. Hamilton|Pandora's Star |768 |1078185600000 +Vernor Vinge |A Fire Upon the Deep|613 |707356800000 +Frank Herbert |Dune |604 |-144720000000 +Alastair Reynolds|Revelation Space |585 |953078400000 +James S.A. Corey |Leviathan Wakes |561 |1306972800000 +-------------------------------------------------- +// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] +// TESTRESPONSE[_cat] + +[[sql-spec-syntax-order-by-score]] +For sorting by score to be meaningful you need to include a full +text query in the `WHERE` clause. For example: + +[source,js] +-------------------------------------------------- +POST /_xpack/sql +{ + "query": "SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY SCORE() DESC" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:library] + +Which results in something like: + +[source,text] +-------------------------------------------------- + SCORE() | author | name | page_count | release_date +---------------+---------------+-------------------+---------------+--------------- +2.288635 |Frank Herbert |Dune |604 |-144720000000 +1.8893257 |Frank Herbert |Dune Messiah |331 |-6739200000 +1.6086555 |Frank Herbert |Children of Dune |408 |198892800000 +1.4005898 |Frank Herbert |God Emperor of Dune|454 |359856000000 +-------------------------------------------------- +// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/ s/\(/\\\(/ s/\)/\\\)/] +// TESTRESPONSE[_cat] + +Note that you can return `SCORE()` by adding it to the where clause. This +is possible even if you are not sorting by `SCORE()`: + +[source,js] +-------------------------------------------------- +POST /_xpack/sql +{ + "query": "SELECT SCORE(), * FROM library WHERE match(name, 'dune') ORDER BY page_count DESC" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:library] + +[source,text] +-------------------------------------------------- + SCORE() | author | name | page_count | release_date +---------------+---------------+-------------------+---------------+--------------- +2.288635 |Frank Herbert |Dune |604 |-144720000000 +1.4005898 |Frank Herbert |God Emperor of Dune|454 |359856000000 +1.6086555 |Frank Herbert |Children of Dune |408 |198892800000 +1.8893257 |Frank Herbert |Dune Messiah |331 |-6739200000 +-------------------------------------------------- +// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/ s/\(/\\\(/ s/\)/\\\)/] +// TESTRESPONSE[_cat] diff --git a/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java b/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java index 93a410b7d21..6dbe559c0e0 100644 --- a/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java +++ b/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java @@ -45,7 +45,14 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(readLine(), startsWith(" \"test_field\"")); assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"excludes\" : [ ]")); - assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" },")); + assertThat(readLine(), startsWith(" \"sort\" : [")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"_doc\" :")); + assertThat(readLine(), startsWith(" \"order\" : \"asc\"")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" ]")); assertThat(readLine(), startsWith("}]")); assertEquals("", readLine()); } @@ -97,6 +104,13 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(readLine(), startsWith(" },")); assertThat(readLine(), startsWith(" \"docvalue_fields\" : [")); assertThat(readLine(), startsWith(" \"i\"")); + assertThat(readLine(), startsWith(" ],")); + assertThat(readLine(), startsWith(" \"sort\" : [")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"_doc\" :")); + assertThat(readLine(), startsWith(" \"order\" : \"asc\"")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" ]")); assertThat(readLine(), startsWith("}]")); assertEquals("", readLine()); @@ -132,7 +146,14 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(readLine(), startsWith("EsQueryExec[test,{")); assertThat(readLine(), startsWith(" \"size\" : 0,")); assertThat(readLine(), startsWith(" \"_source\" : false,")); - assertThat(readLine(), startsWith(" \"stored_fields\" : \"_none_\"")); + assertThat(readLine(), startsWith(" \"stored_fields\" : \"_none_\",")); + assertThat(readLine(), startsWith(" \"sort\" : [")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"_doc\" :")); + assertThat(readLine(), startsWith(" \"order\" : \"asc\"")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" ]")); assertThat(readLine(), startsWith("}]")); assertEquals("", readLine()); } diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java index a08d2682b3c..8421ec96311 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java @@ -16,4 +16,9 @@ public interface ErrorsTestCase { void testSelectFromIndexWithoutTypes() throws Exception; void testSelectMissingField() throws Exception; void testSelectMissingFunction() throws Exception; + void testSelectProjectScoreInAggContext() throws Exception; + void testSelectOrderByScoreInAggContext() throws Exception; + void testSelectGroupByScore() throws Exception; + void testSelectScoreSubField() throws Exception; + void testSelectScoreInScalar() throws Exception; } diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java index 3adc49ff0ff..bc7d275eab1 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java @@ -11,6 +11,8 @@ import org.apache.http.entity.StringEntity; import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.startsWith; + /** * Tests for error messages. */ @@ -49,4 +51,42 @@ public abstract class ErrorsTestCase extends CliIntegrationTestCase implements o assertEquals("[1;31mBad request [[22;3;33mFound 1 problem(s)", command("SELECT missing(foo) FROM test")); assertEquals("line 1:8: Unknown function [missing][1;23;31m][0m", readLine()); } + + @Override + public void testSelectProjectScoreInAggContext() throws Exception { + index("test", body -> body.field("foo", 1)); + assertEquals("[1;31mBad request [[22;3;33mFound 1 problem(s)", command("SELECT foo, SCORE(), COUNT(*) FROM test GROUP BY foo")); + assertEquals("line 1:13: Cannot use non-grouped column [SCORE()], expected [foo][1;23;31m][0m", readLine()); + } + + @Override + public void testSelectOrderByScoreInAggContext() throws Exception { + index("test", body -> body.field("foo", 1)); + assertEquals("[1;31mBad request [[22;3;33mFound 1 problem(s)", + command("SELECT foo, COUNT(*) FROM test GROUP BY foo ORDER BY SCORE()")); + assertEquals("line 1:54: Cannot order by non-grouped column [SCORE()], expected [foo][1;23;31m][0m", readLine()); + } + + @Override + public void testSelectGroupByScore() throws Exception { + index("test", body -> body.field("foo", 1)); + assertEquals("[1;31mBad request [[22;3;33mFound 1 problem(s)", + command("SELECT COUNT(*) FROM test GROUP BY SCORE()")); + assertEquals("line 1:36: Cannot use [SCORE()] for grouping[1;23;31m][0m", readLine()); + } + + @Override + public void testSelectScoreSubField() throws Exception { + index("test", body -> body.field("foo", 1)); + assertThat(command("SELECT SCORE().bar FROM test"), + startsWith("[1;31mBad request [[22;3;33mline 1:15: extraneous input '.' expecting {, ',',")); + } + + @Override + public void testSelectScoreInScalar() throws Exception { + index("test", body -> body.field("foo", 1)); + assertEquals("[1;31mBad request [[22;3;33mFound 1 problem(s)", + command("SELECT SIN(SCORE()) FROM test")); + assertEquals("line 1:12: [SCORE()] cannot be an argument to a function[1;23;31m][0m", readLine()); + } } diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java index 2e12b44df1a..e23de8d8682 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java @@ -39,9 +39,10 @@ public abstract class ShowTestCase extends CliIntegrationTestCase { while (scalarFunction.matcher(line).matches()) { line = readLine(); } - assertEquals("", line); + assertThat(line, RegexMatcher.matches("\\s*SCORE\\s*\\|\\s*SCORE\\s*")); + assertEquals("", readLine()); } - + public void testShowFunctionsLikePrefix() throws IOException { assertThat(command("SHOW FUNCTIONS LIKE 'L%'"), RegexMatcher.matches("\\s*name\\s*\\|\\s*type\\s*")); assertThat(readLine(), containsString("----------")); @@ -49,7 +50,7 @@ public abstract class ShowTestCase extends CliIntegrationTestCase { assertThat(readLine(), RegexMatcher.matches("\\s*LOG10\\s*\\|\\s*SCALAR\\s*")); assertEquals("", readLine()); } - + public void testShowFunctionsLikeInfix() throws IOException { assertThat(command("SHOW FUNCTIONS LIKE '%DAY%'"), RegexMatcher.matches("\\s*name\\s*\\|\\s*type\\s*")); assertThat(readLine(), containsString("----------")); diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java index 8729a4a50bb..0fffb0dac4c 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java @@ -12,6 +12,8 @@ import org.apache.http.entity.StringEntity; import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.startsWith; + /** * Tests for exceptions and their messages. */ @@ -60,4 +62,54 @@ public class ErrorsTestCase extends JdbcIntegrationTestCase implements org.elast assertEquals("Found 1 problem(s)\nline 1:8: Unknown function [missing]", e.getMessage()); } } + + @Override + public void testSelectProjectScoreInAggContext() throws Exception { + index("test", body -> body.field("foo", 1)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT foo, SCORE(), COUNT(*) FROM test GROUP BY foo").executeQuery()); + assertEquals("Found 1 problem(s)\nline 1:13: Cannot use non-grouped column [SCORE()], expected [foo]", e.getMessage()); + } + } + + @Override + public void testSelectOrderByScoreInAggContext() throws Exception { + index("test", body -> body.field("foo", 1)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT foo, COUNT(*) FROM test GROUP BY foo ORDER BY SCORE()").executeQuery()); + assertEquals("Found 1 problem(s)\nline 1:54: Cannot order by non-grouped column [SCORE()], expected [foo]", e.getMessage()); + } + } + + @Override + public void testSelectGroupByScore() throws Exception { + index("test", body -> body.field("foo", 1)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT COUNT(*) FROM test GROUP BY SCORE()").executeQuery()); + assertEquals("Found 1 problem(s)\nline 1:36: Cannot use [SCORE()] for grouping", e.getMessage()); + } + } + + @Override + public void testSelectScoreSubField() throws Exception { + index("test", body -> body.field("foo", 1)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT SCORE().bar FROM test").executeQuery()); + assertThat(e.getMessage(), startsWith("line 1:15: extraneous input '.' expecting {, ','")); + } + } + + @Override + public void testSelectScoreInScalar() throws Exception { + index("test", body -> body.field("foo", 1)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT SIN(SCORE()) FROM test").executeQuery()); + assertThat(e.getMessage(), startsWith("Found 1 problem(s)\nline 1:12: [SCORE()] cannot be an argument to a function")); + } + } } diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java index fbf0944471f..16b783f21fa 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.qa.sql.jdbc; import org.apache.logging.log4j.Logger; - +import org.relique.jdbc.csv.CsvResultSet; import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -63,11 +63,11 @@ public class JdbcAssert { expectedMeta.getColumnCount(), actualMeta.getColumnCount()), expectedCols.toString(), actualCols.toString()); } - + for (int column = 1; column <= expectedMeta.getColumnCount(); column++) { - String expectedName = expectedMeta.getColumnName(column); + String expectedName = expectedMeta.getColumnName(column); String actualName = actualMeta.getColumnName(column); - + if (!expectedName.equals(actualName)) { // to help debugging, indicate the previous column (which also happened to match and thus was correct) String expectedSet = expectedName; @@ -88,6 +88,10 @@ public class JdbcAssert { if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) { expectedType = Types.TIMESTAMP; } + // since csv doesn't support real, we use float instead..... + if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) { + expectedType = Types.REAL; + } assertEquals("Different column type for column [" + expectedName + "] (" + JDBCType.valueOf(expectedType) + " != " + JDBCType.valueOf(actualType) + ")", expectedType, actualType); } @@ -136,4 +140,4 @@ public class JdbcAssert { private static Object getTime(ResultSet rs, int column) throws SQLException { return rs.getTime(column, UTC_CALENDAR).getTime(); } -} \ No newline at end of file +} diff --git a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java index e96ca4e43d9..8b789087225 100644 --- a/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java +++ b/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java @@ -10,6 +10,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentHelper; @@ -71,7 +72,11 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - String request = "{\"query\":\"SELECT text, number, SIN(number) AS s FROM test ORDER BY number\", \"fetch_size\":2}"; + String request = "{\"query\":\"" + + " SELECT text, number, SIN(number) AS s, SCORE()" + + " FROM test" + + " ORDER BY number, SCORE()\", " + + "\"fetch_size\":2}"; String cursor = null; for (int i = 0; i < 20; i += 2) { @@ -87,11 +92,12 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe expected.put("columns", Arrays.asList( columnInfo("text", "text"), columnInfo("number", "long"), - columnInfo("s", "double"))); + columnInfo("s", "double"), + columnInfo("SCORE()", "float"))); } expected.put("rows", Arrays.asList( - Arrays.asList("text" + i, i, Math.sin(i)), - Arrays.asList("text" + (i + 1), i + 1, Math.sin(i + 1)))); + Arrays.asList("text" + i, i, Math.sin(i), 1.0), + Arrays.asList("text" + (i + 1), i + 1, Math.sin(i + 1), 1.0))); expected.put("size", 2); cursor = (String) response.remove("cursor"); assertResponse(expected, response); @@ -118,6 +124,26 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe new StringEntity("{\"query\":\"SELECT DAY_OF_YEAR(test), COUNT(*) FROM test\"}", ContentType.APPLICATION_JSON))); } + public void testScoreWithFieldNamedScore() throws IOException { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"name\":\"test\", \"score\":10}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("name", "text"), + columnInfo("score", "long"), + columnInfo("SCORE()", "float"))); + expected.put("rows", singletonList(Arrays.asList( + "test", 10, 1.0))); + expected.put("size", 1); + + assertResponse(expected, runSql("SELECT *, SCORE() FROM test ORDER BY SCORE()")); + assertResponse(expected, runSql("SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()")); + } + public void testSelectWithJoinFails() throws Exception { // Normal join not supported expectBadRequest(() -> runSql("SELECT * FROM test JOIN other"), @@ -177,10 +203,92 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); } - private void expectBadRequest(ThrowingRunnable code, Matcher errorMessageMatcher) { - ResponseException e = expectThrows(ResponseException.class, code); - assertEquals(e.getMessage(), 400, e.getResponse().getStatusLine().getStatusCode()); - assertThat(e.getMessage(), errorMessageMatcher); + @Override + public void testSelectProjectScoreInAggContext() throws Exception { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"foo\":1}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + expectBadRequest(() -> runSql( + " SELECT foo, SCORE(), COUNT(*)" + + " FROM test" + + " GROUP BY foo"), + containsString("Cannot use non-grouped column [SCORE()], expected [foo]")); + } + + @Override + public void testSelectOrderByScoreInAggContext() throws Exception { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"foo\":1}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + expectBadRequest(() -> runSql( + " SELECT foo, COUNT(*)" + + " FROM test" + + " GROUP BY foo" + + " ORDER BY SCORE()"), + containsString("Cannot order by non-grouped column [SCORE()], expected [foo]")); + } + + @Override + public void testSelectGroupByScore() throws Exception { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"foo\":1}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + expectBadRequest(() -> runSql("SELECT COUNT(*) FROM test GROUP BY SCORE()"), + containsString("Cannot use [SCORE()] for grouping")); + } + + @Override + public void testSelectScoreSubField() throws Exception { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"foo\":1}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + expectBadRequest(() -> runSql("SELECT SCORE().bar FROM test"), + containsString("line 1:15: extraneous input '.' expecting {, ','")); + } + + @Override + public void testSelectScoreInScalar() throws Exception { + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); + bulk.append("{\"foo\":1}\n"); + client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + + expectBadRequest(() -> runSql("SELECT SIN(SCORE()) FROM test"), + containsString("line 1:12: [SCORE()] cannot be an argument to a function")); + } + + private void expectBadRequest(CheckedSupplier, Exception> code, Matcher errorMessageMatcher) { + try { + Map result = code.get(); + fail("expected ResponseException but got " + result); + } catch (ResponseException e) { + if (400 != e.getResponse().getStatusLine().getStatusCode()) { + String body; + try { + body = Streams.copyToString(new InputStreamReader( + e.getResponse().getEntity().getContent(), StandardCharsets.UTF_8)); + } catch (IOException bre) { + throw new RuntimeException("error reading body after remote sent bad status", bre); + } + fail("expected [400] response but get [" + e.getResponse().getStatusLine().getStatusCode() + "] with body:\n" + body); + } + assertThat(e.getMessage(), errorMessageMatcher); + } catch (Exception e) { + throw new AssertionError("expected ResponseException but got [" + e.getClass() + "]", e); + } } private Map runSql(String sql) throws IOException { @@ -278,13 +386,12 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe "{\"test\":\"test\"}"); String expected = + " test \n" + + "---------------\n" + "test \n" + - "---------------\n" + - "test \n" + - "test \n"; + "test \n"; Tuple response = runSqlAsText("SELECT * FROM test"); - logger.warn(expected); - logger.warn(response.v1()); + assertEquals(expected, response.v1()); } public void testNextPageText() throws IOException { diff --git a/qa/sql/src/main/resources/command.csv-spec b/qa/sql/src/main/resources/command.csv-spec index 23c45baaa72..11f9de82088 100644 --- a/qa/sql/src/main/resources/command.csv-spec +++ b/qa/sql/src/main/resources/command.csv-spec @@ -64,6 +64,7 @@ SIN |SCALAR SINH |SCALAR SQRT |SCALAR TAN |SCALAR +SCORE |SCORE ; showFunctionsWithExactMatch diff --git a/qa/sql/src/main/resources/fulltext.csv-spec b/qa/sql/src/main/resources/fulltext.csv-spec index 73ef6363e8a..1484d5cca25 100644 --- a/qa/sql/src/main/resources/fulltext.csv-spec +++ b/qa/sql/src/main/resources/fulltext.csv-spec @@ -29,3 +29,17 @@ SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_na emp_no:i | first_name:s | gender:s | last_name:s 10095 |Hilari |M |Morton ; + +score +SELECT emp_no, first_name, SCORE() FROM test_emp WHERE MATCH(first_name, 'Erez') ORDER BY SCORE(); + + emp_no:i | first_name:s | SCORE():f +10076 |Erez |4.2096553 +; + +scoreAsSomething +SELECT emp_no, first_name, SCORE() as s FROM test_emp WHERE MATCH(first_name, 'Erez') ORDER BY SCORE(); + + emp_no:i | first_name:s | s:f +10076 |Erez |4.2096553 +; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 43bcc52d674..a9ca50b8ed9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -1013,4 +1013,4 @@ public class Analyzer extends RuleExecutor { return true; } } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 992975288fc..cb49af38e9a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -6,13 +6,16 @@ package org.elasticsearch.xpack.sql.analysis.analyzer; import org.elasticsearch.xpack.sql.capabilities.Unresolvable; +import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.Expressions; +import org.elasticsearch.xpack.sql.expression.Order; import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.sql.expression.function.Function; import org.elasticsearch.xpack.sql.expression.function.FunctionAttribute; import org.elasticsearch.xpack.sql.expression.function.Functions; +import org.elasticsearch.xpack.sql.expression.function.Score; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; import org.elasticsearch.xpack.sql.plan.logical.Filter; @@ -150,7 +153,6 @@ abstract class Verifier { // Concrete verifications - // // if there are no (major) unresolved failures, do more in-depth analysis if (failures.isEmpty()) { @@ -182,14 +184,17 @@ abstract class Verifier { if (!groupingFailures.contains(p)) { checkGroupBy(p, localFailures, resolvedFunctions, groupingFailures); } - // everything checks out - // mark the plan as analyzed - if (localFailures.isEmpty()) { - p.setAnalyzed(); - } - failures.addAll(localFailures); - }); + checkForScoreInsideFunctions(p, localFailures); + + // everything checks out + // mark the plan as analyzed + if (localFailures.isEmpty()) { + p.setAnalyzed(); + } + + failures.addAll(localFailures); + }); } return failures; @@ -256,6 +261,8 @@ abstract class Verifier { } return true; } + + // check whether plain columns specified in an agg are mentioned in the group-by private static boolean checkGroupByAgg(LogicalPlan p, Set localFailures, Set groupingFailures, Map functions) { if (p instanceof Aggregate) { @@ -266,6 +273,9 @@ abstract class Verifier { if (Functions.isAggregate(c)) { localFailures.add(fail(c, "Cannot use an aggregate [" + c.nodeName().toUpperCase(Locale.ROOT) + "] for grouping")); } + if (c instanceof Score) { + localFailures.add(fail(c, "Cannot use [SCORE()] for grouping")); + } })); if (!localFailures.isEmpty()) { @@ -306,24 +316,30 @@ abstract class Verifier { // TODO: this should be handled by a different rule if (function == null) { return false; - } - e = function; } + e = function; + } // scalar functions can be a binary tree // first test the function against the grouping // and if that fails, start unpacking hoping to find matches if (e instanceof ScalarFunction) { ScalarFunction sf = (ScalarFunction) e; + // found group for the expression if (Expressions.anyMatch(groupings, e::semanticEquals)) { return true; - } + } + // unwrap function to find the base for (Expression arg : sf.arguments()) { arg.collectFirstChildren(c -> checkGroupMatch(c, source, groupings, missing, functions)); - } + } + return true; + } else if (e instanceof Score) { + // Score can't be an aggretate function + missing.put(e, source); return true; } @@ -347,4 +363,14 @@ abstract class Verifier { } return false; } -} \ No newline at end of file + + private static void checkForScoreInsideFunctions(LogicalPlan p, Set localFailures) { + // Make sure that SCORE is only used in "top level" functions + p.forEachExpressions(e -> + e.forEachUp((Function f) -> + f.arguments().stream() + .filter(exp -> exp.anyMatch(Score.class::isInstance)) + .forEach(exp -> localFailures.add(fail(exp, "[SCORE()] cannot be an argument to a function"))), + Function.class)); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 54a701e5acc..e22c90fcd07 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -26,13 +26,16 @@ import org.elasticsearch.xpack.sql.expression.NestedFieldAttribute; import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ScoreProcessorDefinition; import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; import org.elasticsearch.xpack.sql.querydsl.agg.GroupByColumnAgg; import org.elasticsearch.xpack.sql.querydsl.agg.GroupingAgg; +import org.elasticsearch.xpack.sql.querydsl.container.AggRef; import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; import org.elasticsearch.xpack.sql.querydsl.container.ColumnReference; import org.elasticsearch.xpack.sql.querydsl.container.ComputedRef; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; +import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; import org.elasticsearch.xpack.sql.querydsl.container.ScriptFieldRef; import org.elasticsearch.xpack.sql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.sql.querydsl.container.SearchHitFieldRef; @@ -50,6 +53,7 @@ import java.util.Set; import static java.util.Collections.singletonList; import static org.elasticsearch.search.sort.SortBuilders.fieldSort; +import static org.elasticsearch.search.sort.SortBuilders.scoreSort; import static org.elasticsearch.search.sort.SortBuilders.scriptSort; public abstract class SourceGenerator { @@ -77,7 +81,7 @@ public abstract class SourceGenerator { Map scriptFields = new LinkedHashMap<>(); for (ColumnReference ref : container.columns()) { - collectFields(ref, sourceFields, docFields, scriptFields); + collectFields(source, ref, sourceFields, docFields, scriptFields); } if (!sourceFields.isEmpty()) { @@ -92,8 +96,6 @@ public abstract class SourceGenerator { source.scriptField(entry.getKey(), entry.getValue()); } - sorting(container, source); - // add the aggs Aggs aggs = container.aggs(); @@ -115,6 +117,8 @@ public abstract class SourceGenerator { source.aggregation(builder); } + sorting(container, source); + // add the pipeline aggs for (PipelineAggregationBuilder builder : aggs.asPipelineBuilders()) { source.aggregation(builder); @@ -133,97 +137,110 @@ public abstract class SourceGenerator { return source; } - private static void collectFields(ColumnReference ref, Set sourceFields, Set docFields, Map scriptFields) { + private static void collectFields(SearchSourceBuilder source, ColumnReference ref, + Set sourceFields, Set docFields, Map scriptFields) { if (ref instanceof ComputedRef) { ProcessorDefinition proc = ((ComputedRef) ref).processor(); - proc.forEachUp(l -> collectFields(l.context(), sourceFields, docFields, scriptFields), ReferenceInput.class); - } - else if (ref instanceof SearchHitFieldRef) { + if (proc instanceof ScoreProcessorDefinition) { + /* + * If we're SELECTing SCORE then force tracking scores just in case + * we're not sorting on them. + */ + source.trackScores(true); + } + proc.forEachUp(l -> collectFields(source, l.context(), sourceFields, docFields, scriptFields), ReferenceInput.class); + } else if (ref instanceof SearchHitFieldRef) { SearchHitFieldRef sh = (SearchHitFieldRef) ref; Set collection = sh.useDocValue() ? docFields : sourceFields; collection.add(sh.name()); - } - else if (ref instanceof ScriptFieldRef) { + } else if (ref instanceof ScriptFieldRef) { ScriptFieldRef sfr = (ScriptFieldRef) ref; scriptFields.put(sfr.name(), sfr.script().toPainless()); + } else if (ref instanceof AggRef) { + // Nothing to do + } else { + throw new IllegalStateException("unhandled field in collectFields [" + ref.getClass() + "][" + ref + "]"); } } private static void sorting(QueryContainer container, SearchSourceBuilder source) { - if (container.sort() != null) { - - for (Sort sortable : container.sort()) { - SortBuilder sortBuilder = null; - - if (sortable instanceof AttributeSort) { - AttributeSort as = (AttributeSort) sortable; - Attribute attr = as.attribute(); - - // sorting only works on not-analyzed fields - look for a multi-field replacement - if (attr instanceof FieldAttribute) { - FieldAttribute fa = (FieldAttribute) attr; - attr = fa.isAnalyzed() ? fa.notAnalyzedAttribute() : attr; - } - - // top-level doc value - if (attr instanceof RootFieldAttribute) { - sortBuilder = fieldSort(((RootFieldAttribute) attr).name()); - } - if (attr instanceof NestedFieldAttribute) { - NestedFieldAttribute nfa = (NestedFieldAttribute) attr; - FieldSortBuilder fieldSort = fieldSort(nfa.name()); - - String nestedPath = nfa.parentPath(); - NestedSortBuilder newSort = new NestedSortBuilder(nestedPath); - NestedSortBuilder nestedSort = fieldSort.getNestedSort(); - - if (nestedSort == null) { - fieldSort.setNestedSort(newSort); - } else { - for (; nestedSort.getNestedSort() != null; nestedSort = nestedSort.getNestedSort()) { - } - nestedSort.setNestedSort(newSort); - } - - nestedSort = newSort; - - List nestedQuery = new ArrayList<>(1); - - // copy also the nested queries fr(if any) - if (container.query() != null) { - container.query().forEachDown(nq -> { - // found a match - if (nestedPath.equals(nq.path())) { - // get the child query - the nested wrapping and inner hits are not needed - nestedQuery.add(nq.child().asBuilder()); - } - }, NestedQuery.class); - } - - if (nestedQuery.size() > 0) { - if (nestedQuery.size() > 1) { - throw new SqlIllegalArgumentException("nested query should have been grouped in one place"); - } - nestedSort.setFilter(nestedQuery.get(0)); - } - - sortBuilder = fieldSort; - } - } - if (sortable instanceof ScriptSort) { - ScriptSort ss = (ScriptSort) sortable; - sortBuilder = scriptSort(ss.script().toPainless(), ss.script().outputType().isNumeric() ? ScriptSortType.NUMBER : ScriptSortType.STRING); - } - - if (sortBuilder != null) { - sortBuilder.order(sortable.direction() == Direction.ASC ? SortOrder.ASC : SortOrder.DESC); - source.sort(sortBuilder); - } - } + if (source.aggregations() != null && source.aggregations().count() > 0) { + // Aggs can't be sorted using search sorting. That sorting is handled elsewhere. + return; } - else { + if (container.sort() == null || container.sort().isEmpty()) { // if no sorting is specified, use the _doc one source.sort("_doc"); + return; + } + for (Sort sortable : container.sort()) { + SortBuilder sortBuilder = null; + + if (sortable instanceof AttributeSort) { + AttributeSort as = (AttributeSort) sortable; + Attribute attr = as.attribute(); + + // sorting only works on not-analyzed fields - look for a multi-field replacement + if (attr instanceof FieldAttribute) { + FieldAttribute fa = (FieldAttribute) attr; + attr = fa.isAnalyzed() ? fa.notAnalyzedAttribute() : attr; + } + + // top-level doc value + if (attr instanceof RootFieldAttribute) { + sortBuilder = fieldSort(((RootFieldAttribute) attr).name()); + } + if (attr instanceof NestedFieldAttribute) { + NestedFieldAttribute nfa = (NestedFieldAttribute) attr; + FieldSortBuilder fieldSort = fieldSort(nfa.name()); + + String nestedPath = nfa.parentPath(); + NestedSortBuilder newSort = new NestedSortBuilder(nestedPath); + NestedSortBuilder nestedSort = fieldSort.getNestedSort(); + + if (nestedSort == null) { + fieldSort.setNestedSort(newSort); + } else { + for (; nestedSort.getNestedSort() != null; nestedSort = nestedSort.getNestedSort()) { + } + nestedSort.setNestedSort(newSort); + } + + nestedSort = newSort; + + List nestedQuery = new ArrayList<>(1); + + // copy also the nested queries fr(if any) + if (container.query() != null) { + container.query().forEachDown(nq -> { + // found a match + if (nestedPath.equals(nq.path())) { + // get the child query - the nested wrapping and inner hits are not needed + nestedQuery.add(nq.child().asBuilder()); + } + }, NestedQuery.class); + } + + if (nestedQuery.size() > 0) { + if (nestedQuery.size() > 1) { + throw new SqlIllegalArgumentException("nested query should have been grouped in one place"); + } + nestedSort.setFilter(nestedQuery.get(0)); + } + + sortBuilder = fieldSort; + } + } else if (sortable instanceof ScriptSort) { + ScriptSort ss = (ScriptSort) sortable; + sortBuilder = scriptSort(ss.script().toPainless(), ss.script().outputType().isNumeric() ? ScriptSortType.NUMBER : ScriptSortType.STRING); + } else if (sortable instanceof ScoreSort) { + sortBuilder = scoreSort(); + } + + if (sortBuilder != null) { + sortBuilder.order(sortable.direction() == Direction.ASC ? SortOrder.ASC : SortOrder.DESC); + source.sort(sortBuilder); + } } } @@ -236,4 +253,4 @@ public abstract class SourceGenerator { source.storedFields(NO_STORED_FIELD); } } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java index c64aa6368f5..6976884e0bc 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java @@ -20,12 +20,16 @@ import java.util.Objects; * {@link Processor} tree as a leaf (and thus can effectively parse the * {@link SearchHit} while this class is used when scrolling and passing down * the results. - * + * * In the future, the processor might be used across the board for all columns * to reduce API complexity (and keep the {@link HitExtractor} only as an * internal implementation detail). */ public class ComputingHitExtractor implements HitExtractor { + /** + * Stands for {@code comPuting}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ static final String NAME = "p"; private final Processor processor; @@ -79,4 +83,4 @@ public class ComputingHitExtractor implements HitExtractor { public String toString() { return processor.toString(); } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java index b0112c91e9e..f01ffb7b498 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java @@ -16,6 +16,10 @@ import java.util.Objects; * Returns the a constant for every search hit against which it is run. */ public class ConstantExtractor implements HitExtractor { + /** + * Stands for {@code constant}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ static final String NAME = "c"; private final Object constant; @@ -65,4 +69,4 @@ public class ConstantExtractor implements HitExtractor { public String toString() { return "^" + constant; } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java index 1d05748e0a0..d0c4752d791 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java @@ -17,7 +17,11 @@ import java.io.IOException; * Extracts field values from {@link SearchHit#field(String)}. */ public class DocValueExtractor implements HitExtractor { - static final String NAME = "f"; + /** + * Stands for {@code doc_value}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ + static final String NAME = "d"; private final String fieldName; public DocValueExtractor(String name) { @@ -81,4 +85,4 @@ public class DocValueExtractor implements HitExtractor { * values are. */ return "%" + fieldName; } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java index 729b951a831..bf827e945ec 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.List; public abstract class HitExtractors { - /** * All of the named writeables needed to deserialize the instances of * {@linkplain HitExtractor}. @@ -25,6 +24,7 @@ public abstract class HitExtractors { entries.add(new Entry(HitExtractor.class, InnerHitExtractor.NAME, InnerHitExtractor::new)); entries.add(new Entry(HitExtractor.class, SourceExtractor.NAME, SourceExtractor::new)); entries.add(new Entry(HitExtractor.class, ComputingHitExtractor.NAME, ComputingHitExtractor::new)); + entries.add(new Entry(HitExtractor.class, ScoreExtractor.NAME, in -> ScoreExtractor.INSTANCE)); entries.addAll(Processors.getNamedWriteables()); return entries; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java index 7d90659d979..58bc486e6d6 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java @@ -17,6 +17,10 @@ import java.util.Map; import java.util.Objects; public class InnerHitExtractor implements HitExtractor { + /** + * Stands for {@code inner}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ static final String NAME = "i"; private final String hitName, fieldName; private final boolean useDocValue; @@ -109,4 +113,4 @@ public class InnerHitExtractor implements HitExtractor { public int hashCode() { return Objects.hash(hitName, fieldName, useDocValue); } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractor.java new file mode 100644 index 00000000000..515ec8009e7 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractor.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search.extractor; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +/** + * Returns the a constant for every search hit against which it is run. + */ +public class ScoreExtractor implements HitExtractor { + public static final HitExtractor INSTANCE = new ScoreExtractor(); + /** + * Stands for {@code score}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ + static final String NAME = "sc"; + + private ScoreExtractor() {} + + @Override + public void writeTo(StreamOutput out) throws IOException { + // Nothing to write + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object get(SearchHit hit) { + return hit.getScore(); + } + + @Override + public String innerHitName() { + return null; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return 31; + } + + @Override + public String toString() { + return "SCORE"; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java index 2de0fbc862e..d99f8fa1b5f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java @@ -13,6 +13,10 @@ import java.io.IOException; import java.util.Map; public class SourceExtractor implements HitExtractor { + /** + * Stands for {@code _source}. We try to use short names for {@link HitExtractor}s + * to save a few bytes when when we send them back to the user. + */ public static final String NAME = "s"; private final String fieldName; @@ -68,4 +72,4 @@ public class SourceExtractor implements HitExtractor { * me of a hash table lookup. */ return "#" + fieldName; } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Attribute.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Attribute.java index f40192e9616..a55638c2c3c 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Attribute.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Attribute.java @@ -12,6 +12,11 @@ import java.util.Objects; import static java.util.Collections.emptyList; +/** + * {@link Expression}s that can be converted into Elasticsearch + * sorts, aggregations, or queries. They can also be extracted + * from the result of a search. + */ public abstract class Attribute extends NamedExpression { // empty - such as a top level attribute in SELECT cause @@ -99,7 +104,7 @@ public abstract class Attribute extends NamedExpression { public boolean equals(Object obj) { if (super.equals(obj)) { Attribute other = (Attribute) obj; - return Objects.equals(qualifier, other.qualifier) + return Objects.equals(qualifier, other.qualifier) && Objects.equals(nullable, other.nullable); } @@ -112,4 +117,4 @@ public abstract class Attribute extends NamedExpression { } protected abstract String label(); -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/NamedExpression.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/NamedExpression.java index d80e8bfbd51..f123f60620e 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/NamedExpression.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/NamedExpression.java @@ -19,7 +19,7 @@ public abstract class NamedExpression extends Expression { public NamedExpression(Location location, String name, List children, ExpressionId id) { this(location, name, children, id, false); } - + public NamedExpression(Location location, String name, List children, ExpressionId id, boolean synthetic) { super(location, children); this.name = name; @@ -57,7 +57,7 @@ public abstract class NamedExpression extends Expression { NamedExpression other = (NamedExpression) obj; return Objects.equals(synthetic, other.synthetic) - && Objects.equals(id, other.id) + && Objects.equals(id, other.id) && Objects.equals(name(), other.name()) && Objects.equals(children(), other.children()); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/DefaultFunctionRegistry.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/DefaultFunctionRegistry.java index 13771b36256..24312341f7a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/DefaultFunctionRegistry.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/DefaultFunctionRegistry.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.expression.function; +import org.elasticsearch.xpack.sql.expression.function.Score; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.sql.expression.function.aggregate.Avg; import org.elasticsearch.xpack.sql.expression.function.aggregate.Correlation; @@ -56,18 +57,76 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.math.Sin; import org.elasticsearch.xpack.sql.expression.function.scalar.math.Sinh; import org.elasticsearch.xpack.sql.expression.function.scalar.math.Sqrt; import org.elasticsearch.xpack.sql.expression.function.scalar.math.Tan; - +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.TreeMap; import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.xpack.sql.util.CollectionUtils.combine; +import static java.util.Collections.unmodifiableList; public class DefaultFunctionRegistry extends AbstractFunctionRegistry { - private static final Collection> FUNCTIONS = combine(agg(), scalar()); + private static final Collection> FUNCTIONS = unmodifiableList(Arrays.asList( + // Aggregate functions + Avg.class, + Count.class, + Max.class, + Min.class, + Sum.class, + // Statistics + Mean.class, + StddevPop.class, + VarPop.class, + Percentile.class, + PercentileRank.class, + SumOfSquares.class, + // Matrix aggs + MatrixCount.class, + MatrixMean.class, + MatrixVariance.class, + Skewness.class, + Kurtosis.class, + Covariance.class, + Correlation.class, + // Scalar functions + // Date + DayOfMonth.class, + DayOfWeek.class, + DayOfYear.class, + HourOfDay.class, + MinuteOfDay.class, + MinuteOfHour.class, + SecondOfMinute.class, + MonthOfYear.class, + Year.class, + // Math + Abs.class, + ACos.class, + ASin.class, + ATan.class, + Cbrt.class, + Ceil.class, + Cos.class, + Cosh.class, + Degrees.class, + E.class, + Exp.class, + Expm1.class, + Floor.class, + Log.class, + Log10.class, + Pi.class, + Radians.class, + Round.class, + Sin.class, + Sinh.class, + Sqrt.class, + Tan.class, + // Special + Score.class)); private static final Map ALIASES; static { @@ -92,75 +151,4 @@ public class DefaultFunctionRegistry extends AbstractFunctionRegistry { protected Map aliases() { return ALIASES; } - - private static Collection> agg() { - return Arrays.asList( - Avg.class, - Count.class, - Max.class, - Min.class, - Sum.class, - // statistics - Mean.class, - StddevPop.class, - VarPop.class, - Percentile.class, - PercentileRank.class, - SumOfSquares.class, - // Matrix aggs - MatrixCount.class, - MatrixMean.class, - MatrixVariance.class, - Skewness.class, - Kurtosis.class, - Covariance.class, - Correlation.class - ); - } - - private static Collection> scalar() { - return combine(dateTimeFunctions(), - mathFunctions()); - } - - private static Collection> dateTimeFunctions() { - return Arrays.asList( - DayOfMonth.class, - DayOfWeek.class, - DayOfYear.class, - HourOfDay.class, - MinuteOfDay.class, - MinuteOfHour.class, - SecondOfMinute.class, - MonthOfYear.class, - Year.class - ); - } - - private static Collection> mathFunctions() { - return Arrays.asList( - Abs.class, - ACos.class, - ASin.class, - ATan.class, - Cbrt.class, - Ceil.class, - Cos.class, - Cosh.class, - Degrees.class, - E.class, - Exp.class, - Expm1.class, - Floor.class, - Log.class, - Log10.class, - Pi.class, - Radians.class, - Round.class, - Sin.class, - Sinh.class, - Sqrt.class, - Tan.class - ); - } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java index ea7eb8ff61e..4be482e977a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java @@ -71,4 +71,4 @@ public abstract class Function extends NamedExpression { public boolean functionEquals(Function f) { return f != null && getClass() == f.getClass() && arguments().equals(f.arguments()); } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java index 471fc374a92..e0b7816bd2d 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java @@ -8,14 +8,16 @@ package org.elasticsearch.xpack.sql.expression.function; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.Score; + public enum FunctionType { - - AGGREGATE (AggregateFunction.class), - SCALAR(ScalarFunction.class); + AGGREGATE(AggregateFunction.class), + SCALAR(ScalarFunction.class), + SCORE(Score.class); private final Class baseClass; - + FunctionType(Class base) { this.baseClass = base; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Score.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Score.java new file mode 100644 index 00000000000..c1ab18c4d92 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Score.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function; + +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.function.Function; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import static java.util.Collections.emptyList; + +/** + * Function referring to the {@code _score} in a search. Only available + * in the search context, and only at the "root" so it can't be combined + * with other function. + */ +public class Score extends Function { + public Score(Location location) { + super(location, emptyList()); + } + + @Override + public DataType dataType() { + return DataTypes.FLOAT; + } + + @Override + public Attribute toAttribute() { + return new ScoreAttribute(location()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/ScoreAttribute.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/ScoreAttribute.java new file mode 100644 index 00000000000..73a9b3b18f6 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/ScoreAttribute.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function; + +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.ExpressionId; +import org.elasticsearch.xpack.sql.expression.function.FunctionAttribute; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; + +/** + * {@link Attribute} that represents Elasticsearch's {@code _score}. + */ +public class ScoreAttribute extends FunctionAttribute { + /** + * Constructor for normal use. + */ + ScoreAttribute(Location location) { + this(location, "SCORE()", DataTypes.FLOAT, null, false, null, false); + } + + /** + * Constructor for {@link #clone()} + */ + private ScoreAttribute(Location location, String name, DataType dataType, String qualifier, boolean nullable, ExpressionId id, + boolean synthetic) { + super(location, name, dataType, qualifier, nullable, id, synthetic, "SCORE"); + } + + @Override + protected Attribute clone(Location location, String name, DataType dataType, String qualifier, boolean nullable, + ExpressionId id, boolean synthetic) { + return new ScoreAttribute(location, name, dataType, qualifier, nullable, id, synthetic); + } + + @Override + protected String label() { + return "SCORE"; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java index b6f4b1dad36..56be150bedf 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java @@ -38,10 +38,10 @@ public abstract class ScalarFunction extends Function { } @Override - public ScalarFunctionAttribute toAttribute() { + public final ScalarFunctionAttribute toAttribute() { if (lazyAttribute == null) { lazyAttribute = new ScalarFunctionAttribute(location(), name(), dataType(), id(), functionId(), asScript(), orderBy(), - asProcessorDefinition()); + asProcessorDefinition()); } return lazyAttribute; } @@ -110,4 +110,4 @@ public abstract class ScalarFunction extends Function { public Expression orderBy() { return null; } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java index aecda04a375..dfd2a1153e4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java @@ -73,4 +73,4 @@ public class ScalarFunctionAttribute extends FunctionAttribute { protected String label() { return "s->" + functionId(); } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ScoreProcessorDefinition.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ScoreProcessorDefinition.java new file mode 100644 index 00000000000..024d45a50bf --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ScoreProcessorDefinition.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.execution.search.extractor.ScoreExtractor; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; + +import static java.util.Collections.emptyList; + +public class ScoreProcessorDefinition extends ProcessorDefinition { + public ScoreProcessorDefinition(Expression expression) { + super(expression, emptyList()); + } + + @Override + public boolean resolved() { + return true; + } + + @Override + public Processor asProcessor() { + return new HitExtractorProcessor(ScoreExtractor.INSTANCE); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ParsingException.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ParsingException.java index 0f874f044fe..a897cff1677 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ParsingException.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ParsingException.java @@ -45,13 +45,13 @@ public class ParsingException extends ClientSqlException { return super.getMessage(); } - @Override - public String getMessage() { - return format(Locale.ROOT, "line %s:%s: %s", getLineNumber(), getColumnNumber(), getErrorMessage()); - } - @Override public RestStatus status() { return RestStatus.BAD_REQUEST; } + + @Override + public String getMessage() { + return format(Locale.ROOT, "line %s:%s: %s", getLineNumber(), getColumnNumber(), getErrorMessage()); + } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/PlanningException.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/PlanningException.java index 555c5106664..858d545f462 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/PlanningException.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/PlanningException.java @@ -12,28 +12,22 @@ import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.Collection; import java.util.Locale; -import java.util.Optional; import java.util.stream.Collectors; import static java.lang.String.format; public class PlanningException extends ClientSqlException { - - private final Optional status; - public PlanningException(String message, Object... args) { super(message, args); - status = Optional.empty(); - } - - public PlanningException(String message, RestStatus restStatus, Object... args) { - super(message, args); - status = Optional.of(restStatus); } public PlanningException(Collection sources) { super(extractMessage(sources)); - status = Optional.empty(); + } + + @Override + public RestStatus status() { + return RestStatus.BAD_REQUEST; } private static String extractMessage(Collection failures) { @@ -41,9 +35,4 @@ public class PlanningException extends ClientSqlException { .map(f -> format(Locale.ROOT, "line %s:%s: %s", f.source().location().getLineNumber(), f.source().location().getColumnNumber(), f.message())) .collect(Collectors.joining(StringUtils.NEW_LINE, "Found " + failures.size() + " problem(s)\n", StringUtils.EMPTY)); } - - @Override - public RestStatus status() { - return status.orElse(super.status()); - } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 3970a67da18..13ee1ad82a8 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.sql.expression.NamedExpression; import org.elasticsearch.xpack.sql.expression.Order; import org.elasticsearch.xpack.sql.expression.function.Function; import org.elasticsearch.xpack.sql.expression.function.Functions; +import org.elasticsearch.xpack.sql.expression.function.ScoreAttribute; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.sql.expression.function.aggregate.CompoundNumericAggregate; import org.elasticsearch.xpack.sql.expression.function.aggregate.Count; @@ -46,6 +47,7 @@ import org.elasticsearch.xpack.sql.querydsl.agg.LeafAgg; import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; import org.elasticsearch.xpack.sql.querydsl.container.ComputedRef; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; +import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; import org.elasticsearch.xpack.sql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; import org.elasticsearch.xpack.sql.querydsl.container.TotalCountRef; @@ -355,13 +357,12 @@ class QueryFolder extends RuleExecutor { //FIXME: what about inner key queryC = withAgg.v1().addAggColumn(withAgg.v2().context()); if (withAgg.v2().innerKey() != null) { - throw new PlanningException("innerkey/matrix stats not handled (yet)", RestStatus.BAD_REQUEST); + throw new PlanningException("innerkey/matrix stats not handled (yet)"); } } } - } - // not an Alias or a Function, means it's an Attribute so apply the same logic as above - else { + // not an Alias or Function means it's an Attribute so apply the same logic as above + } else { GroupingAgg matchingGroup = null; if (groupingContext != null) { matchingGroup = groupingContext.groupFor(ne); @@ -432,7 +433,6 @@ class QueryFolder extends RuleExecutor { private static class FoldOrderBy extends FoldingRule { @Override protected PhysicalPlan rule(OrderExec plan) { - if (plan.child() instanceof EsQueryExec) { EsQueryExec exec = (EsQueryExec) plan.child(); QueryContainer qContainer = exec.queryContainer(); @@ -464,23 +464,20 @@ class QueryFolder extends RuleExecutor { ScalarFunctionAttribute sfa = (ScalarFunctionAttribute) attr; // is there an expression to order by? if (sfa.orderBy() != null) { - Expression ob = sfa.orderBy(); - if (ob instanceof NamedExpression) { - Attribute at = ((NamedExpression) ob).toAttribute(); + if (sfa.orderBy() instanceof NamedExpression) { + Attribute at = ((NamedExpression) sfa.orderBy()).toAttribute(); at = qContainer.aliases().getOrDefault(at, at); qContainer = qContainer.sort(new AttributeSort(at, direction)); - } - // ignore constant - else if (!ob.foldable()) { - throw new PlanningException("does not know how to order by expression %s", ob); + } else if (!sfa.orderBy().foldable()) { + // ignore constant + throw new PlanningException("does not know how to order by expression %s", sfa.orderBy()); } } // nope, use scripted sorting - else { - qContainer = qContainer.sort(new ScriptSort(sfa.script(), direction)); - } - } - else { + qContainer = qContainer.sort(new ScriptSort(sfa.script(), direction)); + } else if (attr instanceof ScoreAttribute) { + qContainer = qContainer.sort(new ScoreSort(direction)); + } else { qContainer = qContainer.sort(new AttributeSort(attr, direction)); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Verifier.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Verifier.java index c9a1b9fa0b1..0aef7ccaf8e 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Verifier.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Verifier.java @@ -104,4 +104,4 @@ abstract class Verifier { return failures; } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index e95208333b2..58211b4872c 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -15,10 +15,12 @@ import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.LiteralAttribute; import org.elasticsearch.xpack.sql.expression.NestedFieldAttribute; import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.ScoreAttribute; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.AttributeInput; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ScoreProcessorDefinition; import org.elasticsearch.xpack.sql.querydsl.agg.AggPath; import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; import org.elasticsearch.xpack.sql.querydsl.agg.GroupingAgg; @@ -287,6 +289,9 @@ public class QueryContainer { if (attr instanceof LiteralAttribute) { return new Tuple<>(this, new ComputedRef(((LiteralAttribute) attr).asProcessorDefinition())); } + if (attr instanceof ScoreAttribute) { + return new Tuple<>(this, new ComputedRef(new ScoreProcessorDefinition(attr))); + } throw new SqlIllegalArgumentException("Unknown output attribute %s", attr); } @@ -371,4 +376,4 @@ public class QueryContainer { throw new RuntimeException("error rendering", e); } } -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ScoreSort.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ScoreSort.java new file mode 100644 index 00000000000..c05864578d9 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ScoreSort.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.container; + +import java.util.Objects; + +public class ScoreSort extends Sort { + public ScoreSort(Direction direction) { + super(direction); + } + + @Override + public int hashCode() { + return Objects.hash(direction()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ScriptSort other = (ScriptSort) obj; + return Objects.equals(direction(), other.direction()); + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java index bd3b37e2fcc..5857b8553e0 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java @@ -13,15 +13,26 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.Score; import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; +import org.elasticsearch.xpack.sql.querydsl.agg.AvgAgg; import org.elasticsearch.xpack.sql.querydsl.agg.GroupByColumnAgg; +import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; +import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; +import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; import org.elasticsearch.xpack.sql.querydsl.query.MatchQuery; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.search.sort.SortBuilders.fieldSort; +import static org.elasticsearch.search.sort.SortBuilders.scoreSort; import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; public class SourceGeneratorTests extends ESTestCase { @@ -61,4 +72,49 @@ public class SourceGeneratorTests extends ESTestCase { TermsAggregationBuilder termsBuilder = (TermsAggregationBuilder) aggBuilder.getAggregatorFactories().get(0); assertEquals(10, termsBuilder.size()); } + + public void testSortNoneSpecified() { + QueryContainer container = new QueryContainer(); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertEquals(singletonList(fieldSort("_doc")), sourceBuilder.sorts()); + } + + public void testSelectScoreForcesTrackingScore() { + QueryContainer container = new QueryContainer() + .addColumn(new Score(new Location(1, 1)).toAttribute()); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertTrue(sourceBuilder.trackScores()); + } + + public void testSortScoreSpecified() { + QueryContainer container = new QueryContainer() + .sort(new ScoreSort(Direction.DESC)); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertEquals(singletonList(scoreSort()), sourceBuilder.sorts()); + } + + public void testSortFieldSpecified() { + QueryContainer container = new QueryContainer() + .sort(new AttributeSort(new RootFieldAttribute(new Location(1, 1), "test", DataTypes.KEYWORD), Direction.ASC)); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertEquals(singletonList(fieldSort("test").order(SortOrder.ASC)), sourceBuilder.sorts()); + + container = new QueryContainer() + .sort(new AttributeSort(new RootFieldAttribute(new Location(1, 1), "test", DataTypes.KEYWORD), Direction.DESC)); + sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertEquals(singletonList(fieldSort("test").order(SortOrder.DESC)), sourceBuilder.sorts()); + } + + public void testNoSort() { + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(new QueryContainer(), null, randomIntBetween(1, 10)); + assertEquals(singletonList(fieldSort("_doc").order(SortOrder.ASC)), sourceBuilder.sorts()); + } + + public void testNoSortIfAgg() { + QueryContainer container = new QueryContainer() + .addGroups(singletonList(new GroupByColumnAgg("group_id", "", "group_column"))) + .addAgg("group_id", new AvgAgg("agg_id", "", "avg_column")); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); + assertNull(sourceBuilder.sorts()); + } } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractorTests.java new file mode 100644 index 00000000000..7f8945d9faf --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ScoreExtractorTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search.extractor; + +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESTestCase; + +public class ScoreExtractorTests extends ESTestCase { + public void testGet() { + int times = between(1, 1000); + for (int i = 0; i < times; i++) { + float score = randomFloat(); + SearchHit hit = new SearchHit(1); + hit.score(score); + assertEquals(score, ScoreExtractor.INSTANCE.get(hit)); + } + } + + public void testToString() { + assertEquals("SCORE", ScoreExtractor.INSTANCE.toString()); + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java new file mode 100644 index 00000000000..2d166d3b106 --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.parser; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.expression.NamedExpression; +import org.elasticsearch.xpack.sql.expression.Order; +import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.sql.expression.UnresolvedStar; +import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; +import org.elasticsearch.xpack.sql.parser.SqlParser; +import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.sql.plan.logical.OrderBy; +import org.elasticsearch.xpack.sql.plan.logical.Project; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; + +import static java.util.stream.Collectors.toList; + +public class SqlParserTests extends ESTestCase { + public void testSelectStar() { + singleProjection(project(parseStatement("SELECT * FROM foo")), UnresolvedStar.class); + } + + private T singleProjection(Project project, Class type) { + assertThat(project.projections(), hasSize(1)); + NamedExpression p = project.projections().get(0); + assertThat(p, instanceOf(type)); + return type.cast(p); + } + + public void testSelectField() { + UnresolvedAttribute a = singleProjection(project(parseStatement("SELECT bar FROM foo")), UnresolvedAttribute.class); + assertEquals("bar", a.name()); + } + + public void testSelectScore() { + UnresolvedFunction f = singleProjection(project(parseStatement("SELECT SCORE() FROM foo")), UnresolvedFunction.class); + assertEquals("SCORE", f.functionName()); + } + + public void testOrderByField() { + Order.OrderDirection dir = randomFrom(Order.OrderDirection.values()); + OrderBy ob = orderBy(parseStatement("SELECT * FROM foo ORDER BY bar" + stringForDirection(dir))); + assertThat(ob.order(), hasSize(1)); + Order o = ob.order().get(0); + assertEquals(dir, o.direction()); + assertThat(o.child(), instanceOf(UnresolvedAttribute.class)); + UnresolvedAttribute a = (UnresolvedAttribute) o.child(); + assertEquals("bar", a.name()); + } + + public void testOrderByScore() { + Order.OrderDirection dir = randomFrom(Order.OrderDirection.values()); + OrderBy ob = orderBy(parseStatement("SELECT * FROM foo ORDER BY SCORE()" + stringForDirection(dir))); + assertThat(ob.order(), hasSize(1)); + Order o = ob.order().get(0); + assertEquals(dir, o.direction()); + assertThat(o.child(), instanceOf(UnresolvedFunction.class)); + UnresolvedFunction f = (UnresolvedFunction) o.child(); + assertEquals("SCORE", f.functionName()); + } + + public void testOrderByTwo() { + Order.OrderDirection dir0 = randomFrom(Order.OrderDirection.values()); + Order.OrderDirection dir1 = randomFrom(Order.OrderDirection.values()); + OrderBy ob = orderBy(parseStatement( + " SELECT *" + + " FROM foo" + + " ORDER BY bar" + stringForDirection(dir0) + ", baz" + stringForDirection(dir1))); + assertThat(ob.order(), hasSize(2)); + Order o = ob.order().get(0); + assertEquals(dir0, o.direction()); + assertThat(o.child(), instanceOf(UnresolvedAttribute.class)); + UnresolvedAttribute a = (UnresolvedAttribute) o.child(); + assertEquals("bar", a.name()); + o = ob.order().get(1); + assertEquals(dir1, o.direction()); + assertThat(o.child(), instanceOf(UnresolvedAttribute.class)); + a = (UnresolvedAttribute) o.child(); + assertEquals("baz", a.name()); + } + + private LogicalPlan parseStatement(String sql) { + return new SqlParser().createStatement(sql); + } + + private Project project(LogicalPlan plan) { + List sync = new ArrayList(1); + projectRecur(plan, sync); + assertThat("expected only one SELECT", sync, hasSize(1)); + return sync.get(0); + } + + private void projectRecur(LogicalPlan plan, List sync) { + if (plan instanceof Project) { + sync.add((Project) plan); + return; + } + for (LogicalPlan child : plan.children()) { + projectRecur(child, sync); + } + } + + /** + * Find the one and only {@code ORDER BY} in a plan. + */ + private OrderBy orderBy(LogicalPlan plan) { + List l = plan.children().stream() + .filter(c -> c instanceof OrderBy) + .collect(toList()); + assertThat("expected only one ORDER BY", l, hasSize(1)); + return (OrderBy) l.get(0); + } + + /** + * Convert a direction into a string that represents that parses to + * that direction. + */ + private String stringForDirection(Order.OrderDirection dir) { + String dirStr = dir.toString(); + return randomBoolean() && dirStr.equals("ASC") ? "" : " " + dirStr; + } +}