diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index e21071a49db..1f434093626 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -133,7 +133,7 @@ Which returns: [float] === Paginating through a large response -Using the example above, onu can continue to the next page by sending back the `cursor` field. In +Using the example above, one can continue to the next page by sending back the `cursor` field. In case of text format the cursor is returned as `Cursor` http header. [source,js] @@ -235,6 +235,81 @@ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] // TESTRESPONSE[_cat] +[[sql-rest-columnar]] +[float] +=== Columnar results + +The most well known way of displaying the results of an SQL query result in general is the one where each +individual record/document represents one line/row. For certain formats, {es-sql} can return the results +in a columnar fashion: one row represents all the values of a certain column from the current page of results. + +The following formats can be returned in columnar orientation: `json`, `yaml`, `cbor` and `smile`. + +[source,js] +-------------------------------------------------- +POST /_sql?format=json +{ + "query": "SELECT * FROM library ORDER BY page_count DESC", + "fetch_size": 5, + "columnar": true +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:library] + +Which returns: + +[source,js] +-------------------------------------------------- +{ + "columns": [ + {"name": "author", "type": "text"}, + {"name": "name", "type": "text"}, + {"name": "page_count", "type": "short"}, + {"name": "release_date", "type": "datetime"} + ], + "values": [ + ["Peter F. Hamilton", "Vernor Vinge", "Frank Herbert", "Alastair Reynolds", "James S.A. Corey"], + ["Pandora's Star", "A Fire Upon the Deep", "Dune", "Revelation Space", "Leviathan Wakes"], + [768, 613, 604, 585, 561], + ["2004-03-02T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1965-06-01T00:00:00.000Z", "2000-03-15T00:00:00.000Z", "2011-06-02T00:00:00.000Z"] + ], + "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=" +} +-------------------------------------------------- +// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/] + +Any subsequent calls using a `cursor` still have to contain the `columnar` parameter to preserve the orientation, +meaning the initial query will not _remember_ the columnar option. + +[source,js] +-------------------------------------------------- +POST /_sql?format=json +{ + "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=", + "columnar": true +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] +// TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/] + +Which looks like: + +[source,js] +-------------------------------------------------- +{ + "values": [ + ["Dan Simmons", "Iain M. Banks", "Neal Stephenson", "Frank Herbert", "Frank Herbert"], + ["Hyperion", "Consider Phlebas", "Snow Crash", "God Emperor of Dune", "Children of Dune"], + [482, 471, 470, 454, 408], + ["1989-05-26T00:00:00.000Z", "1987-04-23T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1981-05-28T00:00:00.000Z", "1976-04-21T00:00:00.000Z"] + ], + "cursor": "46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3/////DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP" +} +-------------------------------------------------- +// TESTRESPONSE[s/46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3\/\/\/\/\/DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP/$body.cursor/] + [[sql-rest-fields]] [float] === Supported REST parameters @@ -277,7 +352,11 @@ s|Description |Time-zone in ISO 8601 used for executing the query on the server. More information available https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html[here]. +|columnar +|false +|Return the results in a columnar fashion, rather than row-based fashion. Valid for `json`, `yaml`, `cbor` and `smile`. + |=== -Do note that most parameters (outside the timeout ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. +Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. That's because the query has already been executed and the calls are simply about returning the found results - thus the parameters are simply ignored. \ No newline at end of file diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java index 1b0e71736e9..ea5b99180ce 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java @@ -53,7 +53,7 @@ class JdbcHttpClient { SqlQueryRequest sqlRequest = new SqlQueryRequest(sql, params, null, Protocol.TIME_ZONE, fetch, TimeValue.timeValueMillis(meta.timeoutInMs()), TimeValue.timeValueMillis(meta.queryTimeoutInMs()), - new RequestInfo(Mode.JDBC)); + false, new RequestInfo(Mode.JDBC)); SqlQueryResponse response = httpClient.query(sqlRequest); return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta); } diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java index 0b71fe1c88c..2d93597efc1 100644 --- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java +++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java @@ -105,7 +105,6 @@ public class UserFunctionIT extends ESRestTestCase { assertResponse(expected, actual); } - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35980") public void testSingleRandomUserWithWhereEvaluatingFalse() throws IOException { index("{\"test\":\"doc1\"}", "{\"test\":\"doc2\"}", diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index e346bfc649b..233c4b6a420 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -121,7 +121,8 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase { @SuppressWarnings({ "unchecked" }) private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize, Mode mode) throws IOException { - Map response = runSql(mode.toString(), sql); + boolean columnar = randomBoolean(); + Map response = runSql(mode.toString(), sql, columnar); List columns = (ArrayList) response.get("columns"); assertEquals(1, columns.size()); @@ -135,7 +136,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase { assertEquals(2, column.size()); } - List rows = (ArrayList) response.get("rows"); + List rows = (ArrayList) response.get(columnar == true ? "values" : "rows"); assertEquals(1, rows.size()); List row = (ArrayList) rows.get(0); assertEquals(1, row.size()); @@ -149,7 +150,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase { } } - private Map runSql(String mode, String sql) throws IOException { + private Map runSql(String mode, String sql, boolean columnar) throws IOException { Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT); String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}"; String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT); @@ -177,6 +178,11 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase { options.addHeader("Accept", randomFrom("*/*", "application/" + format)); request.setOptions(options); } + if ((false == columnar && randomBoolean()) || columnar) { + // randomly set the "columnar" parameter, either "true" (non-default) or explicit "false" (the default anyway) + requestContent = new StringBuilder(requestContent) + .insert(requestContent.length() - 1, ",\"columnar\":" + columnar).toString(); + } // send the query either as body or as request parameter if (randomBoolean()) { diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 6f070491df1..afcabeedf59 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.qa.rest; import com.fasterxml.jackson.core.io.JsonStringEncoder; + import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; @@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.containsString; public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTestCase { public static String SQL_QUERY_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT; + private static String SQL_TRANSLATE_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT; /** * Builds that map that is returned in the header for each column. */ @@ -70,9 +72,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe Map expected = new HashMap<>(); String mode = randomMode(); + boolean columnar = randomBoolean(); + expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0))); - expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test"))); - assertResponse(expected, runSql(mode, "SELECT * FROM test")); + if (columnar) { + expected.put("values", singletonList(Arrays.asList("test", "test"))); + } else { + expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test"))); + } + assertResponse(expected, runSql(mode, "SELECT * FROM test", columnar)); } public void testNextPage() throws IOException { @@ -86,14 +94,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe } request.setJsonEntity(bulk.toString()); client().performRequest(request); - + + boolean columnar = randomBoolean(); String sqlRequest = "{\"query\":\"" + " SELECT text, number, SQRT(number) AS s, SCORE()" + " FROM test" + " ORDER BY number, SCORE()\", " + "\"mode\":\"" + mode + "\", " - + "\"fetch_size\":2}"; + + "\"fetch_size\":2" + columnarParameter(columnar) + "}"; String cursor = null; for (int i = 0; i < 20; i += 2) { @@ -101,7 +110,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe if (i == 0) { response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), ""); } else { - response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + "}", + columnar = randomBoolean(); + response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY); } @@ -113,32 +123,52 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe columnInfo(mode, "s", "double", JDBCType.DOUBLE, 25), columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15))); } - expected.put("rows", Arrays.asList( - Arrays.asList("text" + i, i, Math.sqrt(i), 1.0), - Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0))); + + if (columnar) { + expected.put("values", Arrays.asList( + Arrays.asList("text" + i, "text" + (i + 1)), + Arrays.asList(i, i + 1), + Arrays.asList(Math.sqrt(i), Math.sqrt(i + 1)), + Arrays.asList(1.0, 1.0))); + } else { + expected.put("rows", Arrays.asList( + Arrays.asList("text" + i, i, Math.sqrt(i), 1.0), + Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0))); + } cursor = (String) response.remove("cursor"); assertResponse(expected, response); assertNotNull(cursor); } Map expected = new HashMap<>(); - expected.put("rows", emptyList()); - assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + "}", + columnar = randomBoolean(); + if (columnar) { + expected.put("values", emptyList()); + } else { + expected.put("rows", emptyList()); + } + assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY)); } @AwaitsFix(bugUrl = "Unclear status, https://github.com/elastic/x-pack-elasticsearch/issues/2074") public void testTimeZone() throws IOException { String mode = randomMode(); + boolean columnar = randomBoolean(); index("{\"test\":\"2017-07-27 00:00:00\"}", "{\"test\":\"2017-07-27 01:00:00\"}"); Map expected = new HashMap<>(); expected.put("columns", singletonMap("test", singletonMap("type", "text"))); - expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test"))); + if (columnar) { + expected.put("values", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test"))); + } else { + // TODO: what exactly is this test suppossed to do. We need to check the 2074 issue above. + expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test"))); + } expected.put("size", 2); // Default TimeZone is UTC - assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test")); + assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test", columnar)); } public void testScoreWithFieldNamedScore() throws IOException { @@ -152,15 +182,19 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe client().performRequest(request); Map expected = new HashMap<>(); + boolean columnar = randomBoolean(); expected.put("columns", Arrays.asList( columnInfo(mode, "name", "text", JDBCType.VARCHAR, 0), columnInfo(mode, "score", "long", JDBCType.BIGINT, 20), columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15))); - expected.put("rows", singletonList(Arrays.asList( - "test", 10, 1.0))); - - assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()")); - assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()")); + if (columnar) { + expected.put("values", Arrays.asList(singletonList("test"), singletonList(10), singletonList(1.0))); + } else { + expected.put("rows", singletonList(Arrays.asList("test", 10, 1.0))); + } + + assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()", columnar)); + assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()", columnar)); } public void testSelectWithJoinFails() throws Exception { @@ -195,8 +229,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe public void testSelectWhereExistsFails() throws Exception { index("{\"foo\":1}", "{\"foo\":2}"); - expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)"), - containsString("line 1:28: EXISTS is not yet supported")); + expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)", + randomBoolean()), containsString("line 1:28: EXISTS is not yet supported")); } @@ -280,6 +314,34 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"), containsString("line 1:12: [SCORE()] cannot be an argument to a function")); } + + public void testUseColumnarForUnsupportedFormats() throws Exception { + String format = randomFrom("txt", "csv", "tsv"); + index("{\"foo\":1}"); + + Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT); + request.addParameter("error_trace", "true"); + request.addParameter("pretty", "true"); + request.addParameter("format", format); + request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}", + ContentType.APPLICATION_JSON)); + expectBadRequest(() -> { + client().performRequest(request); + return Collections.emptyMap(); + }, containsString("Invalid use of [columnar] argument: cannot be used in combination with txt, csv or tsv formats")); + } + + public void testUseColumnarForTranslateRequest() throws IOException { + index("{\"test\":\"test\"}", "{\"test\":\"test\"}"); + + Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT); + request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}", + ContentType.APPLICATION_JSON)); + expectBadRequest(() -> { + client().performRequest(request); + return Collections.emptyMap(); + }, containsString("unknown field [columnar], parser not found")); + } protected void expectBadRequest(CheckedSupplier, Exception> code, Matcher errorMessageMatcher) { try { @@ -303,11 +365,25 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe } private Map runSql(String mode, String sql) throws IOException { - return runSql(mode, sql, StringUtils.EMPTY); + return runSql(mode, sql, StringUtils.EMPTY, randomBoolean()); + } + + private Map runSql(String mode, String sql, boolean columnar) throws IOException { + return runSql(mode, sql, StringUtils.EMPTY, columnar); } - private Map runSql(String mode, String sql, String suffix) throws IOException { - return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), suffix); + private Map runSql(String mode, String sql, String suffix, boolean columnar) throws IOException { + // put an explicit "columnar": false parameter or omit it altogether, it should make no difference + return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + columnarParameter(columnar) + "}", + ContentType.APPLICATION_JSON), suffix); + } + + private String columnarParameter(boolean columnar) { + if (columnar == false && randomBoolean()) { + return ""; + } else { + return ",\"columnar\":" + columnar; + } } protected Map runSql(HttpEntity sql, String suffix) throws IOException { @@ -334,8 +410,9 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe public void testBasicTranslateQuery() throws IOException { index("{\"test\":\"test\"}", "{\"test\":\"test\"}"); - - Map response = runSql(randomMode(), "SELECT * FROM test", "/translate/"); + + Map response = runSql(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}", + ContentType.APPLICATION_JSON), "/translate/"); assertEquals(1000, response.get("size")); @SuppressWarnings("unchecked") Map source = (Map) response.get("_source"); @@ -359,6 +436,7 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe public void testBasicQueryWithParameters() throws IOException { String mode = randomMode(); + boolean columnar = randomBoolean(); index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}"); @@ -367,10 +445,14 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0), columnInfo(mode, "param", "integer", JDBCType.INTEGER, 11) )); - expected.put("rows", singletonList(Arrays.asList("foo", 10))); + if (columnar) { + expected.put("values", Arrays.asList(singletonList("foo"), singletonList(10))); + } else { + expected.put("rows", Arrays.asList(Arrays.asList("foo", 10))); + } assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT test, ? param FROM test WHERE test = ?\", " + "\"params\":[{\"type\": \"integer\", \"value\": 10}, {\"type\": \"keyword\", \"value\": \"foo\"}]" - + mode(mode) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY)); + + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY)); } public void testBasicTranslateQueryWithFilter() throws IOException { diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java index 60c7b66352c..7eddd7496cc 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.action; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -29,21 +30,30 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; */ public class SqlQueryRequest extends AbstractSqlQueryRequest { private static final ObjectParser PARSER = objectParser(SqlQueryRequest::new); + static final ParseField COLUMNAR = new ParseField("columnar"); static { PARSER.declareString(SqlQueryRequest::cursor, CURSOR); + PARSER.declareBoolean(SqlQueryRequest::columnar, COLUMNAR); } private String cursor = ""; + /* + * Using the Boolean object here so that SqlTranslateRequest to set this to null (since it doesn't need a "columnar" parameter). + * See {@code SqlTranslateRequest.toXContent} + */ + private Boolean columnar = Boolean.FALSE; public SqlQueryRequest() { super(); } public SqlQueryRequest(String query, List params, QueryBuilder filter, ZoneId zoneId, - int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, String cursor, RequestInfo requestInfo) { + int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, Boolean columnar, + String cursor, RequestInfo requestInfo) { super(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo); this.cursor = cursor; + this.columnar = columnar; } @Override @@ -74,26 +84,44 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest { this.cursor = cursor; return this; } + + /** + * Should format the values in a columnar fashion or not (default false). + * Depending on the format used (csv, tsv, txt, json etc) this setting will be taken into + * consideration or not, depending on whether it even makes sense for that specific format or not. + */ + public Boolean columnar() { + return columnar; + } + + public SqlQueryRequest columnar(boolean columnar) { + this.columnar = columnar; + return this; + } public SqlQueryRequest(StreamInput in) throws IOException { super(in); cursor = in.readString(); + columnar = in.readOptionalBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(cursor); + out.writeOptionalBoolean(columnar); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cursor); + return Objects.hash(super.hashCode(), cursor, columnar); } @Override public boolean equals(Object obj) { - return super.equals(obj) && Objects.equals(cursor, ((SqlQueryRequest) obj).cursor); + return super.equals(obj) + && Objects.equals(cursor, ((SqlQueryRequest) obj).cursor) + && Objects.equals(columnar, ((SqlQueryRequest) obj).columnar); } @Override @@ -105,7 +133,7 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // This is needed just to test round-trip compatibility with proto.SqlQueryRequest return new org.elasticsearch.xpack.sql.proto.SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), - pageTimeout(), filter(), cursor(), requestInfo()).toXContent(builder, params); + pageTimeout(), filter(), columnar(), cursor(), requestInfo()).toXContent(builder, params); } public static SqlQueryRequest fromXContent(XContentParser parser) { diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java index 5443f09c5eb..acd3d860371 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java @@ -25,14 +25,14 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder params, QueryBuilder filter, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, - TimeValue pageTimeout, String nextPageInfo, RequestInfo requestInfo) { - super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, nextPageInfo, - requestInfo)); + TimeValue pageTimeout, boolean columnar, String nextPageInfo, RequestInfo requestInfo) { + super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, columnar, + nextPageInfo, requestInfo)); } public SqlQueryRequestBuilder query(String query) { @@ -74,6 +74,11 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder columns; // TODO investigate reusing Page here - it probably is much more efficient private List> rows; @@ -42,9 +43,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject public SqlQueryResponse() { } - public SqlQueryResponse(String cursor, Mode mode, @Nullable List columns, List> rows) { + public SqlQueryResponse(String cursor, Mode mode, boolean columnar, @Nullable List columns, List> rows) { this.cursor = cursor; this.mode = mode; + this.columnar = columnar; this.columns = columns; this.rows = rows; } @@ -64,6 +66,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject public List columns() { return columns; } + + public boolean columnar() { + return columnar; + } public List> rows() { return rows; @@ -150,15 +156,35 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject } builder.endArray(); } - builder.startArray("rows"); - for (List row : rows()) { - builder.startArray(); - for (Object value : row) { - value(builder, mode, value); + + if (columnar) { + // columns can be specified (for the first REST request for example), or not (on a paginated/cursor based request) + // if the columns are missing, we take the first rows' size as the number of columns + long columnsCount = columns != null ? columns.size() : 0; + if (size() > 0) { + columnsCount = rows().get(0).size(); + } + + builder.startArray("values"); + for (int index = 0; index < columnsCount; index++) { + builder.startArray(); + for (List row : rows()) { + value(builder, mode, row.get(index)); + } + builder.endArray(); + } + builder.endArray(); + } else { + builder.startArray("rows"); + for (List row : rows()) { + builder.startArray(); + for (Object value : row) { + value(builder, mode, value); + } + builder.endArray(); } builder.endArray(); } - builder.endArray(); if (cursor.equals("") == false) { builder.field(CURSOR.getPreferredName(), cursor); diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java index 5b9f21105c7..537b4048070 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java @@ -65,7 +65,7 @@ public class SqlTranslateRequest extends AbstractSqlQueryRequest { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // This is needed just to test parsing of SqlTranslateRequest, so we can reuse SqlQuerySerialization return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), - pageTimeout(), filter(), null, requestInfo()).toXContent(builder, params); + pageTimeout(), filter(), null, null, requestInfo()).toXContent(builder, params); } } diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java index 8b8d7ede7a6..f935d1d0d75 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java @@ -54,8 +54,8 @@ public class SqlQueryRequestTests extends AbstractSerializingTestCase request.requestTimeout(randomValueOtherThan(request.requestTimeout(), this::randomTV)), request -> request.filter(randomValueOtherThan(request.filter(), () -> request.filter() == null ? randomFilter(random()) : randomFilterOrNull(random()))), + request -> request.columnar(randomValueOtherThan(request.columnar(), () -> randomBoolean())), request -> request.cursor(randomValueOtherThan(request.cursor(), SqlQueryResponseTests::randomStringCursor)) ); SqlQueryRequest newRequest = new SqlQueryRequest(instance.query(), instance.params(), instance.filter(), - instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.cursor(), - instance.requestInfo()); + instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.columnar(), + instance.cursor(), instance.requestInfo()); mutator.accept(newRequest); return newRequest; } diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java index 26a3c8189c5..966e16e4057 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java @@ -36,10 +36,10 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase columns = null; @@ -55,6 +55,12 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase(rowCount); for (int r = 0; r < rowCount; r++) { List row = new ArrayList<>(rowCount); @@ -65,12 +71,11 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase null)); row.add(value.get()); - } rows.add(row); } } - return new SqlQueryResponse(cursor, mode, columns, rows); + return new SqlQueryResponse(cursor, mode, false, columns, rows); } @Override @@ -100,7 +105,13 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase rows = ((List) rootMap.get("rows")); + List rows; + if (testInstance.columnar()) { + rows = ((List) rootMap.get("values")); + } else { + rows = ((List) rootMap.get("rows")); + } + assertNotNull(rows); assertThat(rows, hasSize(testInstance.rows().size())); for (int i = 0; i < rows.size(); i++) { List row = (List) rows.get(i); @@ -116,6 +127,6 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase params; public SqlQueryRequest(String query, List params, ZoneId zoneId, int fetchSize, - TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, String cursor, RequestInfo requestInfo) { + TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, Boolean columnar, + String cursor, RequestInfo requestInfo) { super(requestInfo); this.query = query; this.params = params; @@ -43,17 +45,18 @@ public class SqlQueryRequest extends AbstractSqlRequest { this.requestTimeout = requestTimeout; this.pageTimeout = pageTimeout; this.filter = filter; + this.columnar = columnar; this.cursor = cursor; } public SqlQueryRequest(String query, List params, ToXContent filter, ZoneId zoneId, - int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { - this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, null, requestInfo); + int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, boolean columnar, RequestInfo requestInfo) { + this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, null, requestInfo); } public SqlQueryRequest(String cursor, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { this("", Collections.emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, requestTimeout, pageTimeout, - null, cursor, requestInfo); + null, false, cursor, requestInfo); } /** @@ -113,6 +116,14 @@ public class SqlQueryRequest extends AbstractSqlRequest { public ToXContent filter() { return filter; } + + /** + * Optional setting for returning the result values in a columnar fashion (as opposed to rows of values). + * Each column will have all its values in a list. Defaults to false. + */ + public Boolean columnar() { + return columnar; + } @Override public boolean equals(Object o) { @@ -133,12 +144,13 @@ public class SqlQueryRequest extends AbstractSqlRequest { Objects.equals(requestTimeout, that.requestTimeout) && Objects.equals(pageTimeout, that.pageTimeout) && Objects.equals(filter, that.filter) && + Objects.equals(columnar, that.columnar) && Objects.equals(cursor, that.cursor); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, cursor); + return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, cursor); } @Override @@ -173,6 +185,9 @@ public class SqlQueryRequest extends AbstractSqlRequest { builder.field("filter"); filter.toXContent(builder, params); } + if (columnar != null) { + builder.field("columnar", columnar); + } if (cursor != null) { builder.field("cursor", cursor); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index d4d4ce16fb8..6ba8219c2b5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -104,6 +104,13 @@ public class RestSqlQueryAction extends BaseRestHandler { TextFormat textFormat = TextFormat.fromMediaTypeOrFormat(accept); + // if we reached this point, the format to be used can be one of TXT, CSV or TSV + // which won't work in a columnar fashion + if (sqlRequest.columnar()) { + throw new IllegalArgumentException("Invalid use of [columnar] argument: cannot be used in combination with " + + "txt, csv or tsv formats"); + } + long startNanos = System.nanoTime(); return channel -> client.execute(SqlQueryAction.INSTANCE, sqlRequest, new RestResponseListener(channel) { @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index 49c296a5105..e875bb37e9d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -79,7 +79,8 @@ public class TransportSqlQueryAction extends HandledTransportAction listener.onResponse(createResponse(request, rowSet)), listener::onFailure)); } else { planExecutor.nextPage(cfg, Cursors.decodeFromString(request.cursor()), - wrap(rowSet -> listener.onResponse(createResponse(request.mode(), rowSet, null)), listener::onFailure)); + wrap(rowSet -> listener.onResponse(createResponse(request.mode(), request.columnar(), rowSet, null)), + listener::onFailure)); } } @@ -93,10 +94,10 @@ public class TransportSqlQueryAction extends HandledTransportAction columns) { + static SqlQueryResponse createResponse(Mode mode, boolean columnar, RowSet rowSet, List columns) { List> rows = new ArrayList<>(); rowSet.forEachRow(rowView -> { List row = new ArrayList<>(rowView.columnCount()); @@ -107,6 +108,7 @@ public class TransportSqlQueryAction extends HandledTransportAction