honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: introduce the columnar option for REST requests (#39287) 

* Add "columnar" option for REST requests (but be lenient for non-"plain"
modes) for json, yaml, smile and cbor formats.
* Updated documentation

(cherry picked from commit 5b7e0de237fb514d14a61a347bc669d4b4adbe56)
This commit is contained in:
Andrei Stefan 2019-02-27 09:24:52 +02:00 committed by Andrei Stefan
parent d955375e9c
commit 4deb69e9e4
19 changed files with 335 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
docs/reference/sql/endpoints/rest.asciidoc
View File

@ -133,7 +133,7 @@ Which returns:
[float]
=== Paginating through a large response
Using the example above, onu can continue to the next page by sending back the `cursor` field. In
Using the example above, one can continue to the next page by sending back the `cursor` field. In
case of text format the cursor is returned as `Cursor` http header.
[source,js]
@ -235,6 +235,81 @@ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T
// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
// TESTRESPONSE[_cat]
[[sql-rest-columnar]]
[float]
=== Columnar results
The most well known way of displaying the results of an SQL query result in general is the one where each
individual record/document represents one line/row. For certain formats, {es-sql} can return the results
in a columnar fashion: one row represents all the values of a certain column from the current page of results.
The following formats can be returned in columnar orientation: `json`, `yaml`, `cbor` and `smile`.
[source,js]
--------------------------------------------------
POST /_sql?format=json
{
"query": "SELECT * FROM library ORDER BY page_count DESC",
"fetch_size": 5,
"columnar": true
}
--------------------------------------------------
// CONSOLE
// TEST[setup:library]
Which returns:
[source,js]
--------------------------------------------------
{
"columns": [
{"name": "author", "type": "text"},
{"name": "name", "type": "text"},
{"name": "page_count", "type": "short"},
{"name": "release_date", "type": "datetime"}
],
"values": [
["Peter F. Hamilton", "Vernor Vinge", "Frank Herbert", "Alastair Reynolds", "James S.A. Corey"],
["Pandora's Star", "A Fire Upon the Deep", "Dune", "Revelation Space", "Leviathan Wakes"],
[768, 613, 604, 585, 561],
["2004-03-02T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1965-06-01T00:00:00.000Z", "2000-03-15T00:00:00.000Z", "2011-06-02T00:00:00.000Z"]
],
"cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8="
}
--------------------------------------------------
// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
Any subsequent calls using a `cursor` still have to contain the `columnar` parameter to preserve the orientation,
meaning the initial query will not _remember_ the columnar option.
[source,js]
--------------------------------------------------
POST /_sql?format=json
{
"cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=",
"columnar": true
}
--------------------------------------------------
// CONSOLE
// TEST[continued]
// TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
Which looks like:
[source,js]
--------------------------------------------------
{
"values": [
["Dan Simmons", "Iain M. Banks", "Neal Stephenson", "Frank Herbert", "Frank Herbert"],
["Hyperion", "Consider Phlebas", "Snow Crash", "God Emperor of Dune", "Children of Dune"],
[482, 471, 470, 454, 408],
["1989-05-26T00:00:00.000Z", "1987-04-23T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1981-05-28T00:00:00.000Z", "1976-04-21T00:00:00.000Z"]
],
"cursor": "46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3/////DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP"
}
--------------------------------------------------
// TESTRESPONSE[s/46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3\/\/\/\/\/DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP/$body.cursor/]
[[sql-rest-fields]]
[float]
=== Supported REST parameters
@ -277,7 +352,11 @@ s|Description
|Time-zone in ISO 8601 used for executing the query on the server.
More information available https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html[here].
|columnar
|false
|Return the results in a columnar fashion, rather than row-based fashion. Valid for `json`, `yaml`, `cbor` and `smile`.
|===
Do note that most parameters (outside the timeout ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.
Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.
That's because the query has already been executed and the calls are simply about returning the found results - thus the parameters are simply ignored.

2
x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
View File

@ -53,7 +53,7 @@ class JdbcHttpClient {
SqlQueryRequest sqlRequest = new SqlQueryRequest(sql, params, null, Protocol.TIME_ZONE,
fetch,
TimeValue.timeValueMillis(meta.timeoutInMs()), TimeValue.timeValueMillis(meta.queryTimeoutInMs()),
new RequestInfo(Mode.JDBC));
false, new RequestInfo(Mode.JDBC));
SqlQueryResponse response = httpClient.query(sqlRequest);
return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta);
}

1
x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
View File

@ -105,7 +105,6 @@ public class UserFunctionIT extends ESRestTestCase {
assertResponse(expected, actual);
}
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35980")
public void testSingleRandomUserWithWhereEvaluatingFalse() throws IOException {
index("{\"test\":\"doc1\"}",
"{\"test\":\"doc2\"}",

12
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
View File

@ -121,7 +121,8 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
@SuppressWarnings({ "unchecked" })
private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize, Mode mode)
throws IOException {
Map<String, Object> response = runSql(mode.toString(), sql);
boolean columnar = randomBoolean();
Map<String, Object> response = runSql(mode.toString(), sql, columnar);
List<Object> columns = (ArrayList<Object>) response.get("columns");
assertEquals(1, columns.size());
@ -135,7 +136,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
assertEquals(2, column.size());
}
List<Object> rows = (ArrayList<Object>) response.get("rows");
List<Object> rows = (ArrayList<Object>) response.get(columnar == true ? "values" : "rows");
assertEquals(1, rows.size());
List<Object> row = (ArrayList<Object>) rows.get(0);
assertEquals(1, row.size());
@ -149,7 +150,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
}
}
private Map<String, Object> runSql(String mode, String sql) throws IOException {
private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}";
String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT);
@ -177,6 +178,11 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
options.addHeader("Accept", randomFrom("*/*", "application/" + format));
request.setOptions(options);
}
if ((false == columnar && randomBoolean()) || columnar) {
// randomly set the "columnar" parameter, either "true" (non-default) or explicit "false" (the default anyway)
requestContent = new StringBuilder(requestContent)
.insert(requestContent.length() - 1, ",\"columnar\":" + columnar).toString();
}
// send the query either as body or as request parameter
if (randomBoolean()) {

134
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.sql.qa.rest;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.containsString;
public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTestCase {
public static String SQL_QUERY_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
private static String SQL_TRANSLATE_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;
/**
* Builds that map that is returned in the header for each column.
*/
@ -70,9 +72,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
Map<String, Object> expected = new HashMap<>();
String mode = randomMode();
boolean columnar = randomBoolean();
expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0)));
expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test")));
assertResponse(expected, runSql(mode, "SELECT * FROM test"));
if (columnar) {
expected.put("values", singletonList(Arrays.asList("test", "test")));
} else {
expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test")));
}
assertResponse(expected, runSql(mode, "SELECT * FROM test", columnar));
}
public void testNextPage() throws IOException {
@ -86,14 +94,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
}
request.setJsonEntity(bulk.toString());
client().performRequest(request);
boolean columnar = randomBoolean();
String sqlRequest =
"{\"query\":\""
+ " SELECT text, number, SQRT(number) AS s, SCORE()"
+ " FROM test"
+ " ORDER BY number, SCORE()\", "
+ "\"mode\":\"" + mode + "\", "
+ "\"fetch_size\":2}";
+ "\"fetch_size\":2" + columnarParameter(columnar) + "}";
String cursor = null;
for (int i = 0; i < 20; i += 2) {
@ -101,7 +110,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
if (i == 0) {
response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "");
} else {
response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
columnar = randomBoolean();
response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
ContentType.APPLICATION_JSON), StringUtils.EMPTY);
}
@ -113,32 +123,52 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
columnInfo(mode, "s", "double", JDBCType.DOUBLE, 25),
columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)));
}
expected.put("rows", Arrays.asList(
Arrays.asList("text" + i, i, Math.sqrt(i), 1.0),
Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0)));
if (columnar) {
expected.put("values", Arrays.asList(
Arrays.asList("text" + i, "text" + (i + 1)),
Arrays.asList(i, i + 1),
Arrays.asList(Math.sqrt(i), Math.sqrt(i + 1)),
Arrays.asList(1.0, 1.0)));
} else {
expected.put("rows", Arrays.asList(
Arrays.asList("text" + i, i, Math.sqrt(i), 1.0),
Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0)));
}
cursor = (String) response.remove("cursor");
assertResponse(expected, response);
assertNotNull(cursor);
}
Map<String, Object> expected = new HashMap<>();
expected.put("rows", emptyList());
assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
columnar = randomBoolean();
if (columnar) {
expected.put("values", emptyList());
} else {
expected.put("rows", emptyList());
}
assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
ContentType.APPLICATION_JSON), StringUtils.EMPTY));
}
@AwaitsFix(bugUrl = "Unclear status, https://github.com/elastic/x-pack-elasticsearch/issues/2074")
public void testTimeZone() throws IOException {
String mode = randomMode();
boolean columnar = randomBoolean();
index("{\"test\":\"2017-07-27 00:00:00\"}",
"{\"test\":\"2017-07-27 01:00:00\"}");
Map<String, Object> expected = new HashMap<>();
expected.put("columns", singletonMap("test", singletonMap("type", "text")));
expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
if (columnar) {
expected.put("values", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
} else {
// TODO: what exactly is this test suppossed to do. We need to check the 2074 issue above.
expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
}
expected.put("size", 2);
// Default TimeZone is UTC
assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test"));
assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test", columnar));
}
public void testScoreWithFieldNamedScore() throws IOException {
@ -152,15 +182,19 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
client().performRequest(request);
Map<String, Object> expected = new HashMap<>();
boolean columnar = randomBoolean();
expected.put("columns", Arrays.asList(
columnInfo(mode, "name", "text", JDBCType.VARCHAR, 0),
columnInfo(mode, "score", "long", JDBCType.BIGINT, 20),
columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)));
expected.put("rows", singletonList(Arrays.asList(
"test", 10, 1.0)));
assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()"));
assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()"));
if (columnar) {
expected.put("values", Arrays.asList(singletonList("test"), singletonList(10), singletonList(1.0)));
} else {
expected.put("rows", singletonList(Arrays.asList("test", 10, 1.0)));
}
assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()", columnar));
assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()", columnar));
}
public void testSelectWithJoinFails() throws Exception {
@ -195,8 +229,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
public void testSelectWhereExistsFails() throws Exception {
index("{\"foo\":1}", "{\"foo\":2}");
expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)"),
containsString("line 1:28: EXISTS is not yet supported"));
expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)",
randomBoolean()), containsString("line 1:28: EXISTS is not yet supported"));
}
@ -280,6 +314,34 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"),
containsString("line 1:12: [SCORE()] cannot be an argument to a function"));
}
public void testUseColumnarForUnsupportedFormats() throws Exception {
String format = randomFrom("txt", "csv", "tsv");
index("{\"foo\":1}");
Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
request.addParameter("error_trace", "true");
request.addParameter("pretty", "true");
request.addParameter("format", format);
request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
ContentType.APPLICATION_JSON));
expectBadRequest(() -> {
client().performRequest(request);
return Collections.emptyMap();
}, containsString("Invalid use of [columnar] argument: cannot be used in combination with txt, csv or tsv formats"));
}
public void testUseColumnarForTranslateRequest() throws IOException {
index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT);
request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
ContentType.APPLICATION_JSON));
expectBadRequest(() -> {
client().performRequest(request);
return Collections.emptyMap();
}, containsString("unknown field [columnar], parser not found"));
}
protected void expectBadRequest(CheckedSupplier<Map<String, Object>, Exception> code, Matcher<String> errorMessageMatcher) {
try {
@ -303,11 +365,25 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
}
private Map<String, Object> runSql(String mode, String sql) throws IOException {
return runSql(mode, sql, StringUtils.EMPTY);
return runSql(mode, sql, StringUtils.EMPTY, randomBoolean());
}
private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
return runSql(mode, sql, StringUtils.EMPTY, columnar);
}
private Map<String, Object> runSql(String mode, String sql, String suffix) throws IOException {
return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), suffix);
private Map<String, Object> runSql(String mode, String sql, String suffix, boolean columnar) throws IOException {
// put an explicit "columnar": false parameter or omit it altogether, it should make no difference
return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + columnarParameter(columnar) + "}",
ContentType.APPLICATION_JSON), suffix);
}
private String columnarParameter(boolean columnar) {
if (columnar == false && randomBoolean()) {
return "";
} else {
return ",\"columnar\":" + columnar;
}
}
protected Map<String, Object> runSql(HttpEntity sql, String suffix) throws IOException {
@ -334,8 +410,9 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
public void testBasicTranslateQuery() throws IOException {
index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
Map<String, Object> response = runSql(randomMode(), "SELECT * FROM test", "/translate/");
Map<String, Object> response = runSql(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
ContentType.APPLICATION_JSON), "/translate/");
assertEquals(1000, response.get("size"));
@SuppressWarnings("unchecked")
Map<String, Object> source = (Map<String, Object>) response.get("_source");
@ -359,6 +436,7 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
public void testBasicQueryWithParameters() throws IOException {
String mode = randomMode();
boolean columnar = randomBoolean();
index("{\"test\":\"foo\"}",
"{\"test\":\"bar\"}");
@ -367,10 +445,14 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0),
columnInfo(mode, "param", "integer", JDBCType.INTEGER, 11)
));
expected.put("rows", singletonList(Arrays.asList("foo", 10)));
if (columnar) {
expected.put("values", Arrays.asList(singletonList("foo"), singletonList(10)));
} else {
expected.put("rows", Arrays.asList(Arrays.asList("foo", 10)));
}
assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT test, ? param FROM test WHERE test = ?\", " +
"\"params\":[{\"type\": \"integer\", \"value\": 10}, {\"type\": \"keyword\", \"value\": \"foo\"}]"
+ mode(mode) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY));
+ mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY));
}
public void testBasicTranslateQueryWithFilter() throws IOException {

36
x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.sql.action;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -29,21 +30,30 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
*/
public class SqlQueryRequest extends AbstractSqlQueryRequest {
private static final ObjectParser<SqlQueryRequest, Void> PARSER = objectParser(SqlQueryRequest::new);
static final ParseField COLUMNAR = new ParseField("columnar");
static {
PARSER.declareString(SqlQueryRequest::cursor, CURSOR);
PARSER.declareBoolean(SqlQueryRequest::columnar, COLUMNAR);
}
private String cursor = "";
/*
* Using the Boolean object here so that SqlTranslateRequest to set this to null (since it doesn't need a "columnar" parameter).
* See {@code SqlTranslateRequest.toXContent}
*/
private Boolean columnar = Boolean.FALSE;
public SqlQueryRequest() {
super();
}
public SqlQueryRequest(String query, List<SqlTypedParamValue> params, QueryBuilder filter, ZoneId zoneId,
int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, String cursor, RequestInfo requestInfo) {
int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, Boolean columnar,
String cursor, RequestInfo requestInfo) {
super(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo);
this.cursor = cursor;
this.columnar = columnar;
}
@Override
@ -74,26 +84,44 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest {
this.cursor = cursor;
return this;
}
/**
* Should format the values in a columnar fashion or not (default false).
* Depending on the format used (csv, tsv, txt, json etc) this setting will be taken into
* consideration or not, depending on whether it even makes sense for that specific format or not.
*/
public Boolean columnar() {
return columnar;
}
public SqlQueryRequest columnar(boolean columnar) {
this.columnar = columnar;
return this;
}
public SqlQueryRequest(StreamInput in) throws IOException {
super(in);
cursor = in.readString();
columnar = in.readOptionalBoolean();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(cursor);
out.writeOptionalBoolean(columnar);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), cursor);
return Objects.hash(super.hashCode(), cursor, columnar);
}
@Override
public boolean equals(Object obj) {
return super.equals(obj) && Objects.equals(cursor, ((SqlQueryRequest) obj).cursor);
return super.equals(obj)
&& Objects.equals(cursor, ((SqlQueryRequest) obj).cursor)
&& Objects.equals(columnar, ((SqlQueryRequest) obj).columnar);
}
@Override
@ -105,7 +133,7 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// This is needed just to test round-trip compatibility with proto.SqlQueryRequest
return new org.elasticsearch.xpack.sql.proto.SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(),
pageTimeout(), filter(), cursor(), requestInfo()).toXContent(builder, params);
pageTimeout(), filter(), columnar(), cursor(), requestInfo()).toXContent(builder, params);
}
public static SqlQueryRequest fromXContent(XContentParser parser) {

13
x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
View File

@ -25,14 +25,14 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder<SqlQueryRequest
public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action) {
this(client, action, "", Collections.emptyList(), null, Protocol.TIME_ZONE, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT,
Protocol.PAGE_TIMEOUT, "", new RequestInfo(Mode.PLAIN));
Protocol.PAGE_TIMEOUT, false, "", new RequestInfo(Mode.PLAIN));
}
public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action, String query, List<SqlTypedParamValue> params,
QueryBuilder filter, ZoneId zoneId, int fetchSize, TimeValue requestTimeout,
TimeValue pageTimeout, String nextPageInfo, RequestInfo requestInfo) {
super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, nextPageInfo,
requestInfo));
TimeValue pageTimeout, boolean columnar, String nextPageInfo, RequestInfo requestInfo) {
super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, columnar,
nextPageInfo, requestInfo));
}
public SqlQueryRequestBuilder query(String query) {
@ -74,6 +74,11 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder<SqlQueryRequest
request.pageTimeout(timeout);
return this;
}
public SqlQueryRequestBuilder columnar(boolean columnar) {
request.columnar(columnar);
return this;
}
public SqlQueryRequestBuilder fetchSize(int fetchSize) {
request.fetchSize(fetchSize);

40
x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java
View File

@ -34,6 +34,7 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
// TODO: Simplify cursor handling
private String cursor;
private Mode mode;
private boolean columnar;
private List<ColumnInfo> columns;
// TODO investigate reusing Page here - it probably is much more efficient
private List<List<Object>> rows;
@ -42,9 +43,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
public SqlQueryResponse() {
}
public SqlQueryResponse(String cursor, Mode mode, @Nullable List<ColumnInfo> columns, List<List<Object>> rows) {
public SqlQueryResponse(String cursor, Mode mode, boolean columnar, @Nullable List<ColumnInfo> columns, List<List<Object>> rows) {
this.cursor = cursor;
this.mode = mode;
this.columnar = columnar;
this.columns = columns;
this.rows = rows;
}
@ -64,6 +66,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
public List<ColumnInfo> columns() {
return columns;
}
public boolean columnar() {
return columnar;
}
public List<List<Object>> rows() {
return rows;
@ -150,15 +156,35 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
}
builder.endArray();
}
builder.startArray("rows");
for (List<Object> row : rows()) {
builder.startArray();
for (Object value : row) {
value(builder, mode, value);
if (columnar) {
// columns can be specified (for the first REST request for example), or not (on a paginated/cursor based request)
// if the columns are missing, we take the first rows' size as the number of columns
long columnsCount = columns != null ? columns.size() : 0;
if (size() > 0) {
columnsCount = rows().get(0).size();
}
builder.startArray("values");
for (int index = 0; index < columnsCount; index++) {
builder.startArray();
for (List<Object> row : rows()) {
value(builder, mode, row.get(index));
}
builder.endArray();
}
builder.endArray();
} else {
builder.startArray("rows");
for (List<Object> row : rows()) {
builder.startArray();
for (Object value : row) {
value(builder, mode, value);
}
builder.endArray();
}
builder.endArray();
}
builder.endArray();
if (cursor.equals("") == false) {
builder.field(CURSOR.getPreferredName(), cursor);

2
x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
View File

@ -65,7 +65,7 @@ public class SqlTranslateRequest extends AbstractSqlQueryRequest {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// This is needed just to test parsing of SqlTranslateRequest, so we can reuse SqlQuerySerialization
return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(),
pageTimeout(), filter(), null, requestInfo()).toXContent(builder, params);
pageTimeout(), filter(), null, null, requestInfo()).toXContent(builder, params);
}
}

9
x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
View File

@ -54,8 +54,8 @@ public class SqlQueryRequestTests extends AbstractSerializingTestCase<SqlQueryRe
@Override
protected SqlQueryRequest createTestInstance() {
return new SqlQueryRequest(randomAlphaOfLength(10), randomParameters(), SqlTestUtils.randomFilterOrNull(random()),
randomZone(), between(1, Integer.MAX_VALUE),
randomTV(), randomTV(), randomAlphaOfLength(10), requestInfo
randomZone(), between(1, Integer.MAX_VALUE), randomTV(),
randomTV(), randomBoolean(), randomAlphaOfLength(10), requestInfo
);
}
@ -109,11 +109,12 @@ public class SqlQueryRequestTests extends AbstractSerializingTestCase<SqlQueryRe
request -> request.requestTimeout(randomValueOtherThan(request.requestTimeout(), this::randomTV)),
request -> request.filter(randomValueOtherThan(request.filter(),
() -> request.filter() == null ? randomFilter(random()) : randomFilterOrNull(random()))),
request -> request.columnar(randomValueOtherThan(request.columnar(), () -> randomBoolean())),
request -> request.cursor(randomValueOtherThan(request.cursor(), SqlQueryResponseTests::randomStringCursor))
);
SqlQueryRequest newRequest = new SqlQueryRequest(instance.query(), instance.params(), instance.filter(),
instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.cursor(),
instance.requestInfo());
instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.columnar(),
instance.cursor(), instance.requestInfo());
mutator.accept(newRequest);
return newRequest;
}

23
x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java
View File

@ -36,10 +36,10 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
@Override
protected SqlQueryResponse createTestInstance() {
return createRandomInstance(randomStringCursor(), randomFrom(Mode.values()));
return createRandomInstance(randomStringCursor(), randomFrom(Mode.values()), randomBoolean());
}
public static SqlQueryResponse createRandomInstance(String cursor, Mode mode) {
public static SqlQueryResponse createRandomInstance(String cursor, Mode mode, boolean columnar) {
int columnCount = between(1, 10);
List<ColumnInfo> columns = null;
@ -55,6 +55,12 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
rows = Collections.emptyList();
} else {
int rowCount = between(1, 10);
if (columnar && columns != null) {
int temp = rowCount;
rowCount = columnCount;
columnCount = temp;
}
rows = new ArrayList<>(rowCount);
for (int r = 0; r < rowCount; r++) {
List<Object> row = new ArrayList<>(rowCount);
@ -65,12 +71,11 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
ESTestCase::randomDouble,
() -> null));
row.add(value.get());
}
rows.add(row);
}
}
return new SqlQueryResponse(cursor, mode, columns, rows);
return new SqlQueryResponse(cursor, mode, false, columns, rows);
}
@Override
@ -100,7 +105,13 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
assertNull(rootMap.get("columns"));
}
List<?> rows = ((List<?>) rootMap.get("rows"));
List<?> rows;
if (testInstance.columnar()) {
rows = ((List<?>) rootMap.get("values"));
} else {
rows = ((List<?>) rootMap.get("rows"));
}
assertNotNull(rows);
assertThat(rows, hasSize(testInstance.rows().size()));
for (int i = 0; i < rows.size(); i++) {
List<?> row = (List<?>) rows.get(i);
@ -116,6 +127,6 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
protected SqlQueryResponse doParseInstance(XContentParser parser) {
org.elasticsearch.xpack.sql.proto.SqlQueryResponse response =
org.elasticsearch.xpack.sql.proto.SqlQueryResponse.fromXContent(parser);
return new SqlQueryResponse(response.cursor(), Mode.JDBC, response.columns(), response.rows());
return new SqlQueryResponse(response.cursor(), Mode.JDBC, false, response.columns(), response.rows());
}
}

2
x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
View File

@ -36,7 +36,7 @@ public class SqlTranslateRequestTests extends AbstractSerializingTestCase<SqlTra
@Override
protected SqlTranslateRequest createTestInstance() {
return new SqlTranslateRequest(randomAlphaOfLength(10), Collections.emptyList(), randomFilterOrNull(random()),
return new SqlTranslateRequest(randomAlphaOfLength(10), Collections.emptyList(), randomFilterOrNull(random()),
randomZone(), between(1, Integer.MAX_VALUE), randomTV(), randomTV(), new RequestInfo(testMode));
}

2
x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
View File

@ -66,7 +66,7 @@ public class HttpClient {
// method called only from CLI
SqlQueryRequest sqlRequest = new SqlQueryRequest(query, Collections.emptyList(), null, ZoneId.of("Z"),
fetchSize, TimeValue.timeValueMillis(cfg.queryTimeout()), TimeValue.timeValueMillis(cfg.pageTimeout()),
new RequestInfo(Mode.CLI));
false, new RequestInfo(Mode.CLI));
return query(sqlRequest);
}

25
x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
View File

@ -30,11 +30,13 @@ public class SqlQueryRequest extends AbstractSqlRequest {
private final TimeValue pageTimeout;
@Nullable
private final ToXContent filter;
private final Boolean columnar;
private final List<SqlTypedParamValue> params;
public SqlQueryRequest(String query, List<SqlTypedParamValue> params, ZoneId zoneId, int fetchSize,
TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, String cursor, RequestInfo requestInfo) {
TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, Boolean columnar,
String cursor, RequestInfo requestInfo) {
super(requestInfo);
this.query = query;
this.params = params;
@ -43,17 +45,18 @@ public class SqlQueryRequest extends AbstractSqlRequest {
this.requestTimeout = requestTimeout;
this.pageTimeout = pageTimeout;
this.filter = filter;
this.columnar = columnar;
this.cursor = cursor;
}
public SqlQueryRequest(String query, List<SqlTypedParamValue> params, ToXContent filter, ZoneId zoneId,
int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) {
this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, null, requestInfo);
int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, boolean columnar, RequestInfo requestInfo) {
this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, null, requestInfo);
}
public SqlQueryRequest(String cursor, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) {
this("", Collections.emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, requestTimeout, pageTimeout,
null, cursor, requestInfo);
null, false, cursor, requestInfo);
}
/**
@ -113,6 +116,14 @@ public class SqlQueryRequest extends AbstractSqlRequest {
public ToXContent filter() {
return filter;
}
/**
* Optional setting for returning the result values in a columnar fashion (as opposed to rows of values).
* Each column will have all its values in a list. Defaults to false.
*/
public Boolean columnar() {
return columnar;
}
@Override
public boolean equals(Object o) {
@ -133,12 +144,13 @@ public class SqlQueryRequest extends AbstractSqlRequest {
Objects.equals(requestTimeout, that.requestTimeout) &&
Objects.equals(pageTimeout, that.pageTimeout) &&
Objects.equals(filter, that.filter) &&
Objects.equals(columnar, that.columnar) &&
Objects.equals(cursor, that.cursor);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, cursor);
return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, cursor);
}
@Override
@ -173,6 +185,9 @@ public class SqlQueryRequest extends AbstractSqlRequest {
builder.field("filter");
filter.toXContent(builder, params);
}
if (columnar != null) {
builder.field("columnar", columnar);
}
if (cursor != null) {
builder.field("cursor", cursor);
}

7
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
View File

@ -104,6 +104,13 @@ public class RestSqlQueryAction extends BaseRestHandler {
TextFormat textFormat = TextFormat.fromMediaTypeOrFormat(accept);
// if we reached this point, the format to be used can be one of TXT, CSV or TSV
// which won't work in a columnar fashion
if (sqlRequest.columnar()) {
throw new IllegalArgumentException("Invalid use of [columnar] argument: cannot be used in combination with "
+ "txt, csv or tsv formats");
}
long startNanos = System.nanoTime();
return channel -> client.execute(SqlQueryAction.INSTANCE, sqlRequest, new RestResponseListener<SqlQueryResponse>(channel) {
@Override

8
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
View File

@ -79,7 +79,8 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
wrap(rowSet -> listener.onResponse(createResponse(request, rowSet)), listener::onFailure));
} else {
planExecutor.nextPage(cfg, Cursors.decodeFromString(request.cursor()),
wrap(rowSet -> listener.onResponse(createResponse(request.mode(), rowSet, null)), listener::onFailure));
wrap(rowSet -> listener.onResponse(createResponse(request.mode(), request.columnar(), rowSet, null)),
listener::onFailure));
}
}
@ -93,10 +94,10 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
}
}
columns = unmodifiableList(columns);
return createResponse(request.mode(), rowSet, columns);
return createResponse(request.mode(), request.columnar(), rowSet, columns);
}
static SqlQueryResponse createResponse(Mode mode, RowSet rowSet, List<ColumnInfo> columns) {
static SqlQueryResponse createResponse(Mode mode, boolean columnar, RowSet rowSet, List<ColumnInfo> columns) {
List<List<Object>> rows = new ArrayList<>();
rowSet.forEachRow(rowView -> {
List<Object> row = new ArrayList<>(rowView.columnCount());
@ -107,6 +108,7 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
return new SqlQueryResponse(
Cursors.encodeToString(Version.CURRENT, rowSet.nextPageCursor()),
mode,
columnar,
columns,
rows);
}

2
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/BasicFormatterTests.java
View File

@ -17,7 +17,7 @@ import static org.hamcrest.Matchers.arrayWithSize;
public class BasicFormatterTests extends ESTestCase {
private final FormatOption format = randomFrom(FormatOption.values());
private final SqlQueryResponse firstResponse = new SqlQueryResponse("", format == CLI ? Mode.CLI : Mode.PLAIN,
private final SqlQueryResponse firstResponse = new SqlQueryResponse("", format == CLI ? Mode.CLI : Mode.PLAIN, false,
Arrays.asList(
new ColumnInfo("", "foo", "string", 0),
new ColumnInfo("", "bar", "long", 15),

2
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CursorTests.java
View File

@ -72,7 +72,7 @@ public class CursorTests extends ESTestCase {
columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(25)));
}
}
return new SqlQueryResponse("", randomFrom(Mode.values()), columns, Collections.emptyList());
return new SqlQueryResponse("", randomFrom(Mode.values()), false, columns, Collections.emptyList());
}
@SuppressWarnings("unchecked")

6
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java
View File

@ -118,7 +118,7 @@ public class TextFormatTests extends ESTestCase {
}
private static SqlQueryResponse emptyData() {
return new SqlQueryResponse(null, Mode.JDBC, singletonList(new ColumnInfo("index", "name", "keyword")), emptyList());
return new SqlQueryResponse(null, Mode.JDBC, false, singletonList(new ColumnInfo("index", "name", "keyword")), emptyList());
}
private static SqlQueryResponse regularData() {
@ -132,7 +132,7 @@ public class TextFormatTests extends ESTestCase {
values.add(asList("Along The River Bank", 11 * 60 + 48));
values.add(asList("Mind Train", 4 * 60 + 40));
return new SqlQueryResponse(null, Mode.JDBC, headers, values);
return new SqlQueryResponse(null, Mode.JDBC, false, headers, values);
}
private static SqlQueryResponse escapedData() {
@ -146,7 +146,7 @@ public class TextFormatTests extends ESTestCase {
values.add(asList("normal", "\"quo\"ted\",\n"));
values.add(asList("commas", "a,b,c,\n,d,e,\t\n"));
return new SqlQueryResponse(null, Mode.JDBC, headers, values);
return new SqlQueryResponse(null, Mode.JDBC, false, headers, values);
}
private static RestRequest req() {