honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: Add protocol tests and remove jdbc_type from drivers response (#37516)

This commit is contained in:
Andrei Stefan 2019-01-16 16:28:46 +02:00 committed by GitHub
parent 8932750efb
commit 659326fdd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 237 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SqlProtocolIT.java Normal file
View File

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa.single_node;
import org.elasticsearch.xpack.sql.qa.SqlProtocolTestCase;
public class SqlProtocolIT extends SqlProtocolTestCase {
}

193
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java Normal file
View File

@ -0,0 +1,193 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.cbor.CborXContent;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.smile.SmileXContent;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.sql.proto.Mode;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
public abstract class SqlProtocolTestCase extends ESRestTestCase {
public void testNulls() throws IOException {
assertQuery("SELECT NULL", "NULL", "null", null, 0);
}
public void testBooleanType() throws IOException {
assertQuery("SELECT TRUE", "TRUE", "boolean", true, 1);
assertQuery("SELECT FALSE", "FALSE", "boolean", false, 1);
}
public void testNumericTypes() throws IOException {
assertQuery("SELECT CAST(3 AS TINYINT)", "CAST(3 AS TINYINT)", "byte", 3, 5);
assertQuery("SELECT CAST(-123 AS TINYINT)", "CAST(-123 AS TINYINT)", "byte", -123, 5);
assertQuery("SELECT CAST(5 AS SMALLINT)", "CAST(5 AS SMALLINT)", "short", 5, 6);
assertQuery("SELECT CAST(-25 AS SMALLINT)", "CAST(-25 AS SMALLINT)", "short", -25, 6);
assertQuery("SELECT 123", "123", "integer", 123, 11);
assertQuery("SELECT -2123", "-2123", "integer", -2123, 11);
assertQuery("SELECT 1234567890123", "1234567890123", "long", 1234567890123L, 20);
assertQuery("SELECT -1234567890123", "-1234567890123", "long", -1234567890123L, 20);
assertQuery("SELECT 1234567890123.34", "1234567890123.34", "double", 1234567890123.34, 25);
assertQuery("SELECT -1234567890123.34", "-1234567890123.34", "double", -1234567890123.34, 25);
assertQuery("SELECT CAST(1234.34 AS REAL)", "CAST(1234.34 AS REAL)", "float", 1234.34f, 15);
assertQuery("SELECT CAST(-1234.34 AS REAL)", "CAST(-1234.34 AS REAL)", "float", -1234.34f, 15);
assertQuery("SELECT CAST(1234567890123.34 AS FLOAT)", "CAST(1234567890123.34 AS FLOAT)", "double", 1234567890123.34, 25);
assertQuery("SELECT CAST(-1234567890123.34 AS FLOAT)", "CAST(-1234567890123.34 AS FLOAT)", "double", -1234567890123.34, 25);
}
public void testTextualType() throws IOException {
assertQuery("SELECT 'abc123'", "'abc123'", "keyword", "abc123", 0);
}
public void testDateTimes() throws IOException {
assertQuery("SELECT CAST('2019-01-14T12:29:25.000Z' AS DATE)", "CAST('2019-01-14T12:29:25.000Z' AS DATE)", "date",
"2019-01-14T12:29:25.000Z", 24);
assertQuery("SELECT CAST(-26853765751000 AS DATE)", "CAST(-26853765751000 AS DATE)", "date", "1119-01-15T12:37:29.000Z", 24);
assertQuery("SELECT CAST(CAST('-26853765751000' AS BIGINT) AS DATE)", "CAST(CAST('-26853765751000' AS BIGINT) AS DATE)", "date",
"1119-01-15T12:37:29.000Z", 24);
}
public void testIPs() throws IOException {
assertQuery("SELECT CAST('12.13.14.15' AS IP)", "CAST('12.13.14.15' AS IP)", "ip", "12.13.14.15", 0);
assertQuery("SELECT CAST('2001:0db8:0000:0000:0000:ff00:0042:8329' AS IP)", "CAST('2001:0db8:0000:0000:0000:ff00:0042:8329' AS IP)",
"ip", "2001:0db8:0000:0000:0000:ff00:0042:8329", 0);
}
public void testDateTimeIntervals() throws IOException {
assertQuery("SELECT INTERVAL '326' YEAR", "INTERVAL '326' YEAR", "interval_year", "P326Y", 7);
assertQuery("SELECT INTERVAL '50' MONTH", "INTERVAL '50' MONTH", "interval_month", "P50M", 7);
assertQuery("SELECT INTERVAL '520' DAY", "INTERVAL '520' DAY", "interval_day", "PT12480H", 23);
assertQuery("SELECT INTERVAL '163' HOUR", "INTERVAL '163' HOUR", "interval_hour", "PT163H", 23);
assertQuery("SELECT INTERVAL '163' MINUTE", "INTERVAL '163' MINUTE", "interval_minute", "PT2H43M", 23);
assertQuery("SELECT INTERVAL '223.16' SECOND", "INTERVAL '223.16' SECOND", "interval_second", "PT3M43.016S", 23);
assertQuery("SELECT INTERVAL '163-11' YEAR TO MONTH", "INTERVAL '163-11' YEAR TO MONTH", "interval_year_to_month", "P163Y11M", 7);
assertQuery("SELECT INTERVAL '163 12' DAY TO HOUR", "INTERVAL '163 12' DAY TO HOUR", "interval_day_to_hour", "PT3924H", 23);
assertQuery("SELECT INTERVAL '163 12:39' DAY TO MINUTE", "INTERVAL '163 12:39' DAY TO MINUTE", "interval_day_to_minute",
"PT3924H39M", 23);
assertQuery("SELECT INTERVAL '163 12:39:59.163' DAY TO SECOND", "INTERVAL '163 12:39:59.163' DAY TO SECOND",
"interval_day_to_second", "PT3924H39M59.163S", 23);
assertQuery("SELECT INTERVAL -'163 23:39:56.23' DAY TO SECOND", "INTERVAL -'163 23:39:56.23' DAY TO SECOND",
"interval_day_to_second", "PT-3935H-39M-56.023S", 23);
assertQuery("SELECT INTERVAL '163:39' HOUR TO MINUTE", "INTERVAL '163:39' HOUR TO MINUTE", "interval_hour_to_minute",
"PT163H39M", 23);
assertQuery("SELECT INTERVAL '163:39:59.163' HOUR TO SECOND", "INTERVAL '163:39:59.163' HOUR TO SECOND", "interval_hour_to_second",
"PT163H39M59.163S", 23);
assertQuery("SELECT INTERVAL '163:59.163' MINUTE TO SECOND", "INTERVAL '163:59.163' MINUTE TO SECOND", "interval_minute_to_second",
"PT2H43M59.163S", 23);
}
@SuppressWarnings({ "unchecked" })
private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize) throws IOException {
for (Mode mode : Mode.values()) {
Map<String, Object> response = runSql(mode.toString(), sql);
List<Object> columns = (ArrayList<Object>) response.get("columns");
assertEquals(1, columns.size());
Map<String, Object> column = (HashMap<String, Object>) columns.get(0);
assertEquals(columnName, column.get("name"));
assertEquals(columnType, column.get("type"));
if (mode != Mode.PLAIN) {
assertEquals(3, column.size());
assertEquals(displaySize, column.get("display_size"));
} else {
assertEquals(2, column.size());
}
List<Object> rows = (ArrayList<Object>) response.get("rows");
assertEquals(1, rows.size());
List<Object> row = (ArrayList<Object>) rows.get(0);
assertEquals(1, row.size());
// from xcontent we can get float or double, depending on the conversion
// method of the specific xcontent format implementation
if (columnValue instanceof Float && row.get(0) instanceof Double) {
assertEquals(columnValue, (float)((Number) row.get(0)).doubleValue());
} else {
assertEquals(columnValue, row.get(0));
}
}
}
private Map<String, Object> runSql(String mode, String sql) throws IOException {
Request request = new Request("POST", "/_sql");
String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}";
String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT);
// add a client_id to the request
if (randomBoolean()) {
String clientId = randomFrom(randomFrom(CLIENT_IDS), randomAlphaOfLengthBetween(10, 20));
requestContent = new StringBuilder(requestContent)
.insert(requestContent.length() - 1, ",\"client_id\":\"" + clientId + "\"").toString();
}
if (randomBoolean()) {
request.addParameter("error_trace", "true");
}
if (randomBoolean()) {
request.addParameter("pretty", "true");
}
if (!"json".equals(format) || randomBoolean()) {
// since we default to JSON if a format is not specified, randomize setting it or not, explicitly;
// for any other format, just set the format explicitly
request.addParameter("format", format);
}
if (randomBoolean()) {
// randomly use the Accept header for the response format
RequestOptions.Builder options = request.getOptions().toBuilder();
options.addHeader("Accept", randomFrom("*/*", "application/" + format));
request.setOptions(options);
}
// send the query either as body or as request parameter
if (randomBoolean()) {
request.setEntity(new StringEntity(requestContent, ContentType.APPLICATION_JSON));
} else {
request.setEntity(null);
request.addParameter("source", requestContent);
request.addParameter("source_content_type", ContentType.APPLICATION_JSON.getMimeType());
RequestOptions.Builder options = request.getOptions().toBuilder();
options.addHeader("Content-Type", "application/json");
request.setOptions(options);
}
Response response = client().performRequest(request);
try (InputStream content = response.getEntity().getContent()) {
switch(format) {
case "cbor": {
return XContentHelper.convertToMap(CborXContent.cborXContent, content, false);
}
case "yaml": {
return XContentHelper.convertToMap(YamlXContent.yamlXContent, content, false);
}
case "smile": {
return XContentHelper.convertToMap(SmileXContent.smileXContent, content, false);
}
default:
return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
}
}
}
}

2
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java
View File

@ -117,7 +117,7 @@ public abstract class JdbcTestUtils {
for (int i = 1; i <= columns; i++) {
cols.add(new ColumnInfo(metaData.getTableName(i), metaData.getColumnName(i), metaData.getColumnTypeName(i),
metaData.getColumnType(i), metaData.getColumnDisplaySize(i)));
metaData.getColumnDisplaySize(i)));
}

1
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
View File

@ -57,7 +57,6 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
column.put("name", name);
column.put("type", type);
if ("jdbc".equals(mode)) {
column.put("jdbc_type", jdbcType.getVendorTypeNumber());
column.put("display_size", size);
}
return unmodifiableMap(column);

8
x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec
View File

@ -98,6 +98,14 @@ SELECT CAST(SUM(salary) AS DOUBLE) FROM test_emp;
4824855.0
;
aggregateWithUpCastAsFloat
SELECT CAST(SUM(salary) AS FLOAT) FROM test_emp;
CAST(SUM(salary) AS FLOAT)
-----------------------------
4824855.0
;
aggregateWithCastNumericToString
SELECT CAST(AVG(salary) AS VARCHAR) FROM test_emp;

21
x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java
View File

@ -183,29 +183,16 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
String table = in.readString();
String name = in.readString();
String esType = in.readString();
Integer jdbcType;
int displaySize;
if (in.readBoolean()) {
jdbcType = in.readVInt();
displaySize = in.readVInt();
} else {
jdbcType = null;
displaySize = 0;
}
return new ColumnInfo(table, name, esType, jdbcType, displaySize);
Integer displaySize = in.readOptionalVInt();
return new ColumnInfo(table, name, esType, displaySize);
}
public static void writeColumnInfo(StreamOutput out, ColumnInfo columnInfo) throws IOException {
out.writeString(columnInfo.table());
out.writeString(columnInfo.name());
out.writeString(columnInfo.esType());
if (columnInfo.jdbcType() != null) {
out.writeBoolean(true);
out.writeVInt(columnInfo.jdbcType());
out.writeVInt(columnInfo.displaySize());
} else {
out.writeBoolean(false);
}
out.writeOptionalVInt(columnInfo.displaySize());
}
@Override

4
x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java
View File

@ -46,8 +46,7 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
if (randomBoolean()) {
columns = new ArrayList<>(columnCount);
for (int i = 0; i < columnCount; i++) {
columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10),
randomInt(), randomInt(25)));
columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(25)));
}
}
@ -96,7 +95,6 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
assertEquals(columnInfo.name(), columnMap.get("name"));
assertEquals(columnInfo.esType(), columnMap.get("type"));
assertEquals(columnInfo.displaySize(), columnMap.get("display_size"));
assertEquals(columnInfo.jdbcType(), columnMap.get("jdbc_type"));
}
} else {
assertNull(rootMap.get("columns"));

3
x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/ServerQueryCliCommandTests.java
View File

@ -12,7 +12,6 @@ import org.elasticsearch.xpack.sql.proto.ColumnInfo;
import org.elasticsearch.xpack.sql.proto.SqlQueryResponse;
import java.sql.SQLException;
import java.sql.Types;
import java.util.Collections;
import java.util.List;
@ -109,7 +108,7 @@ public class ServerQueryCliCommandTests extends ESTestCase {
List<List<Object>> rows;
List<ColumnInfo> columns;
if (includeColumns) {
columns = singletonList(new ColumnInfo("", "field", "string", Types.VARCHAR, 0));
columns = singletonList(new ColumnInfo("", "field", "string", 0));
} else {
columns = null;
}

34
x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/ColumnInfo.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.sql.proto;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
@ -21,7 +20,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optiona
/**
* Information about a column returned with first query response.
* As this represents the response for all drivers, it is important for it to be explicit about
* its structure, in particular types (using es_type and jdbc_type instead of DataType).
* its structure, in particular types (using es_type instead of DataType).
*/
public class ColumnInfo implements ToXContentObject {
@ -31,35 +30,29 @@ public class ColumnInfo implements ToXContentObject {
objects[0] == null ? "" : (String) objects[0],
(String) objects[1],
(String) objects[2],
objects[3] == null ? null : (int) objects[3],
objects[4] == null ? 0 : (int) objects[4]));
(Integer) objects[3]));
private static final ParseField TABLE = new ParseField("table");
private static final ParseField NAME = new ParseField("name");
private static final ParseField ES_TYPE = new ParseField("type");
private static final ParseField JDBC_TYPE = new ParseField("jdbc_type");
private static final ParseField DISPLAY_SIZE = new ParseField("display_size");
static {
PARSER.declareString(optionalConstructorArg(), TABLE);
PARSER.declareString(constructorArg(), NAME);
PARSER.declareString(constructorArg(), ES_TYPE);
PARSER.declareInt(optionalConstructorArg(), JDBC_TYPE);
PARSER.declareInt(optionalConstructorArg(), DISPLAY_SIZE);
}
private final String table;
private final String name;
private final String esType;
@Nullable
private final Integer jdbcType;
private final int displaySize;
private final Integer displaySize;
public ColumnInfo(String table, String name, String esType, Integer jdbcType, int displaySize) {
public ColumnInfo(String table, String name, String esType, Integer displaySize) {
this.table = table;
this.name = name;
this.esType = esType;
this.jdbcType = jdbcType;
this.displaySize = displaySize;
}
@ -67,8 +60,7 @@ public class ColumnInfo implements ToXContentObject {
this.table = table;
this.name = name;
this.esType = esType;
this.jdbcType = null;
this.displaySize = 0;
this.displaySize = null;
}
@Override
@ -79,8 +71,7 @@ public class ColumnInfo implements ToXContentObject {
}
builder.field("name", name);
builder.field("type", esType);
if (jdbcType != null) {
builder.field("jdbc_type", jdbcType);
if (displaySize != null) {
builder.field("display_size", displaySize);
}
return builder.endObject();
@ -112,13 +103,6 @@ public class ColumnInfo implements ToXContentObject {
return esType;
}
/**
* The type of the column as it would be returned by a JDBC driver.
*/
public Integer jdbcType() {
return jdbcType;
}
/**
* Used by JDBC
*/
@ -138,14 +122,12 @@ public class ColumnInfo implements ToXContentObject {
return displaySize == that.displaySize &&
Objects.equals(table, that.table) &&
Objects.equals(name, that.name) &&
Objects.equals(esType, that.esType) &&
Objects.equals(jdbcType, that.jdbcType);
Objects.equals(esType, that.esType);
}
@Override
public int hashCode() {
return Objects.hash(table, name, esType, jdbcType, displaySize);
return Objects.hash(table, name, esType, displaySize);
}
@Override

3
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
View File

@ -100,8 +100,7 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
List<ColumnInfo> columns = new ArrayList<>(rowSet.columnCount());
for (Schema.Entry entry : rowSet.schema()) {
if (Mode.isDriver(request.mode())) {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType, entry.type().sqlType.getVendorTypeNumber(),
entry.type().displaySize));
columns.add(new ColumnInfo("", entry.name(), entry.type().esType, entry.type().displaySize));
} else {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType));
}

11
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/CliFormatterTests.java
View File

@ -9,7 +9,6 @@ import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.sql.proto.ColumnInfo;
import org.elasticsearch.xpack.sql.proto.Mode;
import java.sql.Types;
import java.util.Arrays;
import static org.hamcrest.Matchers.arrayWithSize;
@ -17,11 +16,11 @@ import static org.hamcrest.Matchers.arrayWithSize;
public class CliFormatterTests extends ESTestCase {
private final SqlQueryResponse firstResponse = new SqlQueryResponse("", Mode.PLAIN,
Arrays.asList(
new ColumnInfo("", "foo", "string", Types.VARCHAR, 0),
new ColumnInfo("", "bar", "long", Types.BIGINT, 15),
new ColumnInfo("", "15charwidename!", "double", Types.DOUBLE, 25),
new ColumnInfo("", "superduperwidename!!!", "double", Types.DOUBLE, 25),
new ColumnInfo("", "baz", "keyword", Types.VARCHAR, 0)),
new ColumnInfo("", "foo", "string", 0),
new ColumnInfo("", "bar", "long", 15),
new ColumnInfo("", "15charwidename!", "double", 25),
new ColumnInfo("", "superduperwidename!!!", "double", 25),
new ColumnInfo("", "baz", "keyword", 0)),
Arrays.asList(
Arrays.asList("15charwidedata!", 1, 6.888, 12, "rabbit"),
Arrays.asList("dog", 1.7976931348623157E308, 123124.888, 9912, "goat")));

6
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
View File

@ -10,8 +10,6 @@ import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.xpack.sql.proto.ColumnInfo;
import org.elasticsearch.xpack.sql.proto.Mode;
import java.sql.Types;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
@ -35,8 +33,8 @@ public class SqlActionIT extends AbstractSqlIntegTestCase {
assertThat(response.columns(), hasSize(2));
int dataIndex = dataBeforeCount ? 0 : 1;
int countIndex = dataBeforeCount ? 1 : 0;
assertEquals(new ColumnInfo("", "data", "text", Types.VARCHAR, 0), response.columns().get(dataIndex));
assertEquals(new ColumnInfo("", "count", "long", Types.BIGINT, 20), response.columns().get(countIndex));
assertEquals(new ColumnInfo("", "data", "text", 0), response.columns().get(dataIndex));
assertEquals(new ColumnInfo("", "count", "long", 20), response.columns().get(countIndex));
assertThat(response.rows(), hasSize(2));
assertEquals("bar", response.rows().get(0).get(dataIndex));

3
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CursorTests.java
View File

@ -66,8 +66,7 @@ public class CursorTests extends ESTestCase {
if (randomBoolean()) {
columns = new ArrayList<>(columnCount);
for (int i = 0; i < columnCount; i++) {
columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10),
randomInt(), randomInt(25)));
columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(25)));
}
}
return new SqlQueryResponse("", randomFrom(Mode.values()), columns, Collections.emptyList());