diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index ca6228ac66f..978fae41039 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -24,9 +24,6 @@ Which returns: { "size" : 10, "docvalue_fields" : [ - { - "field": "page_count" - }, { "field": "release_date", "format": "epoch_millis" @@ -35,7 +32,8 @@ Which returns: "_source": { "includes": [ "author", - "name" + "name", + "page_count" ], "excludes": [] }, diff --git a/docs/reference/sql/limitations.asciidoc b/docs/reference/sql/limitations.asciidoc index 2d46d016c6b..5a97c656105 100644 --- a/docs/reference/sql/limitations.asciidoc +++ b/docs/reference/sql/limitations.asciidoc @@ -161,3 +161,21 @@ By default,`geo_points` fields are indexed and have doc values. However only lat indexed with some loss of precision from the original values (4.190951585769653E-8 for the latitude and 8.381903171539307E-8 for longitude). The altitude component is accepted but not stored in doc values nor indexed. Therefore calling `ST_Z` function in the filtering, grouping or sorting will return `null`. + +[float] +[[fields-from-source]] +=== Retrieving from `_source` + +Most of {es-sql}'s columns are retrieved from the document's `_source` and there is no attempt to get the columns content from +`docvalue_fields` not even in the case <> field is disabled in the mapping explicitly. +If a column, for which there is no source stored, is asked for in a query, {es-sql} will not return it. Field types that don't follow +this restriction are: `keyword`, `date`, `scaled_float`, `geo_point`, `geo_shape` since they are NOT returned from `_source` but +from `docvalue_fields`. + +[float] +[[fields-from-docvalues]] +=== Retrieving from `docvalue_fields` + +When the number of columns retrieveable from `docvalue_fields` is greater than the configured <> +the query will fail with `IllegalArgumentException: Trying to retrieve too many docvalue_fields` error. Either the mentioned {es} +setting needs to be adjusted or fewer columns retrieveable from `docvalue_fields` need to be selected. diff --git a/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java b/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java index e187be8c7d0..a1045e2713e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java +++ b/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java @@ -150,7 +150,8 @@ public class NotEqualMessageBuilder { field(field, "same [" + expected + "]"); return; } - field(field, "expected [" + expected + "] but was [" + actual + "]"); + field(field, "expected " + expected.getClass().getSimpleName() + " [" + expected + "] but was " + + actual.getClass().getSimpleName() + " [" + actual + "]"); } private void indent() { diff --git a/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java b/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java index acc3224a8ad..814a3e9d2d3 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertionsTests.java @@ -122,7 +122,7 @@ public class ElasticsearchAssertionsTests extends ESTestCase { AssertionError error = expectThrows(AssertionError.class, () -> assertToXContentEquivalent(BytesReference.bytes(builder), BytesReference.bytes(otherBuilder), builder.contentType())); - assertThat(error.getMessage(), containsString("f2: expected [value2] but was [differentValue2]")); + assertThat(error.getMessage(), containsString("f2: expected String [value2] but was String [differentValue2]")); } { XContentBuilder builder = JsonXContent.contentBuilder(); @@ -155,7 +155,7 @@ public class ElasticsearchAssertionsTests extends ESTestCase { AssertionError error = expectThrows(AssertionError.class, () -> assertToXContentEquivalent(BytesReference.bytes(builder), BytesReference.bytes(otherBuilder), builder.contentType())); - assertThat(error.getMessage(), containsString("2: expected [three] but was [four]")); + assertThat(error.getMessage(), containsString("2: expected String [three] but was String [four]")); } { XContentBuilder builder = JsonXContent.contentBuilder(); diff --git a/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityInsufficientRoleIT.java b/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityInsufficientRoleIT.java index 7d66c7debeb..dd929b397ba 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityInsufficientRoleIT.java +++ b/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityInsufficientRoleIT.java @@ -30,7 +30,7 @@ public class MlWithSecurityInsufficientRoleIT extends MlWithSecurityIT { } catch (AssertionError ae) { // Some tests assert on searches of wildcarded ML indices rather than on ML endpoints. For these we expect no hits. if (ae.getMessage().contains("hits.total didn't match expected value")) { - assertThat(ae.getMessage(), containsString("but was [0]")); + assertThat(ae.getMessage(), containsString("but was Integer [0]")); } else { assertThat(ae.getMessage(), either(containsString("action [cluster:monitor/xpack/ml")).or(containsString("action [cluster:admin/xpack/ml"))); diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java index 102491a95dd..af52e50348c 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java @@ -96,15 +96,11 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(readLine(), startsWith(" },")); assertThat(readLine(), startsWith(" \"_source\" : {")); assertThat(readLine(), startsWith(" \"includes\" : [")); + assertThat(readLine(), startsWith(" \"i\"")); assertThat(readLine(), startsWith(" \"test_field\"")); assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"excludes\" : [ ]")); assertThat(readLine(), startsWith(" },")); - assertThat(readLine(), startsWith(" \"docvalue_fields\" : [")); - assertThat(readLine(), startsWith(" {")); - assertThat(readLine(), startsWith(" \"field\" : \"i\"")); - assertThat(readLine(), startsWith(" }")); - assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"_doc\" :")); diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/FieldExtractorIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/FieldExtractorIT.java new file mode 100644 index 00000000000..5ac403d1116 --- /dev/null +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/FieldExtractorIT.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.single_node; + +import org.elasticsearch.xpack.sql.qa.FieldExtractorTestCase; + +public class FieldExtractorIT extends FieldExtractorTestCase { + +} diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java new file mode 100644 index 00000000000..88d9bd6bcb5 --- /dev/null +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -0,0 +1,853 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.JDBCType; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.assertResponse; +import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo; +import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.expectBadRequest; +import static org.hamcrest.Matchers.containsString; + +/** + * Test class covering parameters/settings that can be used in the mapping of an index + * and which can affect the outcome of _source extraction and parsing when retrieving + * values from Elasticsearch. + */ +public abstract class FieldExtractorTestCase extends ESRestTestCase { + + /* + * "text_field": { + * "text": "keyword" + * } + */ + public void testTextField() throws IOException { + String query = "SELECT text_field FROM test"; + String text = randomAlphaOfLength(20); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + createIndexWithFieldTypeAndProperties("text", null, explicitSourceSetting ? indexProps : null); + index("{\"text_field\":\"" + text + "\"}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "text_field", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(singletonList(text))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } + } + + /* + * "keyword_field": { + * "type": "keyword", + * "ignore_above": 10 + * } + */ + public void testKeywordField() throws IOException { + String keyword = randomAlphaOfLength(20); + // _source for `keyword` fields doesn't matter, as they should be taken from docvalue_fields + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreAbove = randomBoolean(); + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (ignoreAbove) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_above", 10); + fieldProps.put("keyword_field", fieldProp); + } + + createIndexWithFieldTypeAndProperties("keyword", fieldProps, explicitSourceSetting ? indexProps : null); + index("{\"keyword_field\":\"" + keyword + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(singletonList(ignoreAbove ? null : keyword))); + assertResponse(expected, runSql("SELECT keyword_field FROM test")); + } + + /* + * "long/integer/short/byte_field": { + * "type": "long/integer/short/byte" + * } + */ + public void testFractionsForNonFloatingPointTypes() throws IOException { + String floatingPointNumber = "123.456"; + String fieldType = randomFrom("long", "integer", "short", "byte"); + + createIndexWithFieldTypeAndProperties(fieldType, null, null); + index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE) + )); + + // because "coerce" is true, a "123.456" floating point number STRING should be converted to 123, no matter the numeric field type + expected.put("rows", singletonList(singletonList(123))); + assertResponse(expected, runSql("SELECT " + fieldType + "_field FROM test")); + } + + /* + * "double/float/half_float/scaled_float_field": { + * "type": "double/float/half_float/scaled_float", + * "scaling_factor": 10 (for scaled_float type only) + * } + */ + public void testCoerceForFloatingPointTypes() throws IOException { + String floatingPointNumber = "123.456"; + String fieldType = randomFrom("double", "float", "half_float", "scaled_float"); + boolean isScaledFloat = fieldType == "scaled_float"; + + Map> fieldProps = null; + if (isScaledFloat) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("scaling_factor", 10); // scaling_factor is required for "scaled_float" + fieldProps.put(fieldType + "_field", fieldProp); + } + + createIndexWithFieldTypeAndProperties(fieldType, fieldProps, null); + // important here is to pass floatingPointNumber as a string: "float_field": "123.456" + index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE) + )); + + // because "coerce" is true, a "123.456" floating point number STRING should be converted to 123.456 as number + // and converted to 123.5 for "scaled_float" type + expected.put("rows", singletonList(singletonList( + isScaledFloat ? 123.5 : (fieldType != "double" ? Double.valueOf(123.456f) : Double.valueOf(floatingPointNumber))))); + assertResponse(expected, runSql("SELECT " + fieldType + "_field FROM test")); + } + + /* + * "long_field": { + * "type": "long", + * "ignore_malformed": true/false + * } + */ + public void testLongFieldType() throws IOException { + testField("long", randomLong()); + } + + /* + * "integer_field": { + * "type": "integer", + * "ignore_malformed": true/false + * } + */ + public void testIntegerFieldType() throws IOException { + testField("integer", randomInt()); + } + + /* + * "short_field": { + * "type": "short", + * "ignore_malformed": true/false + * } + */ + public void testShortFieldType() throws IOException { + // Use Integer as the json parser that is used to read the values from the response will create + // Integers for short and byte values + testField("short", ((Number) randomShort()).intValue()); + } + + /* + * "byte_field": { + * "type": "byte", + * "ignore_malformed": true/false + * } + */ + public void testByteFieldType() throws IOException { + // Use Integer as the json parser that is used to read the values from the response will create + // Integers for short and byte values + testField("byte", ((Number) randomByte()).intValue()); + } + + private void testField(String fieldType, Object value) throws IOException { + String fieldName = fieldType + "_field"; + String query = "SELECT " + fieldName + " FROM test"; + Object actualValue = value; + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (ignoreMalformed) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + // on purpose use a string instead of a number and check for null when querying the field's value + fieldProp.put("ignore_malformed", true); + fieldProps.put(fieldName, fieldProp); + actualValue = "\"foo\""; + } + + createIndexWithFieldTypeAndProperties(fieldType, fieldProps, explicitSourceSetting ? indexProps : null); + index("{\"" + fieldName + "\":" + actualValue + "}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : actualValue))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } + } + + /* + * "boolean_field": { + * "type": "boolean" + * } + */ + public void testBooleanField() throws IOException { + String query = "SELECT boolean_field FROM test"; + boolean booleanField = randomBoolean(); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean asString = randomBoolean(); // pass true or false as string "true" or "false + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + createIndexWithFieldTypeAndProperties("boolean", null, explicitSourceSetting ? indexProps : null); + if (asString) { + index("{\"boolean_field\":\"" + booleanField + "\"}"); + } else { + index("{\"boolean_field\":" + booleanField + "}"); + } + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "boolean_field", "boolean", JDBCType.BOOLEAN, Integer.MAX_VALUE) + )); + // adding the boolean as a String here because parsing the response will yield a "true"/"false" String + expected.put("rows", singletonList(singletonList(asString ? String.valueOf(booleanField) : booleanField))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } + } + + /* + * "ip_field": { + * "type": "ip" + * } + */ + public void testIpField() throws IOException { + String query = "SELECT ip_field FROM test"; + String ipField = "192.168.1.1"; + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + createIndexWithFieldTypeAndProperties("ip", null, explicitSourceSetting ? indexProps : null); + index("{\"ip_field\":\"" + ipField + "\"}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "ip_field", "ip", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(singletonList(ipField))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } + } + + /* + * "keyword_field": { + * "type": "keyword" + * }, + * "keyword_field_alias": { + * "type": "alias", + * "path": "keyword_field" + * }, + * "a.b.c.keyword_field_alias": { + * "type": "alias", + * "path": "keyword_field" + * } + */ + public void testAliasFromDocValueField() throws IOException { + String keyword = randomAlphaOfLength(20); + + createIndexWithFieldTypeAndAlias("keyword", null, null); + index("{\"keyword_field\":\"" + keyword + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "keyword_field_alias", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "a.b.c.keyword_field_alias", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(Arrays.asList(keyword, keyword, keyword))); + assertResponse(expected, runSql("SELECT keyword_field, keyword_field_alias, a.b.c.keyword_field_alias FROM test")); + } + + /* + * "text_field": { + * "type": "text" + * }, + * "text_field_alias": { + * "type": "alias", + * "path": "text_field" + * }, + * "a.b.c.text_field_alias": { + * "type": "alias", + * "path": "text_field" + * } + */ + public void testAliasFromSourceField() throws IOException { + String text = randomAlphaOfLength(20); + + createIndexWithFieldTypeAndAlias("text", null, null); + index("{\"text_field\":\"" + text + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "text_field", "text", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "text_field_alias", "text", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "a.b.c.text_field_alias", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(Arrays.asList(text, null, null))); + assertResponse(expected, runSql("SELECT text_field, text_field_alias, a.b.c.text_field_alias FROM test")); + } + + /* + * "integer_field": { + * "type": "integer" + * }, + * "integer_field_alias": { + * "type": "alias", + * "path": "integer_field" + * }, + * "a.b.c.integer_field_alias": { + * "type": "alias", + * "path": "integer_field" + * } + */ + public void testAliasAggregatableFromSourceField() throws IOException { + int number = randomInt(); + + createIndexWithFieldTypeAndAlias("integer", null, null); + index("{\"integer_field\":" + number + "}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", "integer_field", "integer", JDBCType.INTEGER, Integer.MAX_VALUE), + columnInfo("plain", "integer_field_alias", "integer", JDBCType.INTEGER, Integer.MAX_VALUE), + columnInfo("plain", "a.b.c.integer_field_alias", "integer", JDBCType.INTEGER, Integer.MAX_VALUE) + )); + expected.put("rows", singletonList(Arrays.asList(number, null, number))); + assertResponse(expected, runSql("SELECT integer_field, integer_field_alias, a.b.c.integer_field_alias FROM test")); + } + + /* + * "text_field": { + * "type": "text", + * "fields": { + * "keyword_subfield": { + * "type": "keyword", + * "ignore_above": 10 + * } + * } + * } + */ + public void testTextFieldWithKeywordSubfield() throws IOException { + String text = randomAlphaOfLength(10) + " " + randomAlphaOfLength(10); + // _source for `keyword` fields doesn't matter, as they should be taken from docvalue_fields + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreAbove = randomBoolean(); + String fieldName = "text_field"; + String subFieldName = "text_field.keyword_subfield"; + String query = "SELECT " + fieldName + "," + subFieldName + " FROM test"; + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> subFieldsProps = null; + if (ignoreAbove) { + subFieldsProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_above", 10); + subFieldsProps.put(subFieldName, fieldProp); + } + + createIndexWithFieldTypeAndSubFields("text", null, explicitSourceSetting ? indexProps : null, subFieldsProps, "keyword"); + index("{\"" + fieldName + "\":\"" + text + "\"}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, "text", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + + expected.put("rows", singletonList(Arrays.asList(text, ignoreAbove ? null : text))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + + // even if the _source is disabled, selecting only the keyword sub-field should work as expected + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + + expected.put("rows", singletonList(singletonList(ignoreAbove ? null : text))); + assertResponse(expected, runSql("SELECT text_field.keyword_subfield FROM test")); + } + } + + /* + * "text_field": { + * "type": "text", + * "fields": { + * "integer_subfield": { + * "type": "integer", + * "ignore_malformed": true/false + * } + * } + * } + */ + public void testTextFieldWithIntegerNumberSubfield() throws IOException { + Integer number = randomInt(); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value + Object actualValue = number; + String fieldName = "text_field"; + String subFieldName = "text_field.integer_subfield"; + String query = "SELECT " + fieldName + "," + subFieldName +" FROM test"; + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> subFieldsProps = null; + if (ignoreMalformed) { + subFieldsProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + // on purpose use a string instead of a number and check for null when querying the field's value + fieldProp.put("ignore_malformed", true); + subFieldsProps.put(subFieldName, fieldProp); + actualValue = "foo"; + } + + createIndexWithFieldTypeAndSubFields("text", null, explicitSourceSetting ? indexProps : null, subFieldsProps, "integer"); + index("{\"" + fieldName + "\":\"" + actualValue + "\"}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, "text", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", subFieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE) + )); + if (ignoreMalformed) { + expected.put("rows", singletonList(Arrays.asList("foo", null))); + } else { + expected.put("rows", singletonList(Arrays.asList(String.valueOf(number), number))); + } + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + // if the _source is disabled, selecting only the integer sub-field shouldn't work as well + expectSourceDisabledError("SELECT " + subFieldName + " FROM test"); + } + } + + /* + * "integer_field": { + * "type": "integer", + * "ignore_malformed": true/false, + * "fields": { + * "keyword_subfield/text_subfield": { + * "type": "keyword/text" + * } + * } + * } + */ + public void testNumberFieldWithTextOrKeywordSubfield() throws IOException { + Integer number = randomInt(); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value + boolean isKeyword = randomBoolean(); // text or keyword subfield + Object actualValue = number; + String fieldName = "integer_field"; + String subFieldName = "integer_field." + (isKeyword ? "keyword_subfield" : "text_subfield"); + String query = "SELECT " + fieldName + "," + subFieldName +" FROM test"; + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (ignoreMalformed) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + // on purpose use a string instead of a number and check for null when querying the field's value + fieldProp.put("ignore_malformed", true); + fieldProps.put(fieldName, fieldProp); + actualValue = "foo"; + } + + createIndexWithFieldTypeAndSubFields("integer", fieldProps, explicitSourceSetting ? indexProps : null, null, + isKeyword ? "keyword" : "text"); + index("{\"" + fieldName + "\":\"" + actualValue + "\"}"); + + if (explicitSourceSetting == false || enableSource == true) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE), + columnInfo("plain", subFieldName, isKeyword ? "keyword" : "text", JDBCType.VARCHAR, Integer.MAX_VALUE) + )); + if (ignoreMalformed) { + expected.put("rows", singletonList(Arrays.asList(null, "foo"))); + } else { + expected.put("rows", singletonList(Arrays.asList(number, String.valueOf(number)))); + } + assertResponse(expected, runSql(query)); + } else { + if (isKeyword) { + // selecting only the keyword subfield when the _source is disabled should work + Map expected = new HashMap<>(); + expected.put("columns", singletonList(columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + if (ignoreMalformed) { + expected.put("rows", singletonList(singletonList("foo"))); + } else { + expected.put("rows", singletonList(singletonList(String.valueOf(number)))); + } + assertResponse(expected, runSql("SELECT integer_field.keyword_subfield FROM test")); + } else { + expectSourceDisabledError(query); + } + + // if the _source is disabled, selecting only the integer field shouldn't work + expectSourceDisabledError("SELECT " + fieldName + " FROM test"); + } + } + + /* + * "integer_field": { + * "type": "integer", + * "ignore_malformed": true/false, + * "fields": { + * "byte_subfield": { + * "type": "byte", + * "ignore_malformed": true/false + * } + * } + * } + */ + public void testIntegerFieldWithByteSubfield() throws IOException { + boolean isByte = randomBoolean(); + Integer number = isByte == true ? randomByte() : randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean rootIgnoreMalformed = randomBoolean(); // root field ignore_malformed + boolean subFieldIgnoreMalformed = randomBoolean(); // sub-field ignore_malformed + String fieldName = "integer_field"; + String subFieldName = "integer_field.byte_subfield"; + String query = "SELECT " + fieldName + "," + subFieldName + " FROM test"; + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (rootIgnoreMalformed) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_malformed", true); + fieldProps.put(fieldName, fieldProp); + } + Map> subFieldProps = null; + if (subFieldIgnoreMalformed) { + subFieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_malformed", true); + subFieldProps.put(subFieldName, fieldProp); + } + + createIndexWithFieldTypeAndSubFields("integer", fieldProps, explicitSourceSetting ? indexProps : null, subFieldProps, "byte"); + index("{\"" + fieldName + "\":" + number + "}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE), + columnInfo("plain", subFieldName, "byte", JDBCType.TINYINT, Integer.MAX_VALUE) + )); + if (explicitSourceSetting == false || enableSource == true) { + if (isByte == true || subFieldIgnoreMalformed == true) { + expected.put("rows", singletonList(Arrays.asList(number, isByte ? number : null))); + } else { + expected.put("rows", Collections.emptyList()); + } + assertResponse(expected, runSql(query)); + } else { + if (isByte == true || subFieldIgnoreMalformed == true) { + expectSourceDisabledError(query); + } else { + expected.put("rows", Collections.emptyList()); + assertResponse(expected, runSql(query)); + } + } + } + + /* + * "byte_field": { + * "type": "byte", + * "ignore_malformed": true/false, + * "fields": { + * "integer_subfield": { + * "type": "integer", + * "ignore_malformed": true/false + * } + * } + * } + */ + public void testByteFieldWithIntegerSubfield() throws IOException { + boolean isByte = randomBoolean(); + Integer number = isByte == true ? randomByte() : randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean rootIgnoreMalformed = randomBoolean(); // root field ignore_malformed + boolean subFieldIgnoreMalformed = randomBoolean(); // sub-field ignore_malformed + String fieldName = "byte_field"; + String subFieldName = "byte_field.integer_subfield"; + String query = "SELECT " + fieldName + "," + subFieldName + " FROM test"; + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (rootIgnoreMalformed) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_malformed", true); + fieldProps.put(fieldName, fieldProp); + } + Map> subFieldProps = null; + if (subFieldIgnoreMalformed) { + subFieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_malformed", true); + subFieldProps.put(subFieldName, fieldProp); + } + + createIndexWithFieldTypeAndSubFields("byte", fieldProps, explicitSourceSetting ? indexProps : null, subFieldProps, "integer"); + index("{\"" + fieldName + "\":" + number + "}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList( + columnInfo("plain", fieldName, "byte", JDBCType.TINYINT, Integer.MAX_VALUE), + columnInfo("plain", subFieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE) + )); + if (explicitSourceSetting == false || enableSource == true) { + if (isByte == true || rootIgnoreMalformed == true) { + expected.put("rows", singletonList(Arrays.asList(isByte ? number : null, number))); + } else { + expected.put("rows", Collections.emptyList()); + } + assertResponse(expected, runSql(query)); + } else { + if (isByte == true || rootIgnoreMalformed == true) { + expectSourceDisabledError(query); + } else { + expected.put("rows", Collections.emptyList()); + assertResponse(expected, runSql(query)); + } + } + } + + private void expectSourceDisabledError(String query) { + expectBadRequest(() -> { + client().performRequest(buildRequest(query)); + return Collections.emptyMap(); + }, containsString("unable to fetch fields from _source field: _source is disabled in the mappings for index [test]")); + } + + private void createIndexWithFieldTypeAndAlias(String type, Map> fieldProps, + Map indexProps) throws IOException { + createIndexWithFieldTypeAndProperties(type, fieldProps, indexProps, true, false, null); + } + + private void createIndexWithFieldTypeAndProperties(String type, Map> fieldProps, + Map indexProps) throws IOException { + createIndexWithFieldTypeAndProperties(type, fieldProps, indexProps, false, false, null); + } + + private void createIndexWithFieldTypeAndSubFields(String type, Map> fieldProps, + Map indexProps, Map> subFieldsProps, + String... subFieldsTypes) throws IOException { + createIndexWithFieldTypeAndProperties(type, fieldProps, indexProps, false, true, subFieldsProps, subFieldsTypes); + } + + private void createIndexWithFieldTypeAndProperties(String type, Map> fieldProps, + Map indexProps, boolean withAlias, boolean withSubFields, Map> subFieldsProps, + String... subFieldsTypes) throws IOException { + Request request = new Request("PUT", "/test"); + XContentBuilder index = JsonXContent.contentBuilder().prettyPrint().startObject(); + + index.startObject("mappings"); { + if (indexProps != null) { + for (Entry prop : indexProps.entrySet()) { + if (prop.getValue() instanceof Boolean) { + index.startObject(prop.getKey()); { + index.field("enabled", prop.getValue()); + } + index.endObject(); + } + } + } + index.startObject("properties"); { + String fieldName = type + "_field"; + index.startObject(fieldName); { + index.field("type", type); + if (fieldProps != null && fieldProps.containsKey(fieldName)) { + for (Entry prop : fieldProps.get(fieldName).entrySet()) { + index.field(prop.getKey(), prop.getValue()); + } + } + + if (withSubFields) { + index.startObject("fields"); + for (String subFieldType : subFieldsTypes) { + String subFieldName = subFieldType + "_subfield"; + String fullSubFieldName = fieldName + "." + subFieldName; + index.startObject(subFieldName); + index.field("type", subFieldType); + if (subFieldsProps != null && subFieldsProps.containsKey(fullSubFieldName)) { + for (Entry prop : subFieldsProps.get(fullSubFieldName).entrySet()) { + index.field(prop.getKey(), prop.getValue()); + } + } + index.endObject(); + } + index.endObject(); + } + } + index.endObject(); + + if (withAlias) { + // create two aliases - one within a hierarchy, the other just a simple field w/o hierarchy + index.startObject(fieldName + "_alias"); { + index.field("type", "alias"); + index.field("path", fieldName); + } + index.endObject(); + index.startObject("a.b.c." + fieldName + "_alias"); { + index.field("type", "alias"); + index.field("path", fieldName); + } + index.endObject(); + } + } + index.endObject(); + } + index.endObject(); + index.endObject(); + + request.setJsonEntity(Strings.toString(index)); + client().performRequest(request); + } + + private void index(String... docs) throws IOException { + Request request = new Request("POST", "/test/_bulk"); + request.addParameter("refresh", "true"); + StringBuilder bulk = new StringBuilder(); + for (String doc : docs) { + bulk.append("{\"index\":{}\n"); + bulk.append(doc + "\n"); + } + request.setJsonEntity(bulk.toString()); + client().performRequest(request); + } + + private Request buildRequest(String query) { + Request request = new Request("POST", RestSqlTestCase.SQL_QUERY_REST_ENDPOINT); + request.addParameter("error_trace", "true"); + request.addParameter("pretty", "true"); + request.setEntity(new StringEntity("{\"query\":\"" + query + "\",\"mode\":\"plain\"}", ContentType.APPLICATION_JSON)); + + return request; + } + + private Map runSql(String query) throws IOException { + Response response = client().performRequest(buildRequest(query)); + try (InputStream content = response.getEntity().getContent()) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + } + } + + private JDBCType jdbcTypeFor(String esType) { + switch(esType) { + case "long": + return JDBCType.BIGINT; + case "integer": + return JDBCType.INTEGER; + case "short": + return JDBCType.SMALLINT; + case "byte": + return JDBCType.TINYINT; + case "float": + return JDBCType.REAL; + case "double": + return JDBCType.DOUBLE; + case "half_float": + return JDBCType.FLOAT; + case "scaled_float": + return JDBCType.DOUBLE; + default: + throw new AssertionError("Illegal value [" + esType + "] for data type"); + } + } +} diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 3d65769a9b8..f938e719e8d 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -128,7 +128,7 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { Object text = results.getObject(2); Object keyword = results.getObject(3); assertEquals(-25, number); - assertEquals("xyz", text); + assertEquals("-25", text); assertEquals("-25", keyword); assertFalse(results.next()); }); @@ -448,8 +448,8 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { assertEquals("For field " + e.getKey(), Math.round(e.getValue().doubleValue()), results.getInt(e.getKey())); assertEquals("For field " + e.getKey(), Math.round(e.getValue().doubleValue()), actual); } else if (e.getValue() instanceof Float) { - assertEquals("For field " + e.getKey(), Math.round(e.getValue().floatValue()), results.getInt(e.getKey())); - assertEquals("For field " + e.getKey(), Math.round(e.getValue().floatValue()), actual); + assertEquals("For field " + e.getKey(), e.getValue(), Integer.valueOf(results.getInt(e.getKey())).floatValue()); + assertEquals("For field " + e.getKey(), e.getValue(), Integer.valueOf(actual).floatValue()); } else { assertEquals("For field " + e.getKey(), e.getValue().intValue(), results.getInt(e.getKey())); assertEquals("For field " + e.getKey(), e.getValue().intValue(), actual); @@ -559,9 +559,9 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { results.next(); for(Entry e : map.entrySet()) { long actual = results.getObject(e.getKey(), Long.class); - if (e.getValue() instanceof Double || e.getValue() instanceof Float) { - assertEquals("For field " + e.getKey(), Math.round(e.getValue().doubleValue()), results.getLong(e.getKey())); - assertEquals("For field " + e.getKey(), Math.round(e.getValue().doubleValue()), actual); + if (e.getValue() instanceof Float) { + assertEquals("For field " + e.getKey(), e.getValue(), Long.valueOf(results.getLong(e.getKey())).floatValue()); + assertEquals("For field " + e.getKey(), e.getValue(), Long.valueOf(actual).floatValue()); } else { assertEquals("For field " + e.getKey(), e.getValue().longValue(), results.getLong(e.getKey())); assertEquals("For field " + e.getKey(), e.getValue().longValue(), actual); @@ -660,9 +660,15 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { doWithQuery(SELECT_WILDCARD, (results) -> { results.next(); for(Entry e : map.entrySet()) { - assertEquals("For field " + e.getKey(), e.getValue().doubleValue(), results.getDouble(e.getKey()), 0.0d); - assertEquals("For field " + e.getKey(), - e.getValue().doubleValue(), results.getObject(e.getKey(), Double.class), 0.0d); + if (e.getValue() instanceof Float) { + assertEquals("For field " + e.getKey(), e.getValue(), Double.valueOf(results.getDouble(e.getKey())).floatValue()); + assertEquals("For field " + e.getKey(), + e.getValue(), Double.valueOf(results.getObject(e.getKey(), Double.class)).floatValue()); + } else { + assertEquals("For field " + e.getKey(), e.getValue().doubleValue(), results.getDouble(e.getKey()), 0.0d); + assertEquals("For field " + e.getKey(), + e.getValue().doubleValue(), results.getObject(e.getKey(), Double.class), 0.0d); + } } }); } @@ -1475,8 +1481,8 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { }); Integer[] values = randomArray(3, 15, s -> new Integer[s], () -> Integer.valueOf(randomInt(50))); - // add the minimal value in the middle yet the test will pick it up since the results are sorted - values[2] = Integer.valueOf(-10); + // add the known value as the first one in list. Parsing from _source the value will pick up the first value in the array. + values[0] = Integer.valueOf(-10); String[] stringValues = new String[values.length]; for (int i = 0; i < values.length; i++) { @@ -1505,14 +1511,14 @@ public class ResultSetTestCase extends JdbcIntegrationTestCase { }); Integer[] values = randomArray(3, 15, s -> new Integer[s], () -> Integer.valueOf(randomInt(50))); - // add the minimal value in the middle yet the test will pick it up since the results are sorted - values[2] = Integer.valueOf(-25); + // add the known value as the first one in list. Parsing from _source the value will pick up the first value in the array. + values[0] = Integer.valueOf(-25); String[] stringValues = new String[values.length]; for (int i = 0; i < values.length; i++) { stringValues[i] = String.valueOf(values[i]); } - stringValues[0] = "xyz"; + stringValues[1] = "xyz"; index("test", "1", builder -> { builder.startArray("object"); diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 52da38657d6..b36fb3d04e5 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -351,7 +351,7 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe }, containsString("unknown field [columnar], parser not found")); } - protected void expectBadRequest(CheckedSupplier, Exception> code, Matcher errorMessageMatcher) { + public static void expectBadRequest(CheckedSupplier, Exception> code, Matcher errorMessageMatcher) { try { Map result = code.get(); fail("expected ResponseException but got " + result); @@ -861,7 +861,7 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe ); } - private void assertResponse(Map expected, Map actual) { + public static void assertResponse(Map expected, Map actual) { if (false == expected.equals(actual)) { NotEqualMessageBuilder message = new NotEqualMessageBuilder(); message.compareMaps(actual, expected); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java index 65b59a6f2ce..0b596a1f894 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.type.KeywordEsField; import org.elasticsearch.xpack.sql.type.TextEsField; import org.elasticsearch.xpack.sql.type.UnsupportedEsField; import org.elasticsearch.xpack.sql.util.CollectionUtils; +import org.elasticsearch.xpack.sql.util.Holder; import java.util.ArrayList; import java.util.Arrays; @@ -365,8 +366,8 @@ public class IndexResolver { // lack of parent implies the field is an alias if (map == null) { - // as such, create the field manually - fieldFunction = s -> createField(s, DataType.OBJECT.name(), new TreeMap<>(), false); + // as such, create the field manually, marking the field to also be an alias + fieldFunction = s -> createField(s, DataType.OBJECT.name(), new TreeMap<>(), false, true); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -374,7 +375,7 @@ public class IndexResolver { parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField(s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable()); + fieldFunction = s -> createField(s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); } parent = createField(parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); @@ -390,22 +391,23 @@ public class IndexResolver { return esField; } - private static EsField createField(String fieldName, String typeName, Map props, boolean isAggregateable) { + private static EsField createField(String fieldName, String typeName, Map props, + boolean isAggregateable, boolean isAlias) { DataType esType = DataType.fromTypeName(typeName); switch (esType) { case TEXT: - return new TextEsField(fieldName, props, false); + return new TextEsField(fieldName, props, false, isAlias); case KEYWORD: int length = DataType.KEYWORD.defaultPrecision; // TODO: to check whether isSearchable/isAggregateable takes into account the presence of the normalizer boolean normalized = false; - return new KeywordEsField(fieldName, props, isAggregateable, length, normalized); + return new KeywordEsField(fieldName, props, isAggregateable, length, normalized, isAlias); case DATETIME: return new DateEsField(fieldName, props, isAggregateable); case UNSUPPORTED: return new UnsupportedEsField(fieldName, typeName); default: - return new EsField(fieldName, esType, props, isAggregateable); + return new EsField(fieldName, esType, props, isAggregateable, isAlias); } } @@ -520,9 +522,26 @@ public class IndexResolver { } EsField field = indexFields.flattedMapping.get(fieldName); if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) { + int dot = fieldName.lastIndexOf('.'); + /* + * Looking up the "tree" at the parent fields here to see if the field is an alias. + * When the upper elements of the "tree" have no elements in fieldcaps, then this is an alias field. But not + * always: if there are two aliases - a.b.c.alias1 and a.b.c.alias2 - only one of them will be considered alias. + */ + Holder isAlias = new Holder<>(false); + if (dot >= 0) { + String parentName = fieldName.substring(0, dot); + if (indexFields.flattedMapping.get(parentName) == null) { + // lack of parent implies the field is an alias + if (fieldCaps.get(parentName) == null) { + isAlias.set(true); + } + } + } + createField(fieldName, fieldCaps, indexFields.hierarchicalMapping, indexFields.flattedMapping, s -> invalidField != null ? invalidField : createField(s, typeCap.getType(), emptyMap(), - typeCap.isAggregatable())); + typeCap.isAggregatable(), isAlias.get())); } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index 810c55e808c..f38d4a4f7dc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -474,7 +474,7 @@ public class Querier { private HitExtractor createExtractor(FieldExtraction ref) { if (ref instanceof SearchHitFieldRef) { SearchHitFieldRef f = (SearchHitFieldRef) ref; - return new FieldHitExtractor(f.name(), f.getDataType(), cfg.zoneId(), + return new FieldHitExtractor(f.name(), f.fullFieldName(), f.getDataType(), cfg.zoneId(), f.useDocValue(), f.hitName(), multiValueFieldLeniency); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index 5446ff2493e..f21a6ee96d1 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.document.DocumentField; @@ -13,6 +14,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.function.scalar.geo.GeoShape; @@ -34,6 +36,7 @@ import java.util.StringJoiner; */ public class FieldHitExtractor implements HitExtractor { + private static final Version SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION = Version.V_7_4_0; /** * Stands for {@code field}. We try to use short names for {@link HitExtractor}s * to save a few bytes when when we send them back to the user. @@ -49,6 +52,7 @@ public class FieldHitExtractor implements HitExtractor { } private final String fieldName, hitName; + private final String fullFieldName; // used to look at the _ignored section of the query response for the actual full field name private final DataType dataType; private final ZoneId zoneId; private final boolean useDocValue; @@ -56,15 +60,17 @@ public class FieldHitExtractor implements HitExtractor { private final String[] path; public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue) { - this(name, dataType, zoneId, useDocValue, null, false); + this(name, null, dataType, zoneId, useDocValue, null, false); } public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue, boolean arrayLeniency) { - this(name, dataType, zoneId, useDocValue, null, arrayLeniency); + this(name, null, dataType, zoneId, useDocValue, null, arrayLeniency); } - public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName, boolean arrayLeniency) { + public FieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName, + boolean arrayLeniency) { this.fieldName = name; + this.fullFieldName = fullFieldName; this.dataType = dataType; this.zoneId = zoneId; this.useDocValue = useDocValue; @@ -82,6 +88,11 @@ public class FieldHitExtractor implements HitExtractor { FieldHitExtractor(StreamInput in) throws IOException { fieldName = in.readString(); + if (in.getVersion().onOrAfter(SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION)) { + fullFieldName = in.readOptionalString(); + } else { + fullFieldName = null; + } String esType = in.readOptionalString(); dataType = esType != null ? DataType.fromTypeName(esType) : null; zoneId = ZoneId.of(in.readString()); @@ -99,6 +110,9 @@ public class FieldHitExtractor implements HitExtractor { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(fieldName); + if (out.getVersion().onOrAfter(SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION)) { + out.writeOptionalString(fullFieldName); + } out.writeOptionalString(dataType == null ? null : dataType.typeName); out.writeString(zoneId.getId()); out.writeBoolean(useDocValue); @@ -115,6 +129,24 @@ public class FieldHitExtractor implements HitExtractor { value = unwrapMultiValue(field.getValues()); } } else { + // if the field was ignored because it was malformed and ignore_malformed was turned on + if (fullFieldName != null + && hit.getFields().containsKey(IgnoredFieldMapper.NAME) + && dataType.isFromDocValuesOnly() == false + && dataType.isNumeric()) { + /* + * ignore_malformed makes sense for extraction from _source for numeric fields only. + * And we check here that the data type is actually a numeric one to rule out + * any non-numeric sub-fields (for which the "parent" field should actually be extracted from _source). + * For example, in the case of a malformed number, a "byte" field with "ignore_malformed: true" + * with a "text" sub-field should return "null" for the "byte" parent field and the actual malformed + * data for the "text" sub-field. Also, the _ignored section of the response contains the full field + * name, thus the need to do the comparison with that and not only the field name. + */ + if (hit.getFields().get(IgnoredFieldMapper.NAME).getValues().contains(fullFieldName)) { + return null; + } + } Map source = hit.getSourceAsMap(); if (source != null) { value = extractFromSource(source); @@ -165,12 +197,38 @@ public class FieldHitExtractor implements HitExtractor { return DateUtils.asDateTime(Long.parseLong(values.toString()), zoneId); } } + // The Jackson json parser can generate for numerics - Integers, Longs, BigIntegers (if Long is not enough) // and BigDecimal (if Double is not enough) - if (values instanceof Number - || values instanceof String - || values instanceof Boolean) { - return values; + if (values instanceof Number || values instanceof String || values instanceof Boolean) { + if (dataType == null) { + return values; + } + if (dataType.isNumeric() && dataType.isFromDocValuesOnly() == false) { + if (dataType == DataType.DOUBLE || dataType == DataType.FLOAT || dataType == DataType.HALF_FLOAT) { + Number result = null; + try { + result = dataType.numberType().parse(values, true); + } catch(IllegalArgumentException iae) { + return null; + } + // docvalue_fields is always returning a Double value even if the underlying floating point data type is not Double + // even if we don't extract from docvalue_fields anymore, the behavior should be consistent + return result.doubleValue(); + } else { + Number result = null; + try { + result = dataType.numberType().parse(values, true); + } catch(IllegalArgumentException iae) { + return null; + } + return result; + } + } else if (dataType.isString()) { + return values.toString(); + } else { + return values; + } } throw new SqlIllegalArgumentException("Type {} (returned by [{}]) is not supported", values.getClass().getSimpleName(), fieldName); } @@ -259,6 +317,10 @@ public class FieldHitExtractor implements HitExtractor { public String fieldName() { return fieldName; } + + public String fullFieldName() { + return fullFieldName; + } public ZoneId zoneId() { return zoneId; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 827eade2e59..a0a8741bdc7 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.sql.querydsl.query.MatchAll; import org.elasticsearch.xpack.sql.querydsl.query.NestedQuery; import org.elasticsearch.xpack.sql.querydsl.query.Query; import org.elasticsearch.xpack.sql.tree.Source; +import org.elasticsearch.xpack.sql.type.DataType; import java.io.IOException; import java.util.AbstractMap; @@ -297,16 +298,48 @@ public class QueryContainer { // reference methods // private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) { - return new SearchHitFieldRef(aliasName(fieldAttr), fieldAttr.field().getDataType(), fieldAttr.field().isAggregatable()); + FieldAttribute actualField = fieldAttr; + FieldAttribute rootField = fieldAttr; + StringBuilder fullFieldName = new StringBuilder(fieldAttr.field().getName()); + + // Only if the field is not an alias (in which case it will be taken out from docvalue_fields if it's isAggregatable()), + // go up the tree of parents until a non-object (and non-nested) type of field is found and use that specific parent + // as the field to extract data from, from _source. We do it like this because sub-fields are not in the _source, only + // the root field to which those sub-fields belong to, are. Instead of "text_field.keyword_subfield" for _source extraction, + // we use "text_field", because there is no source for "keyword_subfield". + /* + * "text_field": { + * "type": "text", + * "fields": { + * "keyword_subfield": { + * "type": "keyword" + * } + * } + * } + */ + if (fieldAttr.field().isAlias() == false) { + while (actualField.parent() != null + && actualField.parent().field().getDataType() != DataType.OBJECT + && actualField.parent().field().getDataType() != DataType.NESTED + && actualField.field().getDataType().isFromDocValuesOnly() == false) { + actualField = actualField.parent(); + } + } + while (rootField.parent() != null) { + fullFieldName.insert(0, ".").insert(0, rootField.parent().field().getName()); + rootField = rootField.parent(); + } + return new SearchHitFieldRef(aliasName(actualField), fullFieldName.toString(), fieldAttr.field().getDataType(), + fieldAttr.field().isAggregatable(), fieldAttr.field().isAlias()); } private Tuple nestedHitFieldRef(FieldAttribute attr) { String name = aliasName(attr); Query q = rewriteToContainNestedField(query, attr.source(), - attr.nestedParent().name(), name, attr.field().getDataType().format(), attr.field().isAggregatable()); + attr.nestedParent().name(), name, attr.field().getDataType().format(), attr.field().getDataType().isFromDocValuesOnly()); - SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), - attr.field().isAggregatable(), attr.parent().name()); + SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, null, attr.field().getDataType(), attr.field().isAggregatable(), + false, attr.parent().name()); return new Tuple<>( new QueryContainer(q, aggs, fields, aliases, pseudoFunctions, scalarFunctions, sort, limit, trackHits, includeFrozen), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java index e0233e1d166..6ecf189d99a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java @@ -10,18 +10,23 @@ import org.elasticsearch.xpack.sql.type.DataType; public class SearchHitFieldRef extends FieldReference { private final String name; + private final String fullFieldName; // path included. If field full path is a.b.c, full field name is "a.b.c" and name is "c" private final DataType dataType; private final boolean docValue; private final String hitName; - public SearchHitFieldRef(String name, DataType dataType, boolean useDocValueInsteadOfSource) { - this(name, dataType, useDocValueInsteadOfSource, null); + public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias) { + this(name, fullFieldName, dataType, useDocValueInsteadOfSource, isAlias, null); } - public SearchHitFieldRef(String name, DataType dataType, boolean useDocValueInsteadOfSource, String hitName) { + public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias, + String hitName) { this.name = name; + this.fullFieldName = fullFieldName; this.dataType = dataType; - this.docValue = useDocValueInsteadOfSource; + // these field types can only be extracted from docvalue_fields (ie, values already computed by Elasticsearch) + // because, for us to be able to extract them from _source, we would need the mapping of those fields (which we don't have) + this.docValue = isAlias ? useDocValueInsteadOfSource : (dataType.isFromDocValuesOnly() ? useDocValueInsteadOfSource : false); this.hitName = hitName; } @@ -33,6 +38,10 @@ public class SearchHitFieldRef extends FieldReference { public String name() { return name; } + + public String fullFieldName() { + return fullFieldName; + } public DataType getDataType() { return dataType; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index 76f2436e862..2ce3c1fac96 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.type; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.xpack.sql.util.DateUtils; import java.sql.JDBCType; @@ -18,7 +19,7 @@ import java.util.Map.Entry; /** * Elasticsearch SQL data types. * This class also implements JDBC {@link SQLType} for properly receiving and setting values. - * Where possible, please use the build-in, JDBC {@link Types} and {@link JDBCType} to avoid coupling + * Where possible, please use the built-in, JDBC {@link Types} and {@link JDBCType} to avoid coupling * to the API. */ public enum DataType { @@ -143,7 +144,6 @@ public enum DataType { SQL_TO_ES.put(entry.getKey().substring(4), entry.getValue()); } - // special ones SQL_TO_ES.put("BOOL", DataType.BOOLEAN); SQL_TO_ES.put("INT", DataType.INTEGER); @@ -181,7 +181,6 @@ public enum DataType { */ public final int defaultPrecision; - /** * Display Size *

@@ -270,6 +269,16 @@ public enum DataType { return isDateBased() || isTimeBased(); } + // data type extract-able from _source or from docvalue_fields + public boolean isFromDocValuesOnly() { + return this == KEYWORD // because of ignore_above. Extracting this from _source wouldn't make sense if it wasn't indexed at all. + || this == DATE // because of date formats + || this == DATETIME + || this == SCALED_FLOAT // because of scaling_factor + || this == GEO_POINT + || this == GEO_SHAPE; + } + public static DataType fromOdbcType(String odbcType) { return ODBC_TO_ES.get(odbcType); } @@ -296,4 +305,11 @@ public enum DataType { public String format() { return isDateOrTimeBased() ? DateUtils.DATE_PARSE_FORMAT : null; } + + /** + * Returns the appropriate NumberType enum corresponding to this es type + */ + public NumberType numberType() { + return NumberType.valueOf(esType.toUpperCase(Locale.ROOT)); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java index 52f531ba6e4..b54510e4864 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java @@ -19,12 +19,18 @@ public class EsField { private final boolean aggregatable; private final Map properties; private final String name; + private final boolean isAlias; public EsField(String name, DataType esDataType, Map properties, boolean aggregatable) { + this(name, esDataType, properties, aggregatable, false); + } + + public EsField(String name, DataType esDataType, Map properties, boolean aggregatable, boolean isAlias) { this.name = name; this.esDataType = esDataType; this.aggregatable = aggregatable; this.properties = properties; + this.isAlias = isAlias; } /** @@ -57,6 +63,13 @@ public class EsField { return properties; } + /** + * This field is an alias to another field + */ + public boolean isAlias() { + return isAlias; + } + /** * Returns the path to the keyword version of this field if this field is text and it has a subfield that is * indexed as keyword, throws an exception if such field is not found or the field name itself in all other cases. @@ -103,14 +116,14 @@ public class EsField { return false; } EsField field = (EsField) o; - return aggregatable == field.aggregatable && esDataType == field.esDataType + return aggregatable == field.aggregatable && isAlias == field.isAlias && esDataType == field.esDataType && Objects.equals(name, field.name) && Objects.equals(properties, field.properties); } @Override public int hashCode() { - return Objects.hash(esDataType, aggregatable, properties, name); + return Objects.hash(esDataType, aggregatable, properties, name, isAlias); } public static final class Exact { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/KeywordEsField.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/KeywordEsField.java index 3b77608fc8b..f44c5de0adb 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/KeywordEsField.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/KeywordEsField.java @@ -22,7 +22,12 @@ public class KeywordEsField extends EsField { } public KeywordEsField(String name, Map properties, boolean hasDocValues, int precision, boolean normalized) { - super(name, DataType.KEYWORD, properties, hasDocValues); + this(name, properties, hasDocValues, precision, normalized, false); + } + + public KeywordEsField(String name, Map properties, boolean hasDocValues, int precision, + boolean normalized, boolean isAlias) { + super(name, DataType.KEYWORD, properties, hasDocValues, isAlias); this.precision = precision; this.normalized = normalized; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/TextEsField.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/TextEsField.java index 4944a472e21..f74757137e3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/TextEsField.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/TextEsField.java @@ -17,7 +17,11 @@ import java.util.function.Function; public class TextEsField extends EsField { public TextEsField(String name, Map properties, boolean hasDocValues) { - super(name, DataType.TEXT, properties, hasDocValues); + this(name, properties, hasDocValues, false); + } + + public TextEsField(String name, Map properties, boolean hasDocValues, boolean isAlias) { + super(name, DataType.TEXT, properties, hasDocValues, isAlias); } @Override diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index 0528b9121d3..742db4eca5d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -20,22 +20,22 @@ public class SqlTranslateActionIT extends AbstractSqlIntegTestCase { public void testSqlTranslateAction() { assertAcked(client().admin().indices().prepareCreate("test").get()); client().prepareBulk() - .add(new IndexRequest("test").id("1").source("data", "bar", "count", 42)) - .add(new IndexRequest("test").id("2").source("data", "baz", "count", 43)) + .add(new IndexRequest("test").id("1").source("data", "bar", "count", 42, "date", "1984-01-04")) + .add(new IndexRequest("test").id("2").source("data", "baz", "count", 43, "date", "1989-12-19")) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); ensureYellow("test"); boolean columnOrder = randomBoolean(); - String columns = columnOrder ? "data, count" : "count, data"; + String columns = columnOrder ? "data, count, date" : "date, data, count"; SqlTranslateResponse response = new SqlTranslateRequestBuilder(client(), SqlTranslateAction.INSTANCE) .query("SELECT " + columns + " FROM test ORDER BY count").get(); SearchSourceBuilder source = response.source(); FetchSourceContext fetch = source.fetchSource(); assertTrue(fetch.fetchSource()); - assertArrayEquals(new String[] { "data" }, fetch.includes()); + assertArrayEquals(new String[] { "data", "count" }, fetch.includes()); assertEquals( - singletonList(new DocValueFieldsContext.FieldAndFormat("count", null)), + singletonList(new DocValueFieldsContext.FieldAndFormat("date", "epoch_millis")), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java index 6db6c5abd16..96713befd94 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.sql.expression.gen.processor.ChainingProcessor; import org.elasticsearch.xpack.sql.expression.gen.processor.ChainingProcessorTests; import org.elasticsearch.xpack.sql.expression.gen.processor.HitExtractorProcessor; import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.sql.type.DataType; import java.io.IOException; import java.util.ArrayList; @@ -71,7 +72,7 @@ public class ComputingExtractorTests extends AbstractWireSerializingTestCase randomFrom(DataType.values())), randomValueOtherThan(instance.zoneId(), ESTestCase::randomZone), randomBoolean(), @@ -188,7 +189,7 @@ public class FieldHitExtractorTests extends AbstractWireSerializingTestCase