honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 01:19:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

SQL: adds format parameter to range queries for constant date comparisons (#45503)

Browse Source 
* Add format parameter to the range queries built for CURRENT_* functions
used in comparison conditions
* Use range queries for date fields equality/non-equality as well.

(cherry picked from commit c1e81e90f937ee5a002524d632bfce74d76962f9)
This commit is contained in:
Andrei Stefan 2019-08-13 23:04:30 +03:00 committed by GitHub
parent 80a3aeaef1
commit adf8e20021
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 314 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CustomDateFormatIT.java Normal file
View File

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa.multi_node;
import org.elasticsearch.xpack.sql.qa.CustomDateFormatTestCase;
public class CustomDateFormatIT extends CustomDateFormatTestCase {
}

6
x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
View File

@ -25,10 +25,10 @@ import java.util.List;
import java.util.Map;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.randomMode;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
/**
* Tests specific to multiple nodes.

4
x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
View File

@ -29,10 +29,10 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.randomMode;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;

4
x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
View File

@ -34,10 +34,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.randomMode;
public class UserFunctionIT extends ESRestTestCase {

13
x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CustomDateFormatIT.java Normal file
View File

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa.single_node;
import org.elasticsearch.xpack.sql.qa.CustomDateFormatTestCase;
public class CustomDateFormatIT extends CustomDateFormatTestCase {
}

97
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/CustomDateFormatTestCase.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase;
import org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase;
import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
/*
* Test class that covers the NOW()/CURRENT_DATE()/CURRENT_TIME() family of functions in a comparison condition
* with different timezones and custom date formats for the date fields in ES.
*/
public abstract class CustomDateFormatTestCase extends BaseRestSqlTestCase {
private static String[] customFormats = new String[] {"HH:mm yyyy-MM-dd", "HH:mm:ss yyyy-dd-MM", "HH:mm:ss VV", "HH:mm:ss VV z",
"yyyy-MM-dd'T'HH:mm:ss'T'VV'T'z"};
private static String[] nowFunctions = new String[] {"NOW()", "CURRENT_DATE()", "CURRENT_TIME()", "CURRENT_TIMESTAMP()"};
private static String[] operators = new String[] {" < ", " > ", " <= ", " >= ", " = ", " != "};
public void testCustomDateFormatsWithNowFunctions() throws IOException {
createIndex();
String[] docs = new String[customFormats.length];
String zID = JdbcIntegrationTestCase.randomKnownTimeZone();
StringBuilder datesConditions = new StringBuilder();
for (int i = 0; i < customFormats.length; i++) {
String field = "date_" + i;
docs[i] = "{\"" + field + "\":\"" +
DateTimeFormatter.ofPattern(customFormats[i], Locale.ROOT).format(DateUtils.nowWithMillisResolution()) + "\"}";
datesConditions.append(i > 0 ? " OR " : "").append(field + randomFrom(operators) + randomFrom(nowFunctions));
}
index(docs);
Request request = new Request("POST", RestSqlTestCase.SQL_QUERY_REST_ENDPOINT);
request.setEntity(new StringEntity("{\"query\":\"SELECT COUNT(*) AS c FROM test WHERE "
+ datesConditions.toString() + "\""
+ mode("plain")
+ ",\"time_zone\":\"" + zID + "\"" + "}", ContentType.APPLICATION_JSON));
Response response = client().performRequest(request);
String expectedJsonSnippet = "{\"columns\":[{\"name\":\"c\",\"type\":\"long\"}],\"rows\":[[";
try (InputStream content = response.getEntity().getContent()) {
ByteArrayOutputStream result = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = content.read(buffer)) != -1) {
result.write(buffer, 0, length);
}
String actualJson = result.toString("UTF-8");
// we just need to get a response that's not a date parsing error
assertTrue(actualJson.startsWith(expectedJsonSnippet));
}
}
private void createIndex() throws IOException {
Request request = new Request("PUT", "/test");
XContentBuilder index = JsonXContent.contentBuilder().prettyPrint().startObject();
index.startObject("mappings"); {
index.startObject("properties"); {
for (int i = 0; i < customFormats.length; i++) {
String fieldName = "date_" + i;
index.startObject(fieldName); {
index.field("type", "date");
index.field("format", customFormats[i]);
}
index.endObject();
}
index.endObject();
}
}
index.endObject();
index.endObject();
request.setJsonEntity(Strings.toString(index));
client().performRequest(request);
}
}

16
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
View File

@ -14,7 +14,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase;
import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase;
import java.io.IOException;
@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.containsString;
* and which can affect the outcome of _source extraction and parsing when retrieving
* values from Elasticsearch.
*/
public abstract class FieldExtractorTestCase extends ESRestTestCase {
public abstract class FieldExtractorTestCase extends BaseRestSqlTestCase {
/*
* "text_field": {
@ -800,18 +800,6 @@ public abstract class FieldExtractorTestCase extends ESRestTestCase {
client().performRequest(request);
}
private void index(String... docs) throws IOException {
Request request = new Request("POST", "/test/_bulk");
request.addParameter("refresh", "true");
StringBuilder bulk = new StringBuilder();
for (String doc : docs) {
bulk.append("{\"index\":{}\n");
bulk.append(doc + "\n");
}
request.setJsonEntity(bulk.toString());
client().performRequest(request);
}
private Request buildRequest(String query) {
Request request = new Request("POST", RestSqlTestCase.SQL_QUERY_REST_ENDPOINT);
request.addParameter("error_trace", "true");

37
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa.rest;
import org.elasticsearch.client.Request;
import org.elasticsearch.common.Strings;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.sql.proto.StringUtils;
import java.io.IOException;
public abstract class BaseRestSqlTestCase extends ESRestTestCase {
protected void index(String... docs) throws IOException {
Request request = new Request("POST", "/test/_bulk");
request.addParameter("refresh", "true");
StringBuilder bulk = new StringBuilder();
for (String doc : docs) {
bulk.append("{\"index\":{}\n");
bulk.append(doc + "\n");
}
request.setJsonEntity(bulk.toString());
client().performRequest(request);
}
public static String mode(String mode) {
return Strings.isEmpty(mode) ? StringUtils.EMPTY : ",\"mode\":\"" + mode + "\"";
}
public static String randomMode() {
return randomFrom(StringUtils.EMPTY, "jdbc", "plain");
}
}

24
x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
View File

@ -15,13 +15,11 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.NotEqualMessageBuilder;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.sql.proto.StringUtils;
import org.elasticsearch.xpack.sql.qa.ErrorsTestCase;
import org.hamcrest.Matcher;
@ -50,7 +48,7 @@ import static org.hamcrest.Matchers.containsString;
* Integration test for the rest sql action. The one that speaks json directly to a
* user rather than to the JDBC driver or CLI.
*/
public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTestCase {
public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements ErrorsTestCase {
public static String SQL_QUERY_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
private static String SQL_TRANSLATE_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;
@ -899,24 +897,4 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
}
}
public static String randomMode() {
return randomFrom(StringUtils.EMPTY, "jdbc", "plain");
}
public static String mode(String mode) {
return Strings.isEmpty(mode) ? StringUtils.EMPTY : ",\"mode\":\"" + mode + "\"";
}
protected void index(String... docs) throws IOException {
Request request = new Request("POST", "/test/_bulk");
request.addParameter("refresh", "true");
StringBuilder bulk = new StringBuilder();
for (String doc : docs) {
bulk.append("{\"index\":{}\n");
bulk.append(doc + "\n");
}
request.setJsonEntity(bulk.toString());
client().performRequest(request);
}
}

35
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.sql.planner;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.geo.geometry.Geometry;
import org.elasticsearch.geo.geometry.Point;
import org.elasticsearch.search.sort.SortOrder;
@ -107,6 +108,8 @@ import org.elasticsearch.xpack.sql.util.Check;
import org.elasticsearch.xpack.sql.util.DateUtils;
import org.elasticsearch.xpack.sql.util.ReflectionUtils;
import java.time.OffsetTime;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
@ -121,6 +124,9 @@ import static org.elasticsearch.xpack.sql.type.DataType.DATE;
final class QueryTranslator {
public static final String DATE_FORMAT = "strict_date_time";
public static final String TIME_FORMAT = "strict_hour_minute_second_millis";
private QueryTranslator(){}
private static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
@ -660,6 +666,25 @@ final class QueryTranslator {
String name = nameOf(bc.left());
Object value = valueOf(bc.right());
String format = dateFormat(bc.left());
boolean isDateLiteralComparison = false;
// for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
// no matter the timezone provided by the user
if ((value instanceof ZonedDateTime || value instanceof OffsetTime) && format == null) {
DateFormatter formatter;
if (value instanceof ZonedDateTime) {
formatter = DateFormatter.forPattern(DATE_FORMAT);
// RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance
// which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
// Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well.
value = formatter.format((ZonedDateTime) value);
} else {
formatter = DateFormatter.forPattern(TIME_FORMAT);
value = formatter.format((OffsetTime) value);
}
format = formatter.pattern();
isDateLiteralComparison = true;
}
// Possible geo optimization
if (bc.left() instanceof StDistance && value instanceof Number) {
@ -697,10 +722,16 @@ final class QueryTranslator {
// (which is important for strings)
name = ((FieldAttribute) bc.left()).exactAttribute().name();
}
Query query = new TermQuery(source, name, value);
Query query;
if (isDateLiteralComparison == true) {
// dates equality uses a range query because it's the one that has a "format" parameter
query = new RangeQuery(source, name, value, true, value, true, format);
} else {
query = new TermQuery(source, name, value);
}
if (bc instanceof NotEquals) {
query = new NotQuery(source, query);
}
}
return query;
}

112
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.sql.planner;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.index.query.ExistsQueryBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@ -56,6 +57,7 @@ import org.elasticsearch.xpack.sql.util.DateUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.time.ZonedDateTime;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
@ -64,6 +66,8 @@ import java.util.stream.Stream;
import static org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation.E;
import static org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation.PI;
import static org.elasticsearch.xpack.sql.planner.QueryTranslator.DATE_FORMAT;
import static org.elasticsearch.xpack.sql.planner.QueryTranslator.TIME_FORMAT;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.instanceOf;
@ -129,6 +133,56 @@ public class QueryTranslatorTests extends ESTestCase {
assertEquals("int", tq.term());
assertEquals(5, tq.value());
}
public void testTermEqualityForDate() {
LogicalPlan p = plan("SELECT some.string FROM test WHERE date = 5");
assertTrue(p instanceof Project);
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
QueryTranslation translation = QueryTranslator.toQuery(condition, false);
Query query = translation.query;
assertTrue(query instanceof TermQuery);
TermQuery tq = (TermQuery) query;
assertEquals("date", tq.term());
assertEquals(5, tq.value());
}
public void testTermEqualityForDateWithLiteralDate() {
LogicalPlan p = plan("SELECT some.string FROM test WHERE date = CAST('2019-08-08T12:34:56' AS DATETIME)");
assertTrue(p instanceof Project);
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
QueryTranslation translation = QueryTranslator.toQuery(condition, false);
Query query = translation.query;
assertTrue(query instanceof RangeQuery);
RangeQuery rq = (RangeQuery) query;
assertEquals("date", rq.field());
assertEquals("2019-08-08T12:34:56.000Z", rq.upper());
assertEquals("2019-08-08T12:34:56.000Z", rq.lower());
assertTrue(rq.includeLower());
assertTrue(rq.includeUpper());
assertEquals(DATE_FORMAT, rq.format());
}
public void testTermEqualityForDateWithLiteralTime() {
LogicalPlan p = plan("SELECT some.string FROM test WHERE date = CAST('12:34:56' AS TIME)");
assertTrue(p instanceof Project);
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
QueryTranslation translation = QueryTranslator.toQuery(condition, false);
Query query = translation.query;
assertTrue(query instanceof RangeQuery);
RangeQuery rq = (RangeQuery) query;
assertEquals("date", rq.field());
assertEquals("12:34:56.000", rq.upper());
assertEquals("12:34:56.000", rq.lower());
assertTrue(rq.includeLower());
assertTrue(rq.includeUpper());
assertEquals(TIME_FORMAT, rq.format());
}
public void testComparisonAgainstColumns() {
LogicalPlan p = plan("SELECT some.string FROM test WHERE date > int");
@ -179,7 +233,63 @@ public class QueryTranslatorTests extends ESTestCase {
assertTrue(query instanceof RangeQuery);
RangeQuery rq = (RangeQuery) query;
assertEquals("date", rq.field());
assertEquals(DateUtils.asDateTime("1969-05-13T12:34:56Z"), rq.lower());
assertEquals("1969-05-13T12:34:56.000Z", rq.lower());
}
public void testDateRangeWithCurrentTimestamp() {
testDateRangeWithCurrentFunctions("CURRENT_TIMESTAMP()", DATE_FORMAT, TestUtils.TEST_CFG.now());
}
public void testDateRangeWithCurrentDate() {
testDateRangeWithCurrentFunctions("CURRENT_DATE()", DATE_FORMAT, DateUtils.asDateOnly(TestUtils.TEST_CFG.now()));
}
public void testDateRangeWithToday() {
testDateRangeWithCurrentFunctions("TODAY()", DATE_FORMAT, DateUtils.asDateOnly(TestUtils.TEST_CFG.now()));
}
public void testDateRangeWithNow() {
testDateRangeWithCurrentFunctions("NOW()", DATE_FORMAT, TestUtils.TEST_CFG.now());
}
public void testDateRangeWithCurrentTime() {
testDateRangeWithCurrentFunctions("CURRENT_TIME()", TIME_FORMAT, TestUtils.TEST_CFG.now());
}
private void testDateRangeWithCurrentFunctions(String function, String pattern, ZonedDateTime now) {
String operator = randomFrom(new String[] {">", ">=", "<", "<=", "=", "!="});
LogicalPlan p = plan("SELECT some.string FROM test WHERE date" + operator + function);
assertTrue(p instanceof Project);
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
QueryTranslation translation = QueryTranslator.toQuery(condition, false);
Query query = translation.query;
RangeQuery rq;
if (operator.equals("!=")) {
assertTrue(query instanceof NotQuery);
NotQuery nq = (NotQuery) query;
assertTrue(nq.child() instanceof RangeQuery);
rq = (RangeQuery) nq.child();
} else {
assertTrue(query instanceof RangeQuery);
rq = (RangeQuery) query;
}
assertEquals("date", rq.field());
if (operator.contains("<") || operator.equals("=") || operator.equals("!=")) {
assertEquals(DateFormatter.forPattern(pattern).format(now.withNano(DateUtils.getNanoPrecision(null, now.getNano()))),
rq.upper());
}
if (operator.contains(">") || operator.equals("=") || operator.equals("!=")) {
assertEquals(DateFormatter.forPattern(pattern).format(now.withNano(DateUtils.getNanoPrecision(null, now.getNano()))),
rq.lower());
}
assertEquals(operator.equals("=") || operator.equals("!=") || operator.equals("<="), rq.includeUpper());
assertEquals(operator.equals("=") || operator.equals("!=") || operator.equals(">="), rq.includeLower());
assertEquals(pattern, rq.format());
}
public void testLikeOnInexact() {