From 90e116738e5df6779348c6c5e2e98601b396c260 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 17 Aug 2020 17:59:58 +0300 Subject: [PATCH] QL: add filtering query dsl support to IndexResolver (#60514) (#61200) (cherry picked from commit 7b3635d796be26af9f87d19963a8ed4ab4bbf13f) --- .../xpack/eql/session/EqlSession.java | 7 +- .../xpack/ql/index/IndexResolver.java | 12 +-- .../sql/qa/rest/BaseRestSqlTestCase.java | 6 +- .../xpack/sql/qa/rest/RestSqlTestCase.java | 80 +++++++++++++++++++ .../sql/plan/logical/command/ShowColumns.java | 2 +- .../plan/logical/command/sys/SysColumns.java | 28 +++---- .../xpack/sql/session/SqlSession.java | 2 +- .../logical/command/sys/SysColumnsTests.java | 18 +---- 8 files changed, 116 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index 52ba43bd5b3..883189ef5ed 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -95,9 +95,10 @@ public class EqlSession { if(configuration.isCancelled()){ throw new TaskCancelledException("cancelled"); } - indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.includeFrozen(), wrap(r -> { - listener.onResponse(preAnalyzer.preAnalyze(parsed, r)); - }, listener::onFailure)); + indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.includeFrozen(), configuration.filter(), + wrap(r -> { + listener.onResponse(preAnalyzer.preAnalyze(parsed, r)); + }, listener::onFailure)); } private LogicalPlan doParse(String eql, ParserParams params) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 61e4340fa86..536e6c825a2 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField; import org.elasticsearch.xpack.ql.type.DataType; @@ -280,9 +281,9 @@ public class IndexResolver { /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, QueryBuilder filter, ActionListener listener) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, filter); client.fieldCaps(fieldRequest, ActionListener.wrap( response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), @@ -458,11 +459,12 @@ public class IndexResolver { return new EsField(fieldName, esType, props, isAggregateable, isAlias); } - private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen) { + private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen, QueryBuilder filter) { return new FieldCapabilitiesRequest() .indices(Strings.commaDelimitedListToStringArray(index)) .fields("*") .includeUnmapped(true) + .indexFilter(filter) //lenient because we throw our own errors looking at the response e.g. if something was not resolved //also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS); @@ -471,9 +473,9 @@ public class IndexResolver { /** * Resolves a pattern to multiple, separate indices. Doesn't perform validation. */ - public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, + public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, QueryBuilder filter, ActionListener> listener) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, filter); client.fieldCaps(fieldRequest, wrap(response -> { client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())), diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java index eed45ef0996..2a6b3310901 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java @@ -133,7 +133,11 @@ public abstract class BaseRestSqlTestCase extends ESRestTestCase { } protected void index(String... docs) throws IOException { - Request request = new Request("POST", "/test/_bulk"); + indexWithIndexName("test", docs); + } + + protected void indexWithIndexName(String indexName, String... docs) throws IOException { + Request request = new Request("POST", "/" + indexName + "/_bulk"); request.addParameter("refresh", "true"); StringBuilder bulk = new StringBuilder(); for (String doc : docs) { diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 9ef1067a498..6b157673d2a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -698,6 +698,86 @@ public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements Err ); } + /** + * Test for filtering the field_caps response with a filter. + * Because there is no actual SELECT involved (thus, the REST request filter not actually being applied on an actual _search), we can + * test if the filtering is correctly applied at field_caps request level. + */ + public void testSysColumnsCommandWithFilter() throws IOException { + String mode = randomMode(); + // create three indices with same @timestamp date field and with differently named one more field + indexWithIndexName("test2018", "{\"@timestamp\":\"2018-06-01\",\"field2018\":\"foo\"}"); + indexWithIndexName("test2019", "{\"@timestamp\":\"2019-06-01\",\"field2019\":\"foo\"}"); + indexWithIndexName("test2020", "{\"@timestamp\":\"2020-06-01\",\"field2020\":\"foo\"}"); + + // filter the results so that only test2020's columns are displayed + Map actual = runSql( + new StringEntity( + query("SYS COLUMNS").mode(mode).filter("{\"range\": {\"@timestamp\": {\"gte\": \"2020\"}}}").toString(), + ContentType.APPLICATION_JSON + ), + StringUtils.EMPTY, + mode + ); + @SuppressWarnings("unchecked") + List> rows = (List>) actual.get("rows"); + assertEquals(3, rows.size()); + List currentRow = rows.get(0); + assertEquals("test2020", currentRow.get(2)); + assertEquals("@timestamp", currentRow.get(3)); + currentRow = rows.get(1); + assertEquals("test2020", currentRow.get(2)); + assertEquals("field2020", currentRow.get(3)); + currentRow = rows.get(2); + assertEquals("test2020", currentRow.get(2)); + assertEquals("field2020.keyword", currentRow.get(3)); + } + + /** + * Similar test with {@link #testSysColumnsCommandWithFilter()} but using "SHOW COLUMNS" command which, compared to "SYS COLUMNS" + * goes through a different calls path in IndexResolver + */ + @SuppressWarnings("unchecked") + public void testShowColumnsCommandWithFilter() throws IOException { + String mode = randomMode(); + // create three indices with same @timestamp date field and with differently named one more field + indexWithIndexName("test2018", "{\"@timestamp\":\"2018-06-01\",\"field2018\":\"foo\"}"); + indexWithIndexName("test2019", "{\"@timestamp\":\"2019-06-01\",\"field2019\":\"foo\"}"); + indexWithIndexName("test2020", "{\"@timestamp\":\"2020-06-01\",\"field2020\":\"foo\"}"); + + // filter the results so that only test2020's columns are displayed + Map actual = runSql( + new StringEntity( + query("SHOW COLUMNS FROM test2020").mode(mode).filter("{\"range\": {\"@timestamp\": {\"gte\": \"2020\"}}}").toString(), + ContentType.APPLICATION_JSON + ), + StringUtils.EMPTY, + mode + ); + + List> rows = (List>) actual.get("rows"); + assertEquals(3, rows.size()); + List currentRow = rows.get(0); + assertEquals("@timestamp", currentRow.get(0)); + currentRow = rows.get(1); + assertEquals("field2020", currentRow.get(0)); + currentRow = rows.get(2); + assertEquals("field2020.keyword", currentRow.get(0)); + + // the second test is from an index that is filtered out by the range filter, so the result list should be empty + actual = runSql( + new StringEntity( + query("SHOW COLUMNS FROM test2019").mode(mode).filter("{\"range\": {\"@timestamp\": {\"gte\": \"2020\"}}}").toString(), + ContentType.APPLICATION_JSON + ), + StringUtils.EMPTY, + mode + ); + + rows = (List>) actual.get("rows"); + assertTrue(rows.isEmpty()); + } + public void testBasicTranslateQueryWithFilter() throws IOException { index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}"); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index d510e2c4926..ba789edd4c8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -66,7 +66,7 @@ public class ShowColumns extends Command { String regex = pattern != null ? pattern.asJavaRegex() : null; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, ActionListener.wrap( + session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, session.configuration().filter(), ActionListener.wrap( indexResult -> { List> rows = emptyList(); if (indexResult.isValid()) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 591fc9faa9e..8730915b5ee 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -128,25 +128,25 @@ public class SysColumns extends Command { // special case for '%' (translated to *) if ("*".equals(idx)) { - session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, ActionListener.wrap(esIndices -> { - List> rows = new ArrayList<>(); - for (EsIndex esIndex : esIndices) { - fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); - } - + session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, session.configuration().filter(), + ActionListener.wrap(esIndices -> { + List> rows = new ArrayList<>(); + for (EsIndex esIndex : esIndices) { + fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); + } listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); }, listener::onFailure)); } // otherwise use a merged mapping else { - session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, ActionListener.wrap(r -> { - List> rows = new ArrayList<>(); - // populate the data only when a target is found - if (r.isValid()) { - EsIndex esIndex = r.get(); - fillInRows(cluster, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); - } - + session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, session.configuration().filter(), + ActionListener.wrap(r -> { + List> rows = new ArrayList<>(); + // populate the data only when a target is found + if (r.isValid()) { + EsIndex esIndex = r.get(); + fillInRows(cluster, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); + } listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); }, listener::onFailure)); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index c93000308cd..9a6e2efacc3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -140,7 +140,7 @@ public class SqlSession implements Session { } boolean includeFrozen = configuration.includeFrozen() || tableInfo.isFrozen(); - indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, + indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, configuration.filter(), wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); } else { try { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index cff6386bbcd..36027999b53 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -12,10 +12,7 @@ import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; -import org.elasticsearch.xpack.ql.index.IndexResolver.IndexInfo; -import org.elasticsearch.xpack.ql.index.IndexResolver.IndexType; import org.elasticsearch.xpack.ql.type.EsField; -import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; import org.elasticsearch.xpack.sql.parser.SqlParser; @@ -55,9 +52,6 @@ public class SysColumnsTests extends ESTestCase { private final SqlParser parser = new SqlParser(); private final Map mapping = loadMapping("mapping-multi-field-with-nested.json", true); - private final IndexInfo index = new IndexInfo("test_emp", IndexType.STANDARD_INDEX); - private final IndexInfo alias = new IndexInfo("alias", IndexType.ALIAS); - public void testSysColumns() { List> rows = new ArrayList<>(); @@ -565,22 +559,18 @@ public class SysColumnsTests extends ESTestCase { IndexResolver resolver = mock(IndexResolver.class); when(resolver.clusterName()).thenReturn(CLUSTER_NAME); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[3]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); } - private Tuple sql(String sql, List params, Map mapping) { - return sql(sql, params, SqlTestUtils.TEST_CFG, mapping); - } - private static void checkOdbcShortTypes(SchemaRowSet r) { assertEquals(15, r.size()); // https://github.com/elastic/elasticsearch/issues/35376