diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java index ae577ef1adb..9a472e13842 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java @@ -23,8 +23,13 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.eql.EqlSearchRequest; import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.index.IndexSettings; import org.junit.Before; +import java.time.format.DateTimeFormatter; + import static org.hamcrest.Matchers.equalTo; public class EqlIT extends ESRestHighLevelClientTestCase { @@ -35,7 +40,6 @@ public class EqlIT extends ESRestHighLevelClientTestCase { } public void testBasicSearch() throws Exception { - Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " + "\"event_type\": \"process\", " + @@ -61,4 +65,33 @@ public class EqlIT extends ESRestHighLevelClientTestCase { assertNotNull(response.hits().events()); assertThat(response.hits().events().size(), equalTo(1)); } + + public void testLargeMapping() throws Exception { + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); + // use more exact fields (dates) than the default to verify that retrieval works and requesting doc values + // would fail + int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50; + String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME); + StringBuilder sb = new StringBuilder(); + sb.append("{"); + for (int i = 0; i < PASS_DEFAULT_DOC_VALUES; i++) { + sb.append("\"datetime" + i + "\":\"" + now + "\""); + sb.append(","); + } + sb.append("\"event_type\": \"process\","); + sb.append("\"serial_event_id\": 1"); + sb.append("}"); + doc1.setJsonEntity(sb.toString()); + + client().performRequest(doc1); + client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); + + + EqlClient eql = highLevelClient().eql(); + EqlSearchRequest request = new EqlSearchRequest("index", "process where true"); + EqlSearchResponse response = execute(request, eql::search, eql::searchAsync); + assertNotNull(response); + assertNotNull(response.hits()); + assertThat(response.hits().events().size(), equalTo(1)); + } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index e85e01db3ea..aed49643452 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -10,7 +10,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; -import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; import java.util.List; @@ -41,14 +40,7 @@ public abstract class SourceGenerator { final SearchSourceBuilder source = new SearchSourceBuilder(); source.query(finalQuery); - QlSourceBuilder sortBuilder = new QlSourceBuilder(); - // Iterate through all the columns requested, collecting the fields that - // need to be retrieved from the result documents - - // NB: the sortBuilder takes care of eliminating duplicates - container.fields().forEach(f -> f.v1().collectFields(sortBuilder)); - sortBuilder.build(source); - optimize(sortBuilder, source); + source.fetchSource(FetchSourceContext.FETCH_SOURCE); // set fetch size if (size != null) { @@ -62,22 +54,9 @@ public abstract class SourceGenerator { return source; } - private static void optimize(QlSourceBuilder qlSource, SearchSourceBuilder builder) { - if (qlSource.noSource()) { - disableSource(builder); - } - } - private static void optimize(QueryContainer query, SearchSourceBuilder builder) { if (query.shouldTrackHits()) { builder.trackTotalHits(true); } } - - private static void disableSource(SearchSourceBuilder builder) { - builder.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE); - if (builder.storedFields() == null) { - builder.storedFields(NO_STORED_FIELD); - } - } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java index ad2b42880c4..9e9cc96cdfd 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java @@ -51,10 +51,6 @@ public class QueryFolderTests extends ESTestCase { // test query term assertThat(query, containsString("\"term\":{\"event_type\":{\"value\":\"process\"")); // test field source extraction - assertThat(query, containsString("\"_source\":{\"includes\":[")); - assertThat(query, containsString("\"pid\"")); - // test docvalue extraction - assertThat(query, containsString("{\"field\":\"command_line\"}")); - assertThat(query, containsString("{\"field\":\"timestamp\",\"format\":\"epoch_millis\"}")); + assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]")); } }