honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-23 13:26:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

EQL: Disable field extraction for returned events (#52884)

Browse Source 
Return the whole source of matching events

(cherry picked from commit 79ca586ab1d89d645fb58142b82202f14ce5d361)
This commit is contained in:
Costin Leau 2020-02-28 13:45:21 +02:00 committed by Costin Leau
parent 6aa9aaa2c6
commit a674085903
3 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
View File

@ -23,8 +23,13 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.elasticsearch.client.eql.EqlSearchRequest;
import org.elasticsearch.client.eql.EqlSearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.index.IndexSettings;
import org.junit.Before;
import java.time.format.DateTimeFormatter;
import static org.hamcrest.Matchers.equalTo;
public class EqlIT extends ESRestHighLevelClientTestCase {
@ -35,7 +40,6 @@ public class EqlIT extends ESRestHighLevelClientTestCase {
}
public void testBasicSearch() throws Exception {
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " +
"\"event_type\": \"process\", " +
@ -61,4 +65,33 @@ public class EqlIT extends ESRestHighLevelClientTestCase {
assertNotNull(response.hits().events());
assertThat(response.hits().events().size(), equalTo(1));
}
public void testLargeMapping() throws Exception {
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
// use more exact fields (dates) than the default to verify that retrieval works and requesting doc values
// would fail
int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50;
String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME);
StringBuilder sb = new StringBuilder();
sb.append("{");
for (int i = 0; i < PASS_DEFAULT_DOC_VALUES; i++) {
sb.append("\"datetime" + i + "\":\"" + now + "\"");
sb.append(",");
}
sb.append("\"event_type\": \"process\",");
sb.append("\"serial_event_id\": 1");
sb.append("}");
doc1.setJsonEntity(sb.toString());
client().performRequest(doc1);
client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
EqlClient eql = highLevelClient().eql();
EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
assertNotNull(response);
assertNotNull(response.hits());
assertThat(response.hits().events().size(), equalTo(1));
}
}

23
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java
View File

@ -10,7 +10,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.StoredFieldsContext;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
import java.util.List;
@ -41,14 +40,7 @@ public abstract class SourceGenerator {
final SearchSourceBuilder source = new SearchSourceBuilder();
source.query(finalQuery);
QlSourceBuilder sortBuilder = new QlSourceBuilder();
// Iterate through all the columns requested, collecting the fields that
// need to be retrieved from the result documents
// NB: the sortBuilder takes care of eliminating duplicates
container.fields().forEach(f -> f.v1().collectFields(sortBuilder));
sortBuilder.build(source);
optimize(sortBuilder, source);
source.fetchSource(FetchSourceContext.FETCH_SOURCE);
// set fetch size
if (size != null) {
@ -62,22 +54,9 @@ public abstract class SourceGenerator {
return source;
}
private static void optimize(QlSourceBuilder qlSource, SearchSourceBuilder builder) {
if (qlSource.noSource()) {
disableSource(builder);
}
}
private static void optimize(QueryContainer query, SearchSourceBuilder builder) {
if (query.shouldTrackHits()) {
builder.trackTotalHits(true);
}
}
private static void disableSource(SearchSourceBuilder builder) {
builder.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE);
if (builder.storedFields() == null) {
builder.storedFields(NO_STORED_FIELD);
}
}
}

6
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
View File

@ -51,10 +51,6 @@ public class QueryFolderTests extends ESTestCase {
// test query term
assertThat(query, containsString("\"term\":{\"event_type\":{\"value\":\"process\""));
// test field source extraction
assertThat(query, containsString("\"_source\":{\"includes\":["));
assertThat(query, containsString("\"pid\""));
// test docvalue extraction
assertThat(query, containsString("{\"field\":\"command_line\"}"));
assertThat(query, containsString("{\"field\":\"timestamp\",\"format\":\"epoch_millis\"}"));
assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
}
}