From cce6da7d520fe7e4387f8d609801295d3f736da3 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 10 Sep 2020 12:36:36 +0300 Subject: [PATCH] EQL: add the wildcard field type to the IT tests (#62166) (#62200) * Add wildcard field type as an option for randomized testing of IT queries (cherry picked from commit 87b14c409c180c4d53c3c61a30bd69f1b81a2823) --- .../elasticsearch/test/eql/DataLoader.java | 43 +++++++++++++++---- .../src/main/resources/mapping-default.json | 32 +++++++------- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java index d64b37bf414..3d213124087 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java @@ -17,19 +17,24 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.common.CheckedBiFunction; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertThat; @@ -38,9 +43,19 @@ public class DataLoader { private static final String TEST_DATA = "/test_data.json"; private static final String MAPPING = "/mapping-default.json"; + private static final Map replacementPatterns = Collections.unmodifiableMap(getReplacementPatterns()); static final String indexPrefix = "endgame"; public static final String testIndexName = indexPrefix + "-1.4.0"; + private static final long FILETIME_EPOCH_DIFF = 11644473600000L; + private static final long FILETIME_ONE_MILLISECOND = 10 * 1000; + + private static Map getReplacementPatterns() { + final Map map = new HashMap<>(1); + map.put("[runtime_random_keyword_type]", new String[] {"keyword", "wildcard"}); + return map; + } + public static void main(String[] args) throws IOException { try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) { loadDatasetIntoEs(new RestHighLevelClient( @@ -60,12 +75,28 @@ public class DataLoader { } private static void createTestIndex(RestHighLevelClient client) throws IOException { - CreateIndexRequest request = new CreateIndexRequest(testIndexName) - .mapping(Streams.readFully(DataLoader.class.getResourceAsStream(MAPPING)), XContentType.JSON); - + CreateIndexRequest request = new CreateIndexRequest(testIndexName).mapping(getMapping(MAPPING), XContentType.JSON); client.indices().create(request, RequestOptions.DEFAULT); } + private static String getMapping(String mappingPath) throws IOException { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(DataLoader.class.getResourceAsStream(mappingPath), StandardCharsets.UTF_8))) + { + StringBuilder b = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#") == false) { + for (Entry entry : replacementPatterns.entrySet()) { + line = line.replace(entry.getKey(), ESRestTestCase.randomFrom(entry.getValue())); + } + b.append(line); + } + } + return b.toString(); + } + } + @SuppressWarnings("unchecked") private static void loadData(RestHighLevelClient client, CheckedBiFunction p) throws IOException { @@ -100,10 +131,6 @@ public class DataLoader { entry.put("@timestamp", winFileTimeToUnix(ts)); } - - private static final long FILETIME_EPOCH_DIFF = 11644473600000L; - private static final long FILETIME_ONE_MILLISECOND = 10 * 1000; - public static long winFileTimeToUnix(final long filetime) { long ts = (filetime / FILETIME_ONE_MILLISECOND); return ts - FILETIME_EPOCH_DIFF; diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json index b8e0e4840e3..ec0d955800c 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json +++ b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json @@ -1,10 +1,12 @@ +# Text patterns like "[runtime_random_keyword_type]" will get replaced at runtime with a random string type. +# See DataLoader class for pattern replacements. { "properties" : { "command_line" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "event_type" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "event" : { "properties" : { @@ -19,13 +21,13 @@ } }, "md5" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "parent_process_name": { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "parent_process_path": { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "pid" : { "type" : "long" @@ -34,13 +36,13 @@ "type" : "long" }, "process_name": { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "process_path": { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "subtype" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "timestamp" : { "type" : "date" @@ -49,19 +51,19 @@ "type" : "date" }, "user" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "user_name" : { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "user_domain": { - "type" : "keyword" + "type" : "[runtime_random_keyword_type]" }, "hostname" : { "type" : "text", "fields" : { - "keyword" : { - "type" : "keyword", + "[runtime_random_keyword_type]" : { + "type" : "[runtime_random_keyword_type]", "ignore_above" : 256 } } @@ -72,8 +74,8 @@ "file_name" : { "type" : "text", "fields" : { - "keyword" : { - "type" : "keyword", + "[runtime_random_keyword_type]" : { + "type" : "[runtime_random_keyword_type]", "ignore_above" : 256 } }