EQL: add the wildcard field type to the IT tests (#62166) (#62200)

* Add wildcard field type as an option for randomized testing of IT queries

(cherry picked from commit 87b14c409c180c4d53c3c61a30bd69f1b81a2823)

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java

@@ -17,19 +17,24 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.common.CheckedBiFunction;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.rest.ESRestTestCase;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertThat;
@@ -38,9 +43,19 @@ public class DataLoader {
 
     private static final String TEST_DATA = "/test_data.json";
     private static final String MAPPING = "/mapping-default.json";
+    private static final Map<String, String[]> replacementPatterns = Collections.unmodifiableMap(getReplacementPatterns());
     static final String indexPrefix = "endgame";
     public static final String testIndexName = indexPrefix + "-1.4.0";
 
+    private static final long FILETIME_EPOCH_DIFF = 11644473600000L;
+    private static final long FILETIME_ONE_MILLISECOND = 10 * 1000;
+
+    private static Map<String, String[]> getReplacementPatterns() {
+        final Map<String, String[]> map = new HashMap<>(1);
+        map.put("[runtime_random_keyword_type]", new String[] {"keyword", "wildcard"});
+        return map;
+    }
+
     public static void main(String[] args) throws IOException {
         try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
             loadDatasetIntoEs(new RestHighLevelClient(
@@ -60,12 +75,28 @@ public class DataLoader {
     }
 
     private static void createTestIndex(RestHighLevelClient client) throws IOException {
-        CreateIndexRequest request = new CreateIndexRequest(testIndexName)
+        CreateIndexRequest request = new CreateIndexRequest(testIndexName).mapping(getMapping(MAPPING), XContentType.JSON);
-            .mapping(Streams.readFully(DataLoader.class.getResourceAsStream(MAPPING)), XContentType.JSON);
-
         client.indices().create(request, RequestOptions.DEFAULT);
     }
 
+    private static String getMapping(String mappingPath) throws IOException {
+        try (BufferedReader reader = new BufferedReader(
+            new InputStreamReader(DataLoader.class.getResourceAsStream(mappingPath), StandardCharsets.UTF_8)))
+        {
+            StringBuilder b = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                if (line.startsWith("#") == false) {
+                    for (Entry<String, String[]> entry : replacementPatterns.entrySet()) {
+                        line = line.replace(entry.getKey(), ESRestTestCase.randomFrom(entry.getValue()));
+                    }
+                    b.append(line);
+                }
+            }
+            return b.toString();
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private static void loadData(RestHighLevelClient client, CheckedBiFunction<XContent, InputStream, XContentParser, IOException> p)
         throws IOException {
@@ -100,10 +131,6 @@ public class DataLoader {
         entry.put("@timestamp", winFileTimeToUnix(ts));
     }
 
-
-    private static final long FILETIME_EPOCH_DIFF = 11644473600000L;
-    private static final long FILETIME_ONE_MILLISECOND = 10 * 1000;
-
     public static long winFileTimeToUnix(final long filetime) {
         long ts = (filetime / FILETIME_ONE_MILLISECOND);
         return ts - FILETIME_EPOCH_DIFF;

x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json

@@ -1,10 +1,12 @@
+# Text patterns like "[runtime_random_keyword_type]" will get replaced at runtime with a random string type.
+# See DataLoader class for pattern replacements.
 {
     "properties" : {
         "command_line" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "event_type" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "event" : {
             "properties" : {
@@ -19,13 +21,13 @@
             }
         },
         "md5" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "parent_process_name": {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "parent_process_path": {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "pid" : {
             "type" : "long"
@@ -34,13 +36,13 @@
             "type" : "long"
         },
         "process_name": {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "process_path": {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "subtype" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "timestamp" : {
             "type" : "date"
@@ -49,19 +51,19 @@
             "type" : "date"
         },
         "user" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "user_name" : {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "user_domain": {
-            "type" : "keyword"
+            "type" : "[runtime_random_keyword_type]"
         },
         "hostname" : {
             "type" : "text",
             "fields" : {
-                "keyword" : {
+                "[runtime_random_keyword_type]" : {
-                    "type" : "keyword",
+                    "type" : "[runtime_random_keyword_type]",
                     "ignore_above" : 256
                 }
             }
@@ -72,8 +74,8 @@
         "file_name" : {
             "type" : "text",
             "fields" : {
-                "keyword" : {
+                "[runtime_random_keyword_type]" : {
-                    "type" : "keyword",
+                    "type" : "[runtime_random_keyword_type]",
                     "ignore_above" : 256
                 }
             }