honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: add the wildcard field type to the IT tests (#62166) (#62200) 

* Add wildcard field type as an option for randomized testing of IT queries

(cherry picked from commit 87b14c409c180c4d53c3c61a30bd69f1b81a2823)
This commit is contained in:
Andrei Stefan 2020-09-10 12:36:36 +03:00 committed by GitHub
parent e3feafc1e9
commit cce6da7d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 52 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
View File

@ -17,19 +17,24 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.ESRestTestCase;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;
@ -38,9 +43,19 @@ public class DataLoader {
private static final String TEST_DATA = "/test_data.json";
private static final String MAPPING = "/mapping-default.json";
private static final Map<String, String[]> replacementPatterns = Collections.unmodifiableMap(getReplacementPatterns());
static final String indexPrefix = "endgame";
public static final String testIndexName = indexPrefix + "-1.4.0";
private static final long FILETIME_EPOCH_DIFF = 11644473600000L;
private static final long FILETIME_ONE_MILLISECOND = 10 * 1000;
private static Map<String, String[]> getReplacementPatterns() {
final Map<String, String[]> map = new HashMap<>(1);
map.put("[runtime_random_keyword_type]", new String[] {"keyword", "wildcard"});
return map;
}
public static void main(String[] args) throws IOException {
try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
loadDatasetIntoEs(new RestHighLevelClient(
@ -60,12 +75,28 @@ public class DataLoader {
}
private static void createTestIndex(RestHighLevelClient client) throws IOException {
CreateIndexRequest request = new CreateIndexRequest(testIndexName)
.mapping(Streams.readFully(DataLoader.class.getResourceAsStream(MAPPING)), XContentType.JSON);
CreateIndexRequest request = new CreateIndexRequest(testIndexName).mapping(getMapping(MAPPING), XContentType.JSON);
client.indices().create(request, RequestOptions.DEFAULT);
}
private static String getMapping(String mappingPath) throws IOException {
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(DataLoader.class.getResourceAsStream(mappingPath), StandardCharsets.UTF_8)))
{
StringBuilder b = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith("#") == false) {
for (Entry<String, String[]> entry : replacementPatterns.entrySet()) {
line = line.replace(entry.getKey(), ESRestTestCase.randomFrom(entry.getValue()));
}
b.append(line);
}
}
return b.toString();
}
}
@SuppressWarnings("unchecked")
private static void loadData(RestHighLevelClient client, CheckedBiFunction<XContent, InputStream, XContentParser, IOException> p)
throws IOException {
@ -100,10 +131,6 @@ public class DataLoader {
entry.put("@timestamp", winFileTimeToUnix(ts));
}
private static final long FILETIME_EPOCH_DIFF = 11644473600000L;
private static final long FILETIME_ONE_MILLISECOND = 10 * 1000;
public static long winFileTimeToUnix(final long filetime) {
long ts = (filetime / FILETIME_ONE_MILLISECOND);
return ts - FILETIME_EPOCH_DIFF;

32
x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json
View File

@ -1,10 +1,12 @@
# Text patterns like "[runtime_random_keyword_type]" will get replaced at runtime with a random string type.
# See DataLoader class for pattern replacements.
{
"properties" : {
"command_line" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"event_type" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"event" : {
"properties" : {
@ -19,13 +21,13 @@
}
},
"md5" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"parent_process_name": {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"parent_process_path": {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"pid" : {
"type" : "long"
@ -34,13 +36,13 @@
"type" : "long"
},
"process_name": {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"process_path": {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"subtype" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"timestamp" : {
"type" : "date"
@ -49,19 +51,19 @@
"type" : "date"
},
"user" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"user_name" : {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"user_domain": {
"type" : "keyword"
"type" : "[runtime_random_keyword_type]"
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"[runtime_random_keyword_type]" : {
"type" : "[runtime_random_keyword_type]",
"ignore_above" : 256
}
}
@ -72,8 +74,8 @@
"file_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"[runtime_random_keyword_type]" : {
"type" : "[runtime_random_keyword_type]",
"ignore_above" : 256
}
}