From 0a9481fcafb253504ae557ee07f9aefe95224ad4 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 31 Oct 2020 11:55:39 +0100 Subject: [PATCH] EQL: [Tests] enable server side debugging (#64308) (#64449) Register a new task `runEqlCorrectnessNode` which enables developers to start an ES node in debug mode, properly restore the correctness data and then run queries against it. Assert the index is restored correctly and use new snapshot. (cherry picked from commit fc8c6dd56d602b4a62ee1ff484f00caab92dc6e2) --- x-pack/plugin/eql/qa/correctness/README.md | 21 +++- x-pack/plugin/eql/qa/correctness/build.gradle | 27 +++-- .../xpack/eql/EqlDataLoader.java | 101 ++++++++++++++++++ .../xpack/eql/EsEQLCorrectnessIT.java | 44 +------- .../javaRestTest/resources/config.properties | 7 +- 5 files changed, 147 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDataLoader.java diff --git a/x-pack/plugin/eql/qa/correctness/README.md b/x-pack/plugin/eql/qa/correctness/README.md index 5e10c274671..c69a6d8e4d9 100644 --- a/x-pack/plugin/eql/qa/correctness/README.md +++ b/x-pack/plugin/eql/qa/correctness/README.md @@ -56,7 +56,7 @@ can be found in queries.toml file) with the desired number of the query: #### Debug queries If one wants to check that the filtering subqueries of a sequence query yields the same results (to pinpoint that the -possible failure is in the sequence algortihm), needs to enable this debug mode with the use of a parameter: +possible failure is in the sequence algorithm), needs to enable this debug mode with the use of a parameter: ```shell script ./gradlew -p x-pack/plugin/eql/qa/correctness check -Dtests.eql_correctness_debug=true @@ -66,3 +66,22 @@ or ./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {}" -Dtests.eql_correctness_debug=true ``` +### Run an ES node manually and run the tests against it + +If one wants to run an ES node manually (most probably to be able to debug the server), needs to run the following: + +```shell script +./gradlew :x-pack:plugin:eql:qa:correctness:runEqlCorrectnessNode --debug-jvm +``` + +**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,** + +Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the +`EqlDataLoader` class. + +Once the data is loaded, a specific query can be run against the running ES node with: +```shell script +./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {}" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=runTask-0 +``` + +**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,** diff --git a/x-pack/plugin/eql/qa/correctness/build.gradle b/x-pack/plugin/eql/qa/correctness/build.gradle index 250b8fb8bf5..1a6af388776 100644 --- a/x-pack/plugin/eql/qa/correctness/build.gradle +++ b/x-pack/plugin/eql/qa/correctness/build.gradle @@ -1,7 +1,10 @@ apply plugin: 'elasticsearch.java-rest-test' apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.testclusters' test.enabled = false +import org.elasticsearch.gradle.testclusters.RunTask + restResources { restApi { includeCore '_common', 'bulk', 'indices', 'snapshot' @@ -19,14 +22,19 @@ dependencies { File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File -testClusters.all { - plugin ':plugins:repository-gcs' - if (serviceAccountFile) { - keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile +testClusters { + all { + plugin ':plugins:repository-gcs' + if (serviceAccountFile) { + keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile + } + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'basic' + jvmArgs '-Xms4g', '-Xmx4g' + } + runTask { + jvmArgs '-Xms8g', '-Xmx8g' } - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'basic' - jvmArgs '-Xms4g', '-Xmx4g' } tasks.named('javaRestTest').configure { @@ -36,3 +44,8 @@ tasks.named('javaRestTest').configure { showStandardStreams = true } } + +tasks.register("runEqlCorrectnessNode", RunTask) { + useCluster testClusters.runTask; + description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials' +} diff --git a/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDataLoader.java b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDataLoader.java new file mode 100644 index 00000000000..f2a4fe3b85e --- /dev/null +++ b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDataLoader.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql; + +import org.apache.http.HttpHost; +import org.apache.logging.log4j.core.config.plugins.util.PluginManager; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.Properties; + +import static org.elasticsearch.test.ESTestCase.assertEquals; + +public class EqlDataLoader { + + private static final String PROPERTIES_FILENAME = "config.properties"; + + public static void main(String[] args) throws IOException { + // Need to setup the log configuration properly to avoid messages when creating a new RestClient + PluginManager.addPackage(LogConfigurator.class.getPackage().getName()); + + try ( + RestClient client = RestClient.builder(new HttpHost("localhost", 9200)) + .setRequestConfigCallback( + requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(30000000) + .setConnectionRequestTimeout(30000000) + .setSocketTimeout(30000000) + ) + .setStrictDeprecationMode(true) + .build() + ) { + Properties configuration = loadConfiguration(); + restoreSnapshot(new RestHighLevelClient(client, ignore -> {}, Collections.emptyList()) { + }, configuration); + } + } + + static Properties loadConfiguration() throws IOException { + try (InputStream is = EqlDataLoader.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME)) { + Properties props = new Properties(); + props.load(is); + return props; + } + } + + static void restoreSnapshot(RestHighLevelClient restHighLevelClient, Properties cfg) throws IOException { + if (restHighLevelClient.getLowLevelClient() + .performRequest(new Request("HEAD", "/" + cfg.getProperty("index_name"))) + .getStatusLine() + .getStatusCode() == 404) { + restHighLevelClient.snapshot() + .createRepository( + new PutRepositoryRequest(cfg.getProperty("gcs_repo_name")).type("gcs") + .settings( + Settings.builder() + .put("bucket", cfg.getProperty("gcs_bucket_name")) + .put("base_path", cfg.getProperty("gcs_base_path")) + .put("client", cfg.getProperty("gcs_client_name")) + .build() + ), + RequestOptions.DEFAULT + ); + RestoreSnapshotResponse resp = restHighLevelClient.snapshot() + .restore( + new RestoreSnapshotRequest(cfg.getProperty("gcs_repo_name"), cfg.getProperty("gcs_snapshot_name")).waitForCompletion( + true + ), + RequestOptions.DEFAULT + ); + + assertEquals( + "Unable to restore snapshot: " + + resp.getRestoreInfo().toString() + + System.lineSeparator() + + "Please check server logs to find the underlying issue.", + 1, + resp.getRestoreInfo().successfulShards() + ); + + assertEquals( + Long.parseLong(cfg.getProperty("index_doc_count")), + restHighLevelClient.count(new CountRequest(cfg.getProperty("index_name")), RequestOptions.DEFAULT).getCount() + ); + } + } +} diff --git a/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EsEQLCorrectnessIT.java b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EsEQLCorrectnessIT.java index 54f98c0904f..3de6b2ee173 100644 --- a/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EsEQLCorrectnessIT.java +++ b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EsEQLCorrectnessIT.java @@ -12,12 +12,8 @@ import org.apache.http.HttpHost; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.TimeUnits; -import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; -import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; @@ -47,7 +43,6 @@ public class EsEQLCorrectnessIT extends ESRestTestCase { private static final String PARAM_FORMATTING = "%1$s"; private static final String QUERIES_FILENAME = "queries.toml"; - private static final String PROPERTIES_FILENAME = "config.properties"; private static Properties CFG; private static RestHighLevelClient highLevelClient; @@ -58,10 +53,7 @@ public class EsEQLCorrectnessIT extends ESRestTestCase { @BeforeClass public static void init() throws IOException { - try (InputStream is = EsEQLCorrectnessIT.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME)) { - CFG = new Properties(); - CFG.load(is); - } + CFG = EqlDataLoader.loadConfiguration(); RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); builder.setHttpAsyncResponseConsumerFactory( @@ -72,27 +64,7 @@ public class EsEQLCorrectnessIT extends ESRestTestCase { @Before public void restoreDataFromGcsRepo() throws Exception { - if (client().performRequest(new Request("HEAD", "/" + CFG.getProperty("index_name"))).getStatusLine().getStatusCode() == 404) { - highLevelClient().snapshot() - .createRepository( - new PutRepositoryRequest(CFG.getProperty("gcs_repo_name")).type("gcs") - .settings( - Settings.builder() - .put("bucket", CFG.getProperty("gcs_bucket_name")) - .put("base_path", CFG.getProperty("gcs_base_path")) - .put("client", CFG.getProperty("gcs_client_name")) - .build() - ), - RequestOptions.DEFAULT - ); - highLevelClient().snapshot() - .restore( - new RestoreSnapshotRequest(CFG.getProperty("gcs_repo_name"), CFG.getProperty("gcs_snapshot_name")).waitForCompletion( - true - ), - RequestOptions.DEFAULT - ); - } + EqlDataLoader.restoreSnapshot(highLevelClient(), CFG); } @After @@ -105,18 +77,6 @@ public class EsEQLCorrectnessIT extends ESRestTestCase { LOGGER.info("Total time: {} ms", totalTime); } - @AfterClass - public static void wipeTestData() throws IOException { - try { - adminClient().performRequest(new Request("DELETE", "/*")); - } catch (ResponseException e) { - // 404 here just means we had no indexes - if (e.getResponse().getStatusLine().getStatusCode() != 404) { - throw e; - } - } - } - @Override protected boolean preserveClusterUponCompletion() { // Need to preserve data between parameterized tests runs diff --git a/x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/config.properties b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/config.properties index d6298f90ac1..89ebba5bc63 100644 --- a/x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/config.properties +++ b/x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/config.properties @@ -5,10 +5,11 @@ # index_name=mitre +index_doc_count=3950632 fetch_size=1000 size=2000 gcs_repo_name=eql_correctness_gcs_repo -gcs_snapshot_name=mitre-snapshot_7.10 -gcs_bucket_name=matriv-gcs -gcs_base_path=mitre-data +gcs_snapshot_name=correctness-snapshot_es7.10_lucene8.6.3 +gcs_bucket_name=eql-gcs +gcs_base_path=correctness-data gcs_client_name=eql_test