EQL: [Tests] enable server side debugging (#64308) (#64449)

Register a new task `runEqlCorrectnessNode` which enables developers to
start an ES node in debug mode, properly restore the correctness data
and then run queries against it.

Assert the index is restored correctly and use new snapshot.

(cherry picked from commit fc8c6dd56d602b4a62ee1ff484f00caab92dc6e2)

x-pack/plugin/eql/qa/correctness/README.md

@@ -56,7 +56,7 @@ can be found in queries.toml file) with the desired number of the query:
 #### Debug queries
 
 If one wants to check that the filtering subqueries of a sequence query yields the same results (to pinpoint that the
-possible failure is in the sequence algortihm), needs to enable this debug mode with the use of a parameter:
+possible failure is in the sequence algorithm), needs to enable this debug mode with the use of a parameter:
 
 ```shell script
 ./gradlew -p x-pack/plugin/eql/qa/correctness check -Dtests.eql_correctness_debug=true
@@ -66,3 +66,22 @@ or
 ./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {<queryNo>}" -Dtests.eql_correctness_debug=true
 ```
 
+### Run an ES node manually and run the tests against it 
+
+If one wants to run an ES node manually (most probably to be able to debug the server), needs to run the following:
+
+```shell script
+./gradlew :x-pack:plugin:eql:qa:correctness:runEqlCorrectnessNode --debug-jvm
+```
+
+**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
+
+Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the 
+`EqlDataLoader` class.
+
+Once the data is loaded, a specific query can be run against the running ES node with:
+```shell script
+./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {<queryNo>}" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=runTask-0
+```
+
+**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**

x-pack/plugin/eql/qa/correctness/build.gradle

@@ -1,7 +1,10 @@
 apply plugin: 'elasticsearch.java-rest-test'
 apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.testclusters'
 test.enabled = false
 
+import org.elasticsearch.gradle.testclusters.RunTask
+
 restResources {
   restApi {
     includeCore '_common', 'bulk', 'indices', 'snapshot'
@@ -19,14 +22,19 @@ dependencies {
 
 File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File
 
-testClusters.all {
+testClusters {
-  plugin ':plugins:repository-gcs'
+  all {
-  if (serviceAccountFile) {
+    plugin ':plugins:repository-gcs'
-    keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
+    if (serviceAccountFile) {
+      keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
+    }
+    testDistribution = 'DEFAULT'
+    setting 'xpack.license.self_generated.type', 'basic'
+    jvmArgs '-Xms4g', '-Xmx4g'
+  }
+  runTask {
+    jvmArgs '-Xms8g', '-Xmx8g'
   }
-  testDistribution = 'DEFAULT'
-  setting 'xpack.license.self_generated.type', 'basic'
-  jvmArgs '-Xms4g', '-Xmx4g'
 }
 
 tasks.named('javaRestTest').configure {
@@ -36,3 +44,8 @@ tasks.named('javaRestTest').configure {
     showStandardStreams = true
   }
 }
+
+tasks.register("runEqlCorrectnessNode", RunTask) {
+  useCluster testClusters.runTask;
+  description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials'
+}

x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDataLoader.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import org.apache.http.HttpHost;
+import org.apache.logging.log4j.core.config.plugins.util.PluginManager;
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Properties;
+
+import static org.elasticsearch.test.ESTestCase.assertEquals;
+
+public class EqlDataLoader {
+
+    private static final String PROPERTIES_FILENAME = "config.properties";
+
+    public static void main(String[] args) throws IOException {
+        // Need to setup the log configuration properly to avoid messages when creating a new RestClient
+        PluginManager.addPackage(LogConfigurator.class.getPackage().getName());
+
+        try (
+            RestClient client = RestClient.builder(new HttpHost("localhost", 9200))
+                .setRequestConfigCallback(
+                    requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(30000000)
+                        .setConnectionRequestTimeout(30000000)
+                        .setSocketTimeout(30000000)
+                )
+                .setStrictDeprecationMode(true)
+                .build()
+        ) {
+            Properties configuration = loadConfiguration();
+            restoreSnapshot(new RestHighLevelClient(client, ignore -> {}, Collections.emptyList()) {
+            }, configuration);
+        }
+    }
+
+    static Properties loadConfiguration() throws IOException {
+        try (InputStream is = EqlDataLoader.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME)) {
+            Properties props = new Properties();
+            props.load(is);
+            return props;
+        }
+    }
+
+    static void restoreSnapshot(RestHighLevelClient restHighLevelClient, Properties cfg) throws IOException {
+        if (restHighLevelClient.getLowLevelClient()
+            .performRequest(new Request("HEAD", "/" + cfg.getProperty("index_name")))
+            .getStatusLine()
+            .getStatusCode() == 404) {
+            restHighLevelClient.snapshot()
+                .createRepository(
+                    new PutRepositoryRequest(cfg.getProperty("gcs_repo_name")).type("gcs")
+                        .settings(
+                            Settings.builder()
+                                .put("bucket", cfg.getProperty("gcs_bucket_name"))
+                                .put("base_path", cfg.getProperty("gcs_base_path"))
+                                .put("client", cfg.getProperty("gcs_client_name"))
+                                .build()
+                        ),
+                    RequestOptions.DEFAULT
+                );
+            RestoreSnapshotResponse resp = restHighLevelClient.snapshot()
+                .restore(
+                    new RestoreSnapshotRequest(cfg.getProperty("gcs_repo_name"), cfg.getProperty("gcs_snapshot_name")).waitForCompletion(
+                        true
+                    ),
+                    RequestOptions.DEFAULT
+                );
+
+            assertEquals(
+                "Unable to restore snapshot: "
+                    + resp.getRestoreInfo().toString()
+                    + System.lineSeparator()
+                    + "Please check server logs to find the underlying issue.",
+                1,
+                resp.getRestoreInfo().successfulShards()
+            );
+
+            assertEquals(
+                Long.parseLong(cfg.getProperty("index_doc_count")),
+                restHighLevelClient.count(new CountRequest(cfg.getProperty("index_name")), RequestOptions.DEFAULT).getCount()
+            );
+        }
+    }
+}

x-pack/plugin/eql/qa/correctness/src/javaRestTest/java/org/elasticsearch/xpack/eql/EsEQLCorrectnessIT.java

@@ -12,12 +12,8 @@ import org.apache.http.HttpHost;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.TimeUnits;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
-import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -47,7 +43,6 @@ public class EsEQLCorrectnessIT extends ESRestTestCase {
 
     private static final String PARAM_FORMATTING = "%1$s";
     private static final String QUERIES_FILENAME = "queries.toml";
-    private static final String PROPERTIES_FILENAME = "config.properties";
 
     private static Properties CFG;
     private static RestHighLevelClient highLevelClient;
@@ -58,10 +53,7 @@ public class EsEQLCorrectnessIT extends ESRestTestCase {
 
     @BeforeClass
     public static void init() throws IOException {
-        try (InputStream is = EsEQLCorrectnessIT.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME)) {
+        CFG = EqlDataLoader.loadConfiguration();
-            CFG = new Properties();
-            CFG.load(is);
-        }
 
         RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
         builder.setHttpAsyncResponseConsumerFactory(
@@ -72,27 +64,7 @@ public class EsEQLCorrectnessIT extends ESRestTestCase {
 
     @Before
     public void restoreDataFromGcsRepo() throws Exception {
-        if (client().performRequest(new Request("HEAD", "/" + CFG.getProperty("index_name"))).getStatusLine().getStatusCode() == 404) {
+        EqlDataLoader.restoreSnapshot(highLevelClient(), CFG);
-            highLevelClient().snapshot()
-                .createRepository(
-                    new PutRepositoryRequest(CFG.getProperty("gcs_repo_name")).type("gcs")
-                        .settings(
-                            Settings.builder()
-                                .put("bucket", CFG.getProperty("gcs_bucket_name"))
-                                .put("base_path", CFG.getProperty("gcs_base_path"))
-                                .put("client", CFG.getProperty("gcs_client_name"))
-                                .build()
-                        ),
-                    RequestOptions.DEFAULT
-                );
-            highLevelClient().snapshot()
-                .restore(
-                    new RestoreSnapshotRequest(CFG.getProperty("gcs_repo_name"), CFG.getProperty("gcs_snapshot_name")).waitForCompletion(
-                        true
-                    ),
-                    RequestOptions.DEFAULT
-                );
-        }
     }
 
     @After
@@ -105,18 +77,6 @@ public class EsEQLCorrectnessIT extends ESRestTestCase {
         LOGGER.info("Total time: {} ms", totalTime);
     }
 
-    @AfterClass
-    public static void wipeTestData() throws IOException {
-        try {
-            adminClient().performRequest(new Request("DELETE", "/*"));
-        } catch (ResponseException e) {
-            // 404 here just means we had no indexes
-            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
-                throw e;
-            }
-        }
-    }
-
     @Override
     protected boolean preserveClusterUponCompletion() {
         // Need to preserve data between parameterized tests runs

x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/config.properties

@@ -5,10 +5,11 @@
 #
 
 index_name=mitre
+index_doc_count=3950632
 fetch_size=1000
 size=2000
 gcs_repo_name=eql_correctness_gcs_repo
-gcs_snapshot_name=mitre-snapshot_7.10
+gcs_snapshot_name=correctness-snapshot_es7.10_lucene8.6.3
-gcs_bucket_name=matriv-gcs
+gcs_bucket_name=eql-gcs
-gcs_base_path=mitre-data
+gcs_base_path=correctness-data
 gcs_client_name=eql_test