Add option to preserve data in test clusters (#65400)

(cherry picked from commit 1ce323e1368cf5231181f1efaba1c4e425066e37)

TESTING.asciidoc

@@ -82,12 +82,13 @@ password: `elastic-password`.
 
 ==== Other useful arguments
 
-In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
-In order to disable assertions add: `-Dtests.asserts=false`
-In order to use a custom data directory: `--data-dir=/tmp/foo`
-In order to remotely attach a debugger to the process: `--debug-jvm`
-In order to set a different keystore password: `--keystore-password`
-In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
+- In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
+- In order to disable assertions add: `-Dtests.asserts=false`
+- In order to use a custom data directory: `--data-dir=/tmp/foo`
+- In order to preserve data in between executions: `--preserve-data`
+- In order to remotely attach a debugger to the process: `--debug-jvm`
+- In order to set a different keystore password: `--keystore-password`
+- In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 
 === Test case filtering.
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -282,6 +282,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         nodes.all(each -> each.jvmArgs(values));
     }
 
+    @Internal
+    public boolean isPreserveDataDir() {
+        return nodes.stream().anyMatch(node -> node.isPreserveDataDir());
+    }
+
+    @Override
+    public void setPreserveDataDir(boolean preserveDataDir) {
+        nodes.all(each -> each.setPreserveDataDir(preserveDataDir));
+    }
+
     @Override
     public void freeze() {
         nodes.forEach(ElasticsearchNode::freeze);

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -167,6 +167,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private String transportPort = "0";
     private Path confPathData;
     private String keystorePassword = "";
+    private boolean preserveDataDir = false;
 
     ElasticsearchNode(
         String path,
@@ -421,6 +422,17 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         return configFile.getParent();
     }
 
+    @Override
+    @Input
+    public boolean isPreserveDataDir() {
+        return preserveDataDir;
+    }
+
+    @Override
+    public void setPreserveDataDir(boolean preserveDataDir) {
+        this.preserveDataDir = preserveDataDir;
+    }
+
     @Override
     public void freeze() {
         requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
@@ -452,7 +464,13 @@ public class ElasticsearchNode implements TestClusterConfiguration {
                 logToProcessStdout("Configuring working directory: " + workingDir);
                 // make sure we always start fresh
                 if (Files.exists(workingDir)) {
-                    fileSystemOperations.delete(d -> d.delete(workingDir));
+                    if (preserveDataDir) {
+                        Files.list(workingDir)
+                            .filter(path -> path.equals(confPathData) == false)
+                            .forEach(path -> fileSystemOperations.delete(d -> d.delete(path)));
+                    } else {
+                        fileSystemOperations.delete(d -> d.delete(workingDir));
+                    }
                 }
                 isWorkingDirConfigured = true;
             }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -46,6 +46,8 @@ public class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
 
+    private Boolean preserveData = false;
+
     private Path dataDir = null;
 
     private String keystorePassword = "";
@@ -65,6 +67,16 @@ public class RunTask extends DefaultTestClustersTask {
         dataDir = Paths.get(dataDirStr).toAbsolutePath();
     }
 
+    @Input
+    public Boolean getPreserveData() {
+        return preserveData;
+    }
+
+    @Option(option = "preserve-data", description = "Preserves data directory contents (path provided to --data-dir is always preserved)")
+    public void setPreserveData(Boolean preserveData) {
+        this.preserveData = preserveData;
+    }
+
     @Option(option = "keystore-password", description = "Set the elasticsearch keystore password")
     public void setKeystorePassword(String password) {
         keystorePassword = password;
@@ -113,6 +125,7 @@ public class RunTask extends DefaultTestClustersTask {
             httpPort++;
             cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
             transportPort++;
+            cluster.setPreserveDataDir(preserveData);
             for (ElasticsearchNode node : cluster.getNodes()) {
                 additionalSettings.forEach(node::setting);
                 if (dataDir != null) {

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -66,6 +66,13 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
                     .filter(task -> task != this)
                     .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false)
             );
+
+        this.getOutputs()
+            .doNotCacheIf(
+                "Caching disabled for this task since it is configured to preserve data directory",
+                // Don't cache the output of this task if it's not running from a clean data directory.
+                t -> getClusters().stream().anyMatch(cluster -> cluster.isPreserveDataDir())
+            );
     }
 
     @Override

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -86,6 +86,10 @@ public interface TestClusterConfiguration {
 
     void jvmArgs(String... values);
 
+    boolean isPreserveDataDir();
+
+    void setPreserveDataDir(boolean preserveDataDir);
+
     void freeze();
 
     void start();

x-pack/plugin/eql/qa/correctness/README.md

@@ -85,3 +85,10 @@ Once the data is loaded, a specific query can be run against the running ES node
 ```
 
 **Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
+
+#### Preserve data across node restarts
+If you'd like to preserve the restored index and avoid the network download and delay of restoring them on every run of the node,
+you can set the `eql.test.preserve.data` system property, e.g.:
+```shell script
+./gradlew :x-pack:plugin:eql:qa:correctness:javaRestTest -Deql.test.preserve.data=true
+```

x-pack/plugin/eql/qa/correctness/build.gradle

@@ -20,7 +20,16 @@ dependencies {
   javaRestTestImplementation 'io.ous:jtoml:2.0.0'
 }
 
-File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File
+File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file')
+  .forUseAtConfigurationTime()
+  .orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime())
+  .map { s -> new File(s)}
+  .getOrNull()
+
+Boolean preserveData = providers.systemProperty('eql.test.preserve.data')
+  .forUseAtConfigurationTime()
+  .map { s -> Boolean.parseBoolean(s) }
+  .getOrElse(false)
 
 testClusters {
   all {
@@ -28,6 +37,9 @@ testClusters {
     if (serviceAccountFile) {
       keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
     }
+    if (preserveData) {
+      preserveDataDir = true
+    }
     testDistribution = 'DEFAULT'
     setting 'xpack.license.self_generated.type', 'basic'
     jvmArgs '-Xms4g', '-Xmx4g'
@@ -46,6 +58,6 @@ tasks.named('javaRestTest').configure {
 }
 
 tasks.register("runEqlCorrectnessNode", RunTask) {
-  useCluster testClusters.runTask;
+  useCluster testClusters.runTask
   description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials'
 }