Remove random when using HLRC sync and async calls (#48211)

This commit removes the randomization used by every execute call in the
high level rest tests. Previously every execute call, which can be many
calls per single test, would rely on a random boolean to determine if
they should use the sync or async methods provided to the execute
method. This commit runs the tests twice, using two different clusters,
both of them providing the value one time via a sysprop. This ensures
that the whole suite of tests is run using the sync and async code
paths.

Closes #39667

client/rest-high-level/build.gradle

@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -103,11 +105,22 @@ File nodeTrustStore = file("./testnode.jks")
 File pkiTrustCert = file("./src/test/resources/org/elasticsearch/client/security/delegate_pki/testRootCA.crt")
 
 integTest.runner {
+  systemProperty 'tests.rest.async', 'false'
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
   systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')
 }
 
-testClusters.integTest {
+RestIntegTestTask asyncIntegTest = tasks.create("asyncIntegTest", RestIntegTestTask) {
+  runner {
+    systemProperty 'tests.rest.async', 'true'
+    systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
+    systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')
+  }
+}
+
+check.dependsOn(asyncIntegTest)
+
+testClusters.all {
   testDistribution = 'DEFAULT'
   systemProperty 'es.scripting.update.ctx_in_params', 'false'
   setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]'

client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java

@@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -52,6 +53,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
 
     private static RestHighLevelClient restHighLevelClient;
+    private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));
 
     @Before
     public void initHighLevelClient() throws IOException {
@@ -84,7 +86,7 @@ public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
      */
     protected static <Req, Resp> Resp execute(Req request, SyncMethod<Req, Resp> syncMethod,
                                        AsyncMethod<Req, Resp> asyncMethod, RequestOptions options) throws IOException {
-        if (randomBoolean()) {
+        if (async == false) {
             return syncMethod.execute(request, options);
         } else {
             PlainActionFuture<Resp> future = PlainActionFuture.newFuture();
@@ -100,7 +102,7 @@ public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
      */
     protected static <Resp> Resp execute(SyncMethodNoRequest<Resp> syncMethodNoRequest, AsyncMethodNoRequest<Resp> asyncMethodNoRequest,
             RequestOptions requestOptions) throws IOException {
-        if (randomBoolean()) {
+        if (async == false) {
             return syncMethodNoRequest.execute(requestOptions);
         } else {
             PlainActionFuture<Resp> future = PlainActionFuture.newFuture();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java

@@ -194,7 +194,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
             //end::get-license-response
 
             assertThat(currentLicense, containsString("trial"));
-            assertThat(currentLicense, containsString("integTest"));
+            assertThat(currentLicense, containsString("ntegTest"));
         }
         {
             GetLicenseRequest request = new GetLicenseRequest();
@@ -233,7 +233,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
             String currentLicense = response.getLicenseDefinition();
             assertThat(currentLicense, startsWith("{"));
             assertThat(currentLicense, containsString("trial"));
-            assertThat(currentLicense, containsString("integTest"));
+            assertThat(currentLicense, containsString("ntegTest"));
             assertThat(currentLicense, endsWith("}"));
         }
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -48,6 +48,7 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -457,8 +458,13 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         List<VerifyRepositoryResponse.NodeView> repositoryMetaDataResponse = response.getNodes();
         // end::verify-repository-response
         assertThat(1, equalTo(repositoryMetaDataResponse.size()));
+        final boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));
+        if (async) {
+            assertThat("asyncIntegTest-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
+        } else {
             assertThat("integTest-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
         }
+    }
 
     public void testSnapshotVerifyRepositoryAsync() throws InterruptedException {
         RestHighLevelClient client = highLevelClient();

modules/transport-netty4/build.gradle

@@ -69,7 +69,7 @@ TaskProvider<Test> pooledTest = tasks.register("pooledTest", Test) {
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
   systemProperty 'es.use_unpooled_allocator', 'false'
 }
-// TODO: we can't use task avoidance here because RestIntegTestTask does the testcluster creation
+
 RestIntegTestTask pooledIntegTest = tasks.create("pooledIntegTest", RestIntegTestTask) {
   runner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'