From 50d7424e7d3219d90f9d5fcc3c8c92fd1241badd Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Tue, 29 Oct 2019 13:39:19 -0700
Subject: [PATCH] Unmute and increase logging on flaky SLM tests (#48612)

The failures in these tests have been remarkably difficult to track
down, in part because they will not reproduce locally. This commit
unmutes the flaky tests and increases logging, as well as introducing
some additional logging, to attempt to pin down the failures.
---
 .../xpack/slm/SnapshotLifecycleRestIT.java    | 34 +++++++++++++++++--
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index e4a38c69ab6..5f6322a087e 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.slm;
 
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -27,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.Step;
@@ -195,6 +195,8 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     }
 
     @SuppressWarnings("unchecked")
+    @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE,org.elasticsearch.xpack.core.slm:TRACE,org.elasticsearch.snapshots:DEBUG",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/48531")
     public void testPolicyManualExecution() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
@@ -204,6 +206,8 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
             index(client(), indexName, "" + i, "foo", "bar");
         }
 
+        logSLMPolicies();
+
         // Create a snapshot repo
         initializeRepo(repoId);
 
@@ -216,6 +220,8 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
 
         final String snapshotName = executePolicy(policyName);
 
+        logSLMPolicies();
+
         // Check that the executed snapshot is created
         assertBusy(() -> {
             try {
@@ -336,7 +342,8 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     }
 
     @SuppressWarnings("unchecked")
-    @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48017")
+    @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE,org.elasticsearch.xpack.core.slm:TRACE,org.elasticsearch.snapshots:TRACE",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/48017")
     public void testBasicTimeBasedRetenion() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
@@ -346,6 +353,7 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
         for (int i = 0; i < docCount; i++) {
             index(client(), indexName, "" + i, "foo", "bar");
         }
+        logSLMPolicies();
 
         // Create a snapshot repo
         initializeRepo(repoId);
@@ -383,8 +391,11 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
             Request r = new Request("PUT", "/_cluster/settings");
             r.setJsonEntity(Strings.toString(builder));
             Response updateSettingsResp = client().performRequest(r);
+            assertAcked(updateSettingsResp);
         }
 
+        logSLMPolicies();
+
         try {
             // Check that the snapshot created by the policy has been removed by retention
             assertBusy(() -> {
@@ -618,7 +629,8 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
         XContentBuilder lifecycleBuilder = JsonXContent.contentBuilder();
         policy.toXContent(lifecycleBuilder, ToXContent.EMPTY_PARAMS);
         putLifecycle.setJsonEntity(Strings.toString(lifecycleBuilder));
-        assertOK(client().performRequest(putLifecycle));
+        final Response response = client().performRequest(putLifecycle);
+        assertAcked(response);
     }
 
     private void initializeRepo(String repoName) throws IOException {
@@ -659,4 +671,20 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
                 m -> (String) m.get(SnapshotLifecycleStats.SnapshotPolicyStats.POLICY_ID.getPreferredName()),
                 Function.identity()));
     }
+
+    private void assertAcked(Response response) throws IOException {
+        assertOK(response);
+        Map<String, Object> putLifecycleResponseMap;
+        try (InputStream is = response.getEntity().getContent()) {
+            putLifecycleResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+        }
+        assertThat(putLifecycleResponseMap.get("acknowledged"), equalTo(true));
+    }
+
+    private void logSLMPolicies() throws IOException {
+        Request request = new Request("GET" , "/_slm/policy?human");
+        Response response = client().performRequest(request);
+        assertOK(response);
+        logger.info("SLM policies: {}", EntityUtils.toString(response.getEntity()));
+    }
 }