From 56e983b5f300bbcc062878a9a1f4267926f7e930 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Thu, 9 Feb 2017 15:28:46 +0000
Subject: [PATCH] After auth with Vault retry until can see bucket
 (elastic/elasticsearch#4912)

After authenticating with vault it can take time for the credentials to be propagated by the AWS API. previously we would just blindly wait for 10 seconds and then try to continue. This change introduces a retry loop where we will do a `headBucket` request every 0.5 seconds until the bucket is accessible or until we have tried 15 times. This means the build is only held up for at most 0.5 seconds after the bucket is accessible.

This is a step towards the final solution since  the authentication with vault still happens on every build in the configuraiton phase. A subsequent change will be made to move this out of the configuration phase so that it only runs when the dependencies are required.

Original commit: elastic/x-pack-elasticsearch@ab3abba1ea082e3ad46b29f8ad7b65ce47381cc7
---
 elasticsearch/build.gradle | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/elasticsearch/build.gradle b/elasticsearch/build.gradle
index 0e0e27172ec..ada844f20ea 100644
--- a/elasticsearch/build.gradle
+++ b/elasticsearch/build.gradle
@@ -6,6 +6,12 @@ import org.elasticsearch.gradle.VersionProperties
 import com.bettercloud.vault.Vault
 import com.bettercloud.vault.VaultConfig
 import com.bettercloud.vault.response.LogicalResponse
+import com.amazonaws.AmazonServiceException
+import com.amazonaws.ClientConfiguration
+import com.amazonaws.auth.AWSCredentials
+import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.services.s3.AmazonS3Client
+import com.amazonaws.services.s3.model.HeadBucketRequest
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
@@ -30,6 +36,7 @@ buildscript {
   }
   dependencies {
     classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0"
+    classpath 'com.amazonaws:aws-java-sdk-s3:1.10.33'
   }
 }
 
@@ -74,8 +81,30 @@ Vault vault = new Vault(config)
 LogicalResponse secret = vault.logical().read("aws-dev/creds/prelertartifacts")
 String mlAwsAccessKey = secret.data.get('access_key')
 String mlAwsSecretKey = secret.data.get('secret_key')
-// Sleeping to give AWS a chance to propagate the credentials
-sleep(10000)
+// Retrying 10 times to give AWS a chance to propagate the credentials
+int retries = 60
+while (retries > 0) {
+  AWSCredentials creds = new BasicAWSCredentials(mlAwsAccessKey, mlAwsSecretKey)
+
+  ClientConfiguration clientConfiguration = new ClientConfiguration()
+  // the response metadata cache is only there for diagnostics purposes,
+  // but can force objects from every response to the old generation.
+  clientConfiguration.setResponseMetadataCacheSize(0)
+
+  AmazonS3Client client = new AmazonS3Client(creds, clientConfiguration)
+  try {
+    client.headBucket(new HeadBucketRequest('prelert-artifacts'))
+    break;
+  } catch (AmazonServiceException e) {
+    if (e.getStatusCode() != 403 || retries == 0) {
+      throw new GradleException('Could not access ml-cpp artifacts. Timed out after 60 attempts', e)
+    }
+  }
+  sleep(500)
+  retries--
+}
+
+
 
 repositories {
   maven {