Build: Move and simplify ml cpp snapshot (elastic/x-pack-elasticsearch#3762)

Now that ML binaries are public, there is no longer a need to use the s3
client to access the bucket, since creds are not needed. This commit
also moves the cpp snapshot project under the ml module, since it is
specific to that and does not need to clutter the plugin dir.

Original commit: elastic/x-pack-elasticsearch@51e77da4ac

plugin/ml-cpp-snapshot/build.gradle

@@ -1,124 +0,0 @@
-import com.amazonaws.AmazonServiceException
-import com.amazonaws.ClientConfiguration
-import com.amazonaws.auth.AWSCredentials
-import com.amazonaws.auth.BasicAWSCredentials
-import com.amazonaws.services.s3.AmazonS3Client
-import com.amazonaws.services.s3.model.S3Object
-import com.amazonaws.services.s3.model.ObjectMetadata
-import com.bettercloud.vault.Vault
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.response.LogicalResponse
-import java.nio.file.Files
-import java.nio.file.attribute.PosixFilePermission
-import java.nio.file.attribute.PosixFilePermissions
-import org.elasticsearch.gradle.VersionProperties
-
-apply plugin: 'distribution'
-
-buildscript {
-  repositories {
-    mavenCentral()
-  }
-  dependencies {
-    classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0"
-    classpath 'com.amazonaws:aws-java-sdk-s3:1.10.33'
-    if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
-      classpath 'com.sun.xml.bind:jaxb-impl:2.2.3-1' // pulled in as external dependency to work on java 9
-    }
-  }
-}
-
-ext.version = VersionProperties.elasticsearch
-
-// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact
-// for this project so it can be used with dependency substitution. We do not use gradle's
-// handling of S3 as a maven repo due to the dynamically generated creds being slow to propagate,
-// necessitating retries.
-
-void checkJavaVersion() {
-  /**
-   * The Elastic Secrets vault is served via HTTPS with a Let's Encrypt certificate. The root certificates that cross-signed the Let's
-   * Encrypt certificates were not trusted by the JDK until 8u101. Therefore, we enforce that the JDK is at least 8u101 here.
-   */
-  final String javaVersion = System.getProperty('java.version')
-  final String javaVendor = System.getProperty('java.vendor')
-  def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/
-  boolean matches = matcher.matches()
-  if (matches) {
-    final int update
-    if (matcher.group(1) == null) {
-      update = 0
-    } else {
-      update = matcher.group(1).toInteger()
-    }
-    if (update < 101) {
-      throw new GradleException("JDK ${javaVendor} ${javaVersion} does not have necessary root certificates " +
-              "(https://bugs.openjdk.java.net/browse/JDK-8154757), update your JDK to at least JDK 8u101+")
-    }
-  }
-}
-
-void getZip(File snapshotZip) {
-  final AmazonS3Client client = new AmazonS3Client()
-  final String key = "maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip"
-  int retries = 5 
-  while (retries > 0) {
-    try {
-      File snapshotMd5 = new File(snapshotZip.toString() + '.md5')
-      // do a HEAD first to check the zip hash against the local file
-      ObjectMetadata metadata = client.getObjectMetadata('prelert-artifacts', key)
-      String remoteMd5 = metadata.getETag()
-      if (snapshotZip.exists()) {
-        // do a HEAD first to check the zip hash against the local file
-        String localMd5 = snapshotMd5.getText('UTF-8')
-        if (remoteMd5.equals(localMd5)) {
-          logger.info('Using cached ML snapshot')
-          return
-        }
-      }
-      S3Object zip = client.getObject('prelert-artifacts', key)
-      InputStream zipStream = zip.getObjectContent()
-      try {
-        project.delete(snapshotZip, snapshotZip)
-        Files.copy(zipStream, snapshotZip.toPath())
-      } finally {
-        zipStream.close()
-      }
-      snapshotMd5.setText(remoteMd5, 'UTF-8')
-      return
-    } catch (AmazonServiceException e) {
-      if (e.getStatusCode() != 403) {
-        throw new GradleException('Error while trying to get ml-cpp snapshot: ' + e.getMessage(), e)
-      }
-      sleep(500)
-      retries--
-    }
-  }
-  throw new GradleException('Could not access ml-cpp artifacts. Timed out after 60 seconds')
-}
-
-File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip")
-task downloadMachineLearningSnapshot {
-  onlyIf {
-    // skip if machine-learning-cpp is being built locally
-    findProject(':machine-learning-cpp') == null &&
-      // skip for offline builds - just rely on the artifact already having been downloaded before here
-      project.gradle.startParameter.isOffline() == false
-  }
-  doFirst {
-    snapshotZip.parentFile.mkdirs()
-    getZip(snapshotZip)
-  }
-}
-
-gradle.taskGraph.whenReady { taskGraph ->
-  // skip if machine-learning-cpp is being built locally and also for offline builds
-  if (findProject(':machine-learning-cpp') == null && project.gradle.startParameter.isOffline() == false) {
-    // do validation of token/java version up front, don't wait for the task to run
-    checkJavaVersion()
-  }
-}
-
-artifacts {
-  'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot
-}

plugin/ml/build.gradle

@@ -22,7 +22,7 @@ configurations {
             if (findProject(':machine-learning-cpp') != null) {
                 substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp")
             } else {
-                substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.parent.path}:ml-cpp-snapshot")
+                substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:cpp-snapshot")
             }
         }
     }

plugin/ml/cpp-snapshot/build.gradle

@@ -0,0 +1,52 @@
+import java.net.HttpURLConnection
+import org.elasticsearch.gradle.VersionProperties
+
+apply plugin: 'distribution'
+
+ext.version = VersionProperties.elasticsearch
+
+// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact
+// for this project so it can be used with dependency substitution.
+
+void getZip(File snapshotZip) {
+  String zipUrl = "http://prelert-artifacts.s3.amazonaws.com/maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip"
+  File snapshotMd5 = new File(snapshotZip.toString() + '.md5')
+  HttpURLConnection conn = (HttpURLConnection) new URL(zipUrl).openConnection();
+  
+  // do a HEAD first to check the zip hash against the local file
+  conn.setRequestMethod('HEAD');
+  if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
+    throw new GradleException('ML cpp snapshot does not exist')
+  }
+
+  String remoteMd5 = conn.getHeaderField('ETag')
+  if (snapshotZip.exists()) {
+    // do a HEAD first to check the zip hash against the local file
+    String localMd5 = snapshotMd5.getText('UTF-8')
+    if (remoteMd5.equals(localMd5)) {
+      logger.info('Using cached ML snapshot')
+      return
+    }
+  }
+
+  snapshotZip.bytes = new URL(zipUrl).bytes
+  snapshotMd5.setText(remoteMd5, 'UTF-8')
+}
+
+File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip")
+task downloadMachineLearningSnapshot {
+  onlyIf {
+    // skip if machine-learning-cpp is being built locally
+    findProject(':machine-learning-cpp') == null &&
+      // skip for offline builds - just rely on the artifact already having been downloaded before here
+      project.gradle.startParameter.isOffline() == false
+  }
+  doFirst {
+    snapshotZip.parentFile.mkdirs()
+    getZip(snapshotZip)
+  }
+}
+
+artifacts {
+  'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot
+}