From e583b72c8dc3fb335178e536ef3932d0a4b1653f Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 26 Jan 2018 20:14:07 -0800 Subject: [PATCH] Build: Move and simplify ml cpp snapshot (elastic/x-pack-elasticsearch#3762) Now that ML binaries are public, there is no longer a need to use the s3 client to access the bucket, since creds are not needed. This commit also moves the cpp snapshot project under the ml module, since it is specific to that and does not need to clutter the plugin dir. Original commit: elastic/x-pack-elasticsearch@51e77da4acbcfc6cf2486e00a220c99cbaa46ed3 --- plugin/ml-cpp-snapshot/build.gradle | 124 ------------------ plugin/ml/build.gradle | 2 +- .../cpp-snapshot}/.gitignore | 0 plugin/ml/cpp-snapshot/build.gradle | 52 ++++++++ 4 files changed, 53 insertions(+), 125 deletions(-) delete mode 100644 plugin/ml-cpp-snapshot/build.gradle rename plugin/{ml-cpp-snapshot => ml/cpp-snapshot}/.gitignore (100%) create mode 100644 plugin/ml/cpp-snapshot/build.gradle diff --git a/plugin/ml-cpp-snapshot/build.gradle b/plugin/ml-cpp-snapshot/build.gradle deleted file mode 100644 index 2be5c83f181..00000000000 --- a/plugin/ml-cpp-snapshot/build.gradle +++ /dev/null @@ -1,124 +0,0 @@ -import com.amazonaws.AmazonServiceException -import com.amazonaws.ClientConfiguration -import com.amazonaws.auth.AWSCredentials -import com.amazonaws.auth.BasicAWSCredentials -import com.amazonaws.services.s3.AmazonS3Client -import com.amazonaws.services.s3.model.S3Object -import com.amazonaws.services.s3.model.ObjectMetadata -import com.bettercloud.vault.Vault -import com.bettercloud.vault.VaultConfig -import com.bettercloud.vault.response.LogicalResponse -import java.nio.file.Files -import java.nio.file.attribute.PosixFilePermission -import java.nio.file.attribute.PosixFilePermissions -import org.elasticsearch.gradle.VersionProperties - -apply plugin: 'distribution' - -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0" - classpath 'com.amazonaws:aws-java-sdk-s3:1.10.33' - if (JavaVersion.current() > JavaVersion.VERSION_1_8) { - classpath 'com.sun.xml.bind:jaxb-impl:2.2.3-1' // pulled in as external dependency to work on java 9 - } - } -} - -ext.version = VersionProperties.elasticsearch - -// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact -// for this project so it can be used with dependency substitution. We do not use gradle's -// handling of S3 as a maven repo due to the dynamically generated creds being slow to propagate, -// necessitating retries. - -void checkJavaVersion() { - /** - * The Elastic Secrets vault is served via HTTPS with a Let's Encrypt certificate. The root certificates that cross-signed the Let's - * Encrypt certificates were not trusted by the JDK until 8u101. Therefore, we enforce that the JDK is at least 8u101 here. - */ - final String javaVersion = System.getProperty('java.version') - final String javaVendor = System.getProperty('java.vendor') - def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/ - boolean matches = matcher.matches() - if (matches) { - final int update - if (matcher.group(1) == null) { - update = 0 - } else { - update = matcher.group(1).toInteger() - } - if (update < 101) { - throw new GradleException("JDK ${javaVendor} ${javaVersion} does not have necessary root certificates " + - "(https://bugs.openjdk.java.net/browse/JDK-8154757), update your JDK to at least JDK 8u101+") - } - } -} - -void getZip(File snapshotZip) { - final AmazonS3Client client = new AmazonS3Client() - final String key = "maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip" - int retries = 5 - while (retries > 0) { - try { - File snapshotMd5 = new File(snapshotZip.toString() + '.md5') - // do a HEAD first to check the zip hash against the local file - ObjectMetadata metadata = client.getObjectMetadata('prelert-artifacts', key) - String remoteMd5 = metadata.getETag() - if (snapshotZip.exists()) { - // do a HEAD first to check the zip hash against the local file - String localMd5 = snapshotMd5.getText('UTF-8') - if (remoteMd5.equals(localMd5)) { - logger.info('Using cached ML snapshot') - return - } - } - S3Object zip = client.getObject('prelert-artifacts', key) - InputStream zipStream = zip.getObjectContent() - try { - project.delete(snapshotZip, snapshotZip) - Files.copy(zipStream, snapshotZip.toPath()) - } finally { - zipStream.close() - } - snapshotMd5.setText(remoteMd5, 'UTF-8') - return - } catch (AmazonServiceException e) { - if (e.getStatusCode() != 403) { - throw new GradleException('Error while trying to get ml-cpp snapshot: ' + e.getMessage(), e) - } - sleep(500) - retries-- - } - } - throw new GradleException('Could not access ml-cpp artifacts. Timed out after 60 seconds') -} - -File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip") -task downloadMachineLearningSnapshot { - onlyIf { - // skip if machine-learning-cpp is being built locally - findProject(':machine-learning-cpp') == null && - // skip for offline builds - just rely on the artifact already having been downloaded before here - project.gradle.startParameter.isOffline() == false - } - doFirst { - snapshotZip.parentFile.mkdirs() - getZip(snapshotZip) - } -} - -gradle.taskGraph.whenReady { taskGraph -> - // skip if machine-learning-cpp is being built locally and also for offline builds - if (findProject(':machine-learning-cpp') == null && project.gradle.startParameter.isOffline() == false) { - // do validation of token/java version up front, don't wait for the task to run - checkJavaVersion() - } -} - -artifacts { - 'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot -} diff --git a/plugin/ml/build.gradle b/plugin/ml/build.gradle index e1a83aef374..9386633f3c4 100644 --- a/plugin/ml/build.gradle +++ b/plugin/ml/build.gradle @@ -22,7 +22,7 @@ configurations { if (findProject(':machine-learning-cpp') != null) { substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp") } else { - substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.parent.path}:ml-cpp-snapshot") + substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:cpp-snapshot") } } } diff --git a/plugin/ml-cpp-snapshot/.gitignore b/plugin/ml/cpp-snapshot/.gitignore similarity index 100% rename from plugin/ml-cpp-snapshot/.gitignore rename to plugin/ml/cpp-snapshot/.gitignore diff --git a/plugin/ml/cpp-snapshot/build.gradle b/plugin/ml/cpp-snapshot/build.gradle new file mode 100644 index 00000000000..c2a396e72b2 --- /dev/null +++ b/plugin/ml/cpp-snapshot/build.gradle @@ -0,0 +1,52 @@ +import java.net.HttpURLConnection +import org.elasticsearch.gradle.VersionProperties + +apply plugin: 'distribution' + +ext.version = VersionProperties.elasticsearch + +// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact +// for this project so it can be used with dependency substitution. + +void getZip(File snapshotZip) { + String zipUrl = "http://prelert-artifacts.s3.amazonaws.com/maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip" + File snapshotMd5 = new File(snapshotZip.toString() + '.md5') + HttpURLConnection conn = (HttpURLConnection) new URL(zipUrl).openConnection(); + + // do a HEAD first to check the zip hash against the local file + conn.setRequestMethod('HEAD'); + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { + throw new GradleException('ML cpp snapshot does not exist') + } + + String remoteMd5 = conn.getHeaderField('ETag') + if (snapshotZip.exists()) { + // do a HEAD first to check the zip hash against the local file + String localMd5 = snapshotMd5.getText('UTF-8') + if (remoteMd5.equals(localMd5)) { + logger.info('Using cached ML snapshot') + return + } + } + + snapshotZip.bytes = new URL(zipUrl).bytes + snapshotMd5.setText(remoteMd5, 'UTF-8') +} + +File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip") +task downloadMachineLearningSnapshot { + onlyIf { + // skip if machine-learning-cpp is being built locally + findProject(':machine-learning-cpp') == null && + // skip for offline builds - just rely on the artifact already having been downloaded before here + project.gradle.startParameter.isOffline() == false + } + doFirst { + snapshotZip.parentFile.mkdirs() + getZip(snapshotZip) + } +} + +artifacts { + 'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot +}