diff --git a/plugin/ml-cpp-snapshot/build.gradle b/plugin/ml-cpp-snapshot/build.gradle deleted file mode 100644 index 2be5c83f181..00000000000 --- a/plugin/ml-cpp-snapshot/build.gradle +++ /dev/null @@ -1,124 +0,0 @@ -import com.amazonaws.AmazonServiceException -import com.amazonaws.ClientConfiguration -import com.amazonaws.auth.AWSCredentials -import com.amazonaws.auth.BasicAWSCredentials -import com.amazonaws.services.s3.AmazonS3Client -import com.amazonaws.services.s3.model.S3Object -import com.amazonaws.services.s3.model.ObjectMetadata -import com.bettercloud.vault.Vault -import com.bettercloud.vault.VaultConfig -import com.bettercloud.vault.response.LogicalResponse -import java.nio.file.Files -import java.nio.file.attribute.PosixFilePermission -import java.nio.file.attribute.PosixFilePermissions -import org.elasticsearch.gradle.VersionProperties - -apply plugin: 'distribution' - -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0" - classpath 'com.amazonaws:aws-java-sdk-s3:1.10.33' - if (JavaVersion.current() > JavaVersion.VERSION_1_8) { - classpath 'com.sun.xml.bind:jaxb-impl:2.2.3-1' // pulled in as external dependency to work on java 9 - } - } -} - -ext.version = VersionProperties.elasticsearch - -// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact -// for this project so it can be used with dependency substitution. We do not use gradle's -// handling of S3 as a maven repo due to the dynamically generated creds being slow to propagate, -// necessitating retries. - -void checkJavaVersion() { - /** - * The Elastic Secrets vault is served via HTTPS with a Let's Encrypt certificate. The root certificates that cross-signed the Let's - * Encrypt certificates were not trusted by the JDK until 8u101. Therefore, we enforce that the JDK is at least 8u101 here. - */ - final String javaVersion = System.getProperty('java.version') - final String javaVendor = System.getProperty('java.vendor') - def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/ - boolean matches = matcher.matches() - if (matches) { - final int update - if (matcher.group(1) == null) { - update = 0 - } else { - update = matcher.group(1).toInteger() - } - if (update < 101) { - throw new GradleException("JDK ${javaVendor} ${javaVersion} does not have necessary root certificates " + - "(https://bugs.openjdk.java.net/browse/JDK-8154757), update your JDK to at least JDK 8u101+") - } - } -} - -void getZip(File snapshotZip) { - final AmazonS3Client client = new AmazonS3Client() - final String key = "maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip" - int retries = 5 - while (retries > 0) { - try { - File snapshotMd5 = new File(snapshotZip.toString() + '.md5') - // do a HEAD first to check the zip hash against the local file - ObjectMetadata metadata = client.getObjectMetadata('prelert-artifacts', key) - String remoteMd5 = metadata.getETag() - if (snapshotZip.exists()) { - // do a HEAD first to check the zip hash against the local file - String localMd5 = snapshotMd5.getText('UTF-8') - if (remoteMd5.equals(localMd5)) { - logger.info('Using cached ML snapshot') - return - } - } - S3Object zip = client.getObject('prelert-artifacts', key) - InputStream zipStream = zip.getObjectContent() - try { - project.delete(snapshotZip, snapshotZip) - Files.copy(zipStream, snapshotZip.toPath()) - } finally { - zipStream.close() - } - snapshotMd5.setText(remoteMd5, 'UTF-8') - return - } catch (AmazonServiceException e) { - if (e.getStatusCode() != 403) { - throw new GradleException('Error while trying to get ml-cpp snapshot: ' + e.getMessage(), e) - } - sleep(500) - retries-- - } - } - throw new GradleException('Could not access ml-cpp artifacts. Timed out after 60 seconds') -} - -File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip") -task downloadMachineLearningSnapshot { - onlyIf { - // skip if machine-learning-cpp is being built locally - findProject(':machine-learning-cpp') == null && - // skip for offline builds - just rely on the artifact already having been downloaded before here - project.gradle.startParameter.isOffline() == false - } - doFirst { - snapshotZip.parentFile.mkdirs() - getZip(snapshotZip) - } -} - -gradle.taskGraph.whenReady { taskGraph -> - // skip if machine-learning-cpp is being built locally and also for offline builds - if (findProject(':machine-learning-cpp') == null && project.gradle.startParameter.isOffline() == false) { - // do validation of token/java version up front, don't wait for the task to run - checkJavaVersion() - } -} - -artifacts { - 'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot -} diff --git a/plugin/ml/build.gradle b/plugin/ml/build.gradle index e1a83aef374..9386633f3c4 100644 --- a/plugin/ml/build.gradle +++ b/plugin/ml/build.gradle @@ -22,7 +22,7 @@ configurations { if (findProject(':machine-learning-cpp') != null) { substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp") } else { - substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.parent.path}:ml-cpp-snapshot") + substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:cpp-snapshot") } } } diff --git a/plugin/ml-cpp-snapshot/.gitignore b/plugin/ml/cpp-snapshot/.gitignore similarity index 100% rename from plugin/ml-cpp-snapshot/.gitignore rename to plugin/ml/cpp-snapshot/.gitignore diff --git a/plugin/ml/cpp-snapshot/build.gradle b/plugin/ml/cpp-snapshot/build.gradle new file mode 100644 index 00000000000..c2a396e72b2 --- /dev/null +++ b/plugin/ml/cpp-snapshot/build.gradle @@ -0,0 +1,52 @@ +import java.net.HttpURLConnection +import org.elasticsearch.gradle.VersionProperties + +apply plugin: 'distribution' + +ext.version = VersionProperties.elasticsearch + +// This project pulls a snapshot version of the ML cpp artifacts and sets that as the artifact +// for this project so it can be used with dependency substitution. + +void getZip(File snapshotZip) { + String zipUrl = "http://prelert-artifacts.s3.amazonaws.com/maven/org/elasticsearch/ml/ml-cpp/${version}/ml-cpp-${version}.zip" + File snapshotMd5 = new File(snapshotZip.toString() + '.md5') + HttpURLConnection conn = (HttpURLConnection) new URL(zipUrl).openConnection(); + + // do a HEAD first to check the zip hash against the local file + conn.setRequestMethod('HEAD'); + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { + throw new GradleException('ML cpp snapshot does not exist') + } + + String remoteMd5 = conn.getHeaderField('ETag') + if (snapshotZip.exists()) { + // do a HEAD first to check the zip hash against the local file + String localMd5 = snapshotMd5.getText('UTF-8') + if (remoteMd5.equals(localMd5)) { + logger.info('Using cached ML snapshot') + return + } + } + + snapshotZip.bytes = new URL(zipUrl).bytes + snapshotMd5.setText(remoteMd5, 'UTF-8') +} + +File snapshotZip = new File(projectDir, ".cache/ml-cpp-${version}.zip") +task downloadMachineLearningSnapshot { + onlyIf { + // skip if machine-learning-cpp is being built locally + findProject(':machine-learning-cpp') == null && + // skip for offline builds - just rely on the artifact already having been downloaded before here + project.gradle.startParameter.isOffline() == false + } + doFirst { + snapshotZip.parentFile.mkdirs() + getZip(snapshotZip) + } +} + +artifacts { + 'default' file: snapshotZip, name: 'ml-cpp', type: 'zip', builtBy: downloadMachineLearningSnapshot +}