diff --git a/build.gradle b/build.gradle index d8130c60d0e..0257b21afb2 100644 --- a/build.gradle +++ b/build.gradle @@ -150,6 +150,13 @@ subprojects { // elasticsearch snapshots mavenLocal() } + maven { + url "s3://prelert-artifacts/maven" + credentials(AwsCredentials) { + accessKey "${project.mlAwsAccessKey}" + secretKey "${project.mlAwsSecretKey}" + } + } mavenCentral() maven { name 'sonatype-snapshots' diff --git a/buildSrc/src/main/groovy/org/elastic/gradle/DownloadS3Task.groovy b/buildSrc/src/main/groovy/org/elastic/gradle/DownloadS3Task.groovy deleted file mode 100644 index 402cd95811d..00000000000 --- a/buildSrc/src/main/groovy/org/elastic/gradle/DownloadS3Task.groovy +++ /dev/null @@ -1,90 +0,0 @@ -package org.elastic.gradle - -import com.amazonaws.ClientConfiguration -import com.amazonaws.auth.AWSCredentials -import com.amazonaws.auth.BasicAWSCredentials -import com.amazonaws.services.s3.AmazonS3Client -import com.amazonaws.services.s3.model.GetObjectRequest -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.TaskAction -import org.gradle.logging.ProgressLogger -import org.gradle.logging.ProgressLoggerFactory - -import javax.inject.Inject - -/** - * A task to download files to s3, which allows delayed resolution of the s3 path - */ -class DownloadS3Task extends DefaultTask { - - - private List toDownload = new ArrayList<>() - - @Input - String bucket - - @Input - File destDir - - /** True if the file paths should be flattened into a single directory when downloaded, false otherwise */ - @Input - boolean flatten = false - - DownloadS3Task() { - ext.set('needs.aws', true) - } - - @Inject - public ProgressLoggerFactory getProgressLoggerFactory() { - throw new UnsupportedOperationException() - } - - /** - * Add a file to be uploaded to s3. The key object will be evaluated at runtime. - * - * If file is a directory, all files in the directory will be uploaded to the key as a prefix. - */ - public void download(Object key) { - toDownload.add(key) - } - - @TaskAction - public void downloadFromS3() { - AWSCredentials creds = new BasicAWSCredentials(project.mlAwsAccessKey, project.mlAwsSecretKey) - - ClientConfiguration clientConfiguration = new ClientConfiguration(); - // the response metadata cache is only there for diagnostics purposes, - // but can force objects from every response to the old generation. - clientConfiguration.setResponseMetadataCacheSize(0); - - AmazonS3Client client = new AmazonS3Client(creds, clientConfiguration); - ProgressLogger progressLogger = getProgressLoggerFactory().newOperation("s3upload") - progressLogger.description = "download files from s3" - progressLogger.started() - - for (Object entry : toDownload) { - String key = entry.toString() - downloadFile(client, progressLogger, destDir, key) - } - progressLogger.completed() - } - - /** Download a single file */ - private void downloadFile(AmazonS3Client client, ProgressLogger progressLogger, File destDir, String key) { - File destPath = getDestinationPath(destDir, key, flatten) - logger.info("Downloading ${destPath} from ${bucket}") - progressLogger.progress("downloading ${destPath}") - client.getObject(new GetObjectRequest(bucket, key), destPath) - } - - private File getDestinationPath(File destDir, String key, boolean flatten) { - String destPath - if (flatten) { - destPath = key.substring(key.lastIndexOf('/') + 1) - } else { - destPath = key - } - return new File(destDir, destPath) - } -} diff --git a/elasticsearch/build.gradle b/elasticsearch/build.gradle index d2a1156148e..448a9077c51 100644 --- a/elasticsearch/build.gradle +++ b/elasticsearch/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks -import org.elastic.gradle.DownloadS3Task apply plugin: 'elasticsearch.esplugin' @@ -13,8 +12,15 @@ version = project.version thirdPartyAudit.enabled = false +configurations { + nativeBundle +} + dependencies { compile group: 'net.sf.supercsv', name: 'super-csv', version:"${supercsvVersion}" + nativeBundle group: "${project.group}", name: 'ml-cpp-test', version:"${project.version}", classifier: 'darwin-x86_64', ext: 'zip' + nativeBundle group: "${project.group}", name: 'ml-cpp-test', version:"${project.version}", classifier: 'linux-x86_64', ext: 'zip' + nativeBundle group: "${project.group}", name: 'ml-cpp-test', version:"${project.version}", classifier: 'windows-x86_64', ext: 'zip' testCompile group: 'org.ini4j', name: 'ini4j', version:"${ini4jVersion}" } @@ -40,22 +46,6 @@ integTest { integTest.mustRunAfter noBootstrapTest -String projectGroupPath = project.group.replaceAll("\\.", "/") - -task downloadCppDist(type: DownloadS3Task) { - enabled project.cppLocalDists == '' - description = 'Download C++ zips from S3 Bucket' - bucket 'prelert-artifacts' - destDir file("${buildDir}/cppDist") - flatten true - download "maven/${projectGroupPath}/ml-cpp/${project.version}/ml-cpp-${project.version}-windows-x86_64.zip" - download "maven/${projectGroupPath}/ml-cpp/${project.version}/ml-cpp-${project.version}-linux-x86_64.zip" - download "maven/${projectGroupPath}/ml-cpp/${project.version}/ml-cpp-${project.version}-darwin-x86_64.zip" - outputs.file(file("${buildDir}/cppDist/ml-cpp-${project.version}-windows-x86_64.zip")) - outputs.file(file("${buildDir}/cppDist/ml-cpp-${project.version}-darwin-x86_64.zip")) - outputs.file(file("${buildDir}/cppDist/ml-cpp-${project.version}-linux-x86_64.zip")) -} - bundlePlugin { if (project.cppLocalDists) { String localZipFile = 'ml-cpp-${project.version}-' + @@ -63,11 +53,10 @@ bundlePlugin { (project.isLinux ? "linux-x86_64" : "sunos-x86_64"))) + ".zip" from { zipTree(cppLocalDists + '/' + localZipFile) } } else { - for (outputFile in downloadCppDist.outputs.files) { + for (outputFile in configurations.nativeBundle) { from(zipTree(outputFile)) { duplicatesStrategy 'exclude' } } - dependsOn 'downloadCppDist' } }