Removes files no longer needed
Original commit: elastic/x-pack-elasticsearch@8f197075a3
This commit is contained in:
Colin Goodheart-Smithe
parent
14a677396e
commit
fa7a82a945
169
build.gradle
169
build.gradle
|
|
@ -1,169 +0,0 @@
|
|||
description = 'Builds the Machine Learning Java classes and UI'
|
||||
|
||||
import org.gradle.plugins.ide.eclipse.model.SourceFolder
|
||||
import org.elasticsearch.gradle.precommit.LicenseHeadersTask
|
||||
import org.elasticsearch.gradle.VersionProperties
|
||||
import com.bettercloud.vault.Vault
|
||||
import com.bettercloud.vault.VaultConfig
|
||||
import com.bettercloud.vault.response.LogicalResponse
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.attribute.PosixFilePermission
|
||||
import java.nio.file.attribute.PosixFilePermissions
|
||||
|
||||
if (project.projectDir.name != 'prelert-legacy') {
|
||||
throw new GradleException('You must checkout prelert-legacy in the following directory: <path to Elasticsearch checkout>/../elasticsearch-extra/prelert-legacy')
|
||||
}
|
||||
|
||||
buildscript {
|
||||
repositories {
|
||||
mavenCentral()
|
||||
}
|
||||
dependencies {
|
||||
classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
// Vault auth to get keys for access to cpp artifacts
|
||||
|
||||
// first need to get an authentication token with vault
|
||||
File githubToken = project.file('github.token')
|
||||
final String VAULT_URL = 'https://secrets.elastic.co:8200'
|
||||
final String VAULT_ROLE_ID = "8e90dd88-5a8e-9c12-0da9-5439f293ff97"
|
||||
final String VAULT_SECRET_ID = System.env.VAULT_SECRET_ID
|
||||
String authBody = null
|
||||
URL vaultUrl = null
|
||||
if (githubToken.exists()) {
|
||||
try {
|
||||
Set<PosixFilePermission> perms = Files.getPosixFilePermissions(githubToken.toPath())
|
||||
if (perms.equals(PosixFilePermissions.fromString("rw-------")) == false) {
|
||||
throw new GradleException('github.token must have 600 permissions')
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// Assume not on a POSIX file system
|
||||
}
|
||||
vaultUrl = new URL(VAULT_URL + '/v1/auth/github/login')
|
||||
authBody = "{\"token\": \"${githubToken.getText('UTF-8').trim()}\"}"
|
||||
} else if (VAULT_SECRET_ID != null) {
|
||||
vaultUrl = new URL(VAULT_URL + '/v1/auth/approle/login')
|
||||
authBody = "{\"role_id\": \"${VAULT_ROLE_ID}\", \"secret_id\": \"${VAULT_SECRET_ID}\"}"
|
||||
} else {
|
||||
throw new GradleException('Missing github.token file or VAULT_SECRET_ID environment variable, needed to authenticate with vault for secrets')
|
||||
}
|
||||
HttpURLConnection vaultConn = (HttpURLConnection) vaultUrl.openConnection()
|
||||
vaultConn.setRequestProperty('Content-Type', 'application/json')
|
||||
vaultConn.setRequestMethod('PUT')
|
||||
vaultConn.setDoOutput(true)
|
||||
vaultConn.outputStream.withWriter('UTF-8') { writer ->
|
||||
writer.write(authBody)
|
||||
}
|
||||
vaultConn.connect()
|
||||
Object authResponse = new groovy.json.JsonSlurper().parseText(vaultConn.content.text)
|
||||
VaultConfig config = new VaultConfig(VAULT_URL, authResponse.auth.client_token)
|
||||
Vault vault = new Vault(config)
|
||||
LogicalResponse secret = vault.logical().read("aws-dev/creds/prelertartifacts")
|
||||
String mlAwsAccessKey = secret.data.get('access_key')
|
||||
String mlAwsSecretKey = secret.data.get('secret_key')
|
||||
// Sleeping to give AWS a chance to propagate the credentials
|
||||
sleep(3000)
|
||||
|
||||
String envCppLocalDists = System.env.CPP_LOCAL_DISTS
|
||||
if (envCppLocalDists != null) {
|
||||
project.ext.cppLocalDists = envCppLocalDists
|
||||
} else if (project.hasProperty("CPP_LOCAL_DISTS")) {
|
||||
project.ext.cppLocalDists = CPP_LOCAL_DISTS
|
||||
} else {
|
||||
project.ext.cppLocalDists = ''
|
||||
}
|
||||
|
||||
allprojects {
|
||||
group = 'org.elasticsearch.ml'
|
||||
}
|
||||
|
||||
task bundlePack(type: Zip) {
|
||||
onlyIf { project(':prelert-legacy:kibana').bundlePlugin.enabled }
|
||||
dependsOn ':prelert-legacy:elasticsearch:bundlePlugin'
|
||||
dependsOn ':prelert-legacy:kibana:bundlePlugin'
|
||||
from { zipTree(project(':prelert-legacy:elasticsearch').bundlePlugin.outputs.files.singleFile) }
|
||||
from { zipTree(project(':prelert-legacy:kibana').bundlePlugin.outputs.files.singleFile) }
|
||||
destinationDir file('build/distributions')
|
||||
baseName = 'ml'
|
||||
version = VersionProperties.elasticsearch
|
||||
}
|
||||
|
||||
subprojects {
|
||||
plugins.withType(MavenPublishPlugin).whenPluginAdded {
|
||||
publishing {
|
||||
publications {
|
||||
// add license information to generated poms
|
||||
all {
|
||||
pom.withXml { XmlProvider xml ->
|
||||
Node node = xml.asNode()
|
||||
|
||||
Node license = node.appendNode('licenses').appendNode('license')
|
||||
license.appendNode('name', 'Elastic Commercial Software End User License Agreement')
|
||||
license.appendNode('url', 'https://www.elastic.co/eula/')
|
||||
license.appendNode('distribution', 'repo')
|
||||
|
||||
Node developer = node.appendNode('developers').appendNode('developer')
|
||||
developer.appendNode('name', 'Elastic')
|
||||
developer.appendNode('url', 'http://www.elastic.co')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task assemble(dependsOn: bundlePack) {
|
||||
group = 'Build'
|
||||
description = 'Assembles the outputs of this project.'
|
||||
}
|
||||
|
||||
task build(dependsOn: assemble) {
|
||||
group = 'Build'
|
||||
description = 'Assembles and tests this project.'
|
||||
}
|
||||
|
||||
task test(dependsOn: [':prelert-legacy:elasticsearch:test', ':prelert-legacy:kibana:test']) {
|
||||
group = 'Build'
|
||||
description = 'Assembles and tests this project.'
|
||||
}
|
||||
|
||||
task clean(type: Delete) {
|
||||
group = 'Build'
|
||||
description = 'Deletes the build directory'
|
||||
delete 'build'
|
||||
}
|
||||
|
||||
subprojects {
|
||||
|
||||
tasks.withType(LicenseHeadersTask.class) {
|
||||
approvedLicenses = ['Elasticsearch Confidential']
|
||||
additionalLicense 'ESCON', 'Elasticsearch Confidential', 'ELASTICSEARCH CONFIDENTIAL'
|
||||
}
|
||||
ext.projectSubstitutions += [ "org.elasticsearch.plugin:ml-api:${version}": ':prelert-legacy:elasticsearch' ]
|
||||
}
|
||||
|
||||
allprojects {
|
||||
repositories {
|
||||
if (System.getProperty("repos.mavenlocal") != null) {
|
||||
// with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is useful for building against
|
||||
// elasticsearch snapshots
|
||||
mavenLocal()
|
||||
}
|
||||
maven {
|
||||
url "s3://prelert-artifacts/maven"
|
||||
credentials(AwsCredentials) {
|
||||
accessKey "${mlAwsAccessKey}"
|
||||
secretKey "${mlAwsSecretKey}"
|
||||
}
|
||||
}
|
||||
mavenCentral()
|
||||
maven {
|
||||
name 'sonatype-snapshots'
|
||||
url "https://oss.sonatype.org/content/repositories/snapshots/"
|
||||
}
|
||||
jcenter()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
File extrasDir = new File(settingsDir, '../..').getCanonicalFile()
|
||||
if (extrasDir.name.endsWith('-extra') == false) {
|
||||
throw new GradleException("prelert-legacy must be checked out under an elasticsearch-extra directory, found ${extrasDir.name}")
|
||||
}
|
||||
File elasticsearchDir = new File(extrasDir.parentFile, extrasDir.name[0..-7])
|
||||
if (elasticsearchDir.exists() == false) {
|
||||
throw new GradleException("${elasticsearchDir.name} is missing as a sibling to ${extrasDir.name}")
|
||||
}
|
||||
|
||||
project(':').projectDir = new File(elasticsearchDir, 'buildSrc')
|
||||
152
dev-tools/ci
152
dev-tools/ci
|
|
@ -1,152 +0,0 @@
|
|||
#!/bin/bash
|
||||
# This script is used as a single command to run the ml tests.
|
||||
#
|
||||
# It will attempt to check out 'elasticsearch' into a sibling directory
|
||||
# unless the environment variable `USE_EXISTING_ES` has a value. The
|
||||
# branch of elasticsearch which will be checked out depends on
|
||||
# environment variables. If running locally, set GIT_BRANCH. When
|
||||
# running in Jenkins, that env var is set. When running a PR
|
||||
# jenkins job, the variables PR_SOURCE_BRANCH and PR_TARGET_BRANCH
|
||||
# will be set and the source branch will be looked for in elasticsearch
|
||||
# before falling back to the target branch name.
|
||||
#
|
||||
# It will also attempt to install the appropriate version of node.js
|
||||
# for the Kibana plugin tests using nvm, unless
|
||||
# `xpack.kibana.build=false` is defined in
|
||||
# ~/.gradle/gradle.properties. Set a custom nvm directory using the
|
||||
# `NVM_DIR` environment variable.
|
||||
#
|
||||
|
||||
# Turn on semi-strict mode
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
# Allow the user choose different test through a single cli arg
|
||||
# default to `check` if no argument has been supplied
|
||||
key=${1-check}
|
||||
case $key in
|
||||
packagingTest)
|
||||
GRADLE_CLI_ARGS=(
|
||||
"--info"
|
||||
"-Pvagrant.boxes=all"
|
||||
":prelert-legacy:qa:vagrant:packagingTest"
|
||||
)
|
||||
;;
|
||||
check)
|
||||
GRADLE_CLI_ARGS=(
|
||||
"--info"
|
||||
"check"
|
||||
"build"
|
||||
"-Dtests.network=true"
|
||||
"-Dtests.badapples=true"
|
||||
)
|
||||
;;
|
||||
jdk9)
|
||||
GRADLE_CLI_ARGS=(
|
||||
"-Pxpack.kibana.build=false"
|
||||
"--info"
|
||||
"check"
|
||||
"-Dtests.network=true"
|
||||
"-Dtests.badapples=true"
|
||||
-Dtests.jvm.argline="--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED"
|
||||
)
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported cli argument $1. Allowed arguments are packagingTest or check. No argument defaults to check."
|
||||
exit 1;;
|
||||
esac
|
||||
|
||||
SCRIPT="$0"
|
||||
|
||||
# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
|
||||
while [ -h "$SCRIPT" ] ; do
|
||||
ls=$(ls -ld "$SCRIPT")
|
||||
# Drop everything prior to ->
|
||||
link=$(expr "$ls" : '.*-> \(.*\)$')
|
||||
if expr "$link" : '/.*' > /dev/null; then
|
||||
SCRIPT="$link"
|
||||
else
|
||||
SCRIPT=$(dirname "$SCRIPT")/"$link"
|
||||
fi
|
||||
done
|
||||
|
||||
# determine base directory
|
||||
BASE_DIR=$(dirname "$SCRIPT")/..
|
||||
|
||||
# make BASE_DIR absolute
|
||||
BASE_DIR=$(cd "$BASE_DIR"; pwd)
|
||||
|
||||
PARENT_DIR=$(cd "$BASE_DIR"/../..; pwd)
|
||||
|
||||
# go to the parent directory
|
||||
cd $PARENT_DIR
|
||||
|
||||
if [ -z ${USE_EXISTING_ES:+x} ]; then
|
||||
if [ -d "./elasticsearch" ]; then
|
||||
echo "I expected a clean workspace but an 'elasticsearch' sibling directory already exists in [$PARENT_DIR]!"
|
||||
echo
|
||||
echo "Either define 'USE_EXISTING_ES' or remove the existing 'elasticsearch' sibling."
|
||||
exit 1
|
||||
fi
|
||||
BRANCH=${PR_SOURCE_BRANCH:-${GIT_BRANCH#*/}} # GIT_BRANCH starts with the repo, i.e., origin/master
|
||||
BRANCH=${BRANCH:-master} # fall back to CI branch if not testing a PR
|
||||
echo "Checking if branch '$BRANCH' has elasticsearch sibling..."
|
||||
ES_REPO_URL="https://github.com/elastic/elasticsearch.git"
|
||||
if [ -z ${USE_SSH:+x} ]; then
|
||||
ES_REPO_URL="git@github.com:elastic/elasticsearch.git"
|
||||
fi
|
||||
if [[ -z "$(git ls-remote --heads $ES_REPO_URL $BRANCH)" ]]; then
|
||||
echo "No sibling branch, using PR target branch"
|
||||
BRANCH=$PR_TARGET_BRANCH
|
||||
fi
|
||||
echo "Checking out Elasticsearch '$BRANCH' branch..."
|
||||
git clone -b $BRANCH $ES_REPO_URL --depth=1
|
||||
else
|
||||
if [ -d "./elasticsearch" ]; then
|
||||
echo "Using existing 'elasticsearch' checkout"
|
||||
else
|
||||
echo "You have defined 'USE_EXISTING_ES' but no existing Elasticsearch directory exists!"
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
ES_COMMIT="$(cd "$PARENT_DIR/elasticsearch"; git show --oneline -s | cut -d\ -f1)"
|
||||
ML_COMMIT="$(cd "$BASE_DIR"; git show --oneline -s | cut -d\ -f1)"
|
||||
|
||||
# back to base directory
|
||||
cd "$BASE_DIR"
|
||||
|
||||
if ! grep -q -e '^xpack\.kibana\.build=false$' ~/.gradle/gradle.properties
|
||||
then
|
||||
# install the correct node.js version
|
||||
if [ -z ${NVM_DIR:+x} ]; then
|
||||
export NVM_DIR="/var/lib/jenkins/.nvm";
|
||||
fi
|
||||
|
||||
NVM_SCRIPT="$NVM_DIR/nvm.sh"
|
||||
if [ -s "$NVM_SCRIPT" ]; then
|
||||
. "$NVM_SCRIPT" # load nvm
|
||||
else
|
||||
echo "Unable to find the nvm script at \"$NVM_SCRIPT\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Installing node.js version $(cat ./kibana/.node-version)..."
|
||||
nvm install "$(cat ./kibana/.node-version)"
|
||||
fi
|
||||
|
||||
echo "Running ML tests..."
|
||||
echo "Running in $PWD"
|
||||
echo "Elasticsearch commit: $ES_COMMIT"
|
||||
echo "Machine Learning commit: $ML_COMMIT"
|
||||
|
||||
# output the commands
|
||||
set -xuf
|
||||
|
||||
# clean
|
||||
gradle --stacktrace clean
|
||||
|
||||
# Actually run the tests
|
||||
gradle "${GRADLE_CLI_ARGS[@]}"
|
||||
|
||||
# ~*~ shell-script-mode ~*~
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
org.gradle.daemon=false
|
||||
|
||||
elasticsearchVersion=6.0.0-alpha1-SNAPSHOT
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
File extrasDir = new File(settingsDir, '..').getCanonicalFile()
|
||||
if (extrasDir.name.endsWith('-extra') == false) {
|
||||
throw new GradleException("prelert-legacy must be checked out under an elasticsearch-extra directory, found ${extrasDir.name}")
|
||||
}
|
||||
File elasticsearchDir = new File(extrasDir.parentFile, extrasDir.name[0..-7])
|
||||
project(':').projectDir = elasticsearchDir
|
||||
apply from: "${elasticsearchDir}/settings.gradle"
|
||||
Loading…
Reference in New Issue