honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14460 from rjernst/distro_tests 

Add back integ tests to distributions
This commit is contained in:
Ryan Ernst 2015-11-03 10:35:12 -08:00
parent fe75c91de6 bd158e5e88
commit 64a01cfb05
8 changed files with 198 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -142,6 +142,7 @@ subprojects {
substitute module("org.elasticsearch:test-framework:${version}") with project("${projectsPrefix}:test-framework")
}
substitute module("org.elasticsearch.distribution.zip:elasticsearch:${version}") with project("${projectsPrefix}:distribution:zip")
substitute module("org.elasticsearch.distribution.tar:elasticsearch:${version}") with project("${projectsPrefix}:distribution:tar")
}
}
}

6
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View File

@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input
/** Configuration for an elasticsearch cluster, used for integration tests. */
class ClusterConfiguration {
@Input
String distribution = 'zip'
@Input
int numNodes = 1
@ -33,6 +36,9 @@ class ClusterConfiguration {
@Input
int transportPort = 9500
@Input
String jvmArgs = System.getProperty('tests.jvm.argline', '')
Map<String, String> systemProperties = new HashMap<>()
@Input

99
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.ElasticsearchProperties
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Copy
@ -42,7 +43,7 @@ class ClusterFormationTasks {
// no need to cluster formation if the task won't run!
return
}
addZipConfiguration(project)
configureDistributionDependency(project, config.distribution)
File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name)
if (config.numNodes == 1) {
addNodeStartupTasks(project, task, config, clusterDir)
@ -57,22 +58,38 @@ class ClusterFormationTasks {
}
static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) {
File pidFile = pidFile(baseDir)
String clusterName = "${task.path.replace(':', '_').substring(1)}"
File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}")
List setupDependsOn = [project.configurations.elasticsearchZip]
setupDependsOn.addAll(task.dependsOn)
Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) {
from { project.zipTree(project.configurations.elasticsearchZip.singleFile) }
into baseDir
File home = homeDir(baseDir, config.distribution)
Map esConfig = [
'cluster.name': clusterName,
'http.port': config.httpPort,
'transport.tcp.port': config.transportPort,
'pidfile': pidFile,
// TODO: make this work for multi node!
'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}",
'path.repo': "${home}/repo",
'path.shared_data': "${home}/../",
// Define a node attribute so we can test that it exists
'node.testattr': 'test',
'repositories.url.allowed_urls': 'http://snapshot.test*'
]
Map esEnv = [
'JAVA_HOME': System.getProperty('java.home'),
'ES_GC_OPTS': config.jvmArgs
]
List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep!
setupDeps.addAll(task.dependsOn)
Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) {
delete baseDir
}
setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup)
// chain setup tasks to maintain their order
setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) {
delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs')
}
setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << {
File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml')
File configFile = new File(home, 'config/elasticsearch.yml')
logger.info("Configuring ${configFile}")
configFile.setText("cluster.name: ${clusterName}", 'UTF-8')
configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
}
for (Map.Entry<String, String> command : config.setupCommands.entrySet()) {
Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) {
@ -100,15 +117,7 @@ class ClusterFormationTasks {
setup = nextSetup
}
File pidFile = pidFile(baseDir)
List esArgs = [
"-Des.http.port=${config.httpPort}",
"-Des.transport.tcp.port=${config.transportPort}",
"-Des.pidfile=${pidFile}",
"-Des.path.repo=${home}/repo",
"-Des.path.shared_data=${home}/../",
]
esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"})
List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"}
Closure esPostStartActions = { ant, logger ->
ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") {
and {
@ -125,12 +134,13 @@ class ClusterFormationTasks {
throw new GradleException('Failed to start elasticsearch')
}
}
Task start;
Task start
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// elasticsearch.bat is spawned as it has no daemon mode
start = project.tasks.create(name: "${task.name}#start", type: DefaultTask, dependsOn: setup) << {
// Fall back to Ant exec task as Gradle Exec task does not support spawning yet
ant.exec(executable: 'cmd', spawn: true, dir: home) {
esEnv.each { env(key: key, value: value) }
(['/C', 'call', 'bin/elasticsearch'] + esArgs).each { arg(value: it) }
}
esPostStartActions(ant, logger)
@ -141,6 +151,7 @@ class ClusterFormationTasks {
executable 'sh'
args 'bin/elasticsearch', '-d' // daemonize!
args esArgs
environment esEnv
errorOutput = new ByteArrayOutputStream()
doLast {
if (errorOutput.toString().isEmpty() == false) {
@ -157,9 +168,44 @@ class ClusterFormationTasks {
task.dependsOn(start)
}
static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) {
List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
Task extract
switch (distro) {
case 'zip':
extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
into baseDir
}
break;
case 'tar':
extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) }
into baseDir
}
break;
default:
throw new InvalidUserDataException("Unknown distribution: ${distro}")
}
return extract
}
static File homeDir(File baseDir, String distro) {
String path
switch (distro) {
case 'zip':
case 'tar':
path = "elasticsearch-${ElasticsearchProperties.version}"
break;
default:
throw new InvalidUserDataException("Unknown distribution: ${distro}")
}
return new File(baseDir, path)
}
static void addNodeStopTask(Project project, Task task, File baseDir) {
LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir))
Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) {
Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable 'Taskkill'
args '/PID', pidFile, '/F'
@ -187,13 +233,14 @@ class ClusterFormationTasks {
return new File(dir, 'es.pid')
}
static void addZipConfiguration(Project project) {
static void configureDistributionDependency(Project project, String distro) {
String elasticsearchVersion = ElasticsearchProperties.version
String packaging = distro == 'tar' ? 'tgz' : distro
project.configurations {
elasticsearchZip
elasticsearchDistro
}
project.dependencies {
elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip"
elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}"
}
}
}

40
distribution/build.gradle
View File

@ -36,11 +36,19 @@ buildscript {
allprojects {
project.ext {
// this is common configuration for distributions, but we also add it here for the license check to use
deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
}
}
subprojects {
/*****************************************************************************
* Rest test config *
*****************************************************************************/
apply plugin: 'elasticsearch.rest-test'
integTest {
includePackaged true
}
/*****************************************************************************
* Maven config *
*****************************************************************************/
@ -50,8 +58,8 @@ subprojects {
// we must create our own install task, because it is only added when the java plugin is added
task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
configuration = configurations.archives
MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention);
repositoriesHandler.mavenInstaller();
MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention)
repositoriesHandler.mavenInstaller()
}
// TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
@ -81,7 +89,7 @@ subprojects {
libFiles = copySpec {
into 'lib'
from project("${projectsPrefix}:core").jar
from deps
from dependencyFiles
}
configFiles = copySpec {
@ -151,7 +159,7 @@ buildDeb.dependsOn createEmptyDir
/*****************************************************************************
* Deb and rpm configuration *
*****************************************************************************/
configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
apply plugin: 'nebula.ospackage-base'
ospackage {
packageName = 'elasticsearch'
@ -173,29 +181,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
}
directory('/etc/elasticsearch/scripts')
}
if (project.name == 'deb') {
task buildDeb(type: Deb) {
dependsOn deps
}
artifacts {
archives buildDeb
}
} else if (project.name == 'rpm') {
task buildRpm(type: Rpm) {
dependsOn deps
}
artifacts {
archives buildRpm
}
}
// TODO: re-enable tests when we have real rpm and deb distros!
integTest.enabled = false
}
// TODO: dependency checks should really be when building the jar itself, which would remove the need
// for this hackery and instead we can do this inside the BuildPlugin
task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task!
DependencyLicensesTask.configure(project) {
dependsOn = [deps]
dependencies = deps
dependsOn = [dependencyFiles]
dependencies = dependencyFiles
mapping from: /lucene-.*/, to: 'lucene'
mapping from: /jackson-.*/, to: 'jackson'
}

30
distribution/deb/build.gradle
View File

@ -1,4 +1,28 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*task buildDeb(type: Deb) {
dependsOn deps
}*/
task buildDeb(type: Deb) {
dependsOn dependencyFiles
}
artifacts {
archives buildDeb
}
integTest.dependsOn buildDeb

30
distribution/rpm/build.gradle
View File

@ -1,4 +1,28 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*task buildRpm(type: Rpm) {
dependsOn deps
}*/
task buildRpm(type: Rpm) {
dependsOn dependencyFiles
}
artifacts {
archives buildRpm
}
integTest.dependsOn buildRpm

26
distribution/tar/build.gradle
View File

@ -1,10 +1,34 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
task buildTar(type: Tar, dependsOn: deps) {
task buildTar(type: Tar, dependsOn: dependencyFiles) {
baseName = 'elasticsearch'
with archivesFiles
compression = Compression.GZIP
}
artifacts {
'default' buildTar
archives buildTar
}
integTest {
dependsOn buildTar
clusterConfig.distribution = 'tar'
}

22
distribution/zip/build.gradle
View File

@ -1,5 +1,23 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
task buildZip(type: Zip, dependsOn: deps) {
task buildZip(type: Zip, dependsOn: dependencyFiles) {
baseName = 'elasticsearch'
with archivesFiles
}
@ -9,3 +27,5 @@ artifacts {
archives buildZip
}
integTest.dependsOn buildZip