honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert vagrant tests to per platform projects (#45064) 

The vagrant based tests currently reside in a single project, creating
dozens of tasks to manage starting and stopping the vagrant VM along
with running java and bats tests within each image. This all-in-one
pattern makes parallelizing packaging tests difficult.

This commit rewrites the vagrant testing infrastructure to be
independent of the actual test runners, thus allowing each platform to
be handled in a separate subproject. Additionally, the java and bats
tests are changed to be run through a "destructive" gradle task, which
is run inside the VM. The combination of these will allow
parallelization both locally (through running several VMs at once) as
well as running the destructive tasks in CI machines dedicated to each
platform (thus removing the need for vagrant in CI).
This commit is contained in:
Ryan Ernst 2019-08-12 16:01:53 -07:00 committed by GitHub
parent ae06a9399a
commit 97efb6a403
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
66 changed files with 1266 additions and 1223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Vagrantfile vendored
View File

@ -41,6 +41,16 @@ Vagrant.configure(2) do |config|
# the elasticsearch project called vagrant.... # the elasticsearch project called vagrant....
config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.synced_folder '.', '/vagrant', disabled: true
config.vm.synced_folder '.', '/elasticsearch' config.vm.synced_folder '.', '/elasticsearch'
# TODO: make these syncs work for windows!!!
config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/jars-3", "/root/.gradle/caches/jars-3",
create: true,
owner: "vagrant"
config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/modules-2", "/root/.gradle/caches/modules-2",
create: true,
owner: "vagrant"
config.vm.synced_folder "#{Dir.home}/.gradle/wrapper", "/root/.gradle/wrapper",
create: true,
owner: "vagrant"
# Expose project directory. Note that VAGRANT_CWD may not be the same as Dir.pwd # Expose project directory. Note that VAGRANT_CWD may not be the same as Dir.pwd
PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
@ -380,10 +390,6 @@ export ZIP=/elasticsearch/distribution/zip/build/distributions
export TAR=/elasticsearch/distribution/tar/build/distributions export TAR=/elasticsearch/distribution/tar/build/distributions
export RPM=/elasticsearch/distribution/rpm/build/distributions export RPM=/elasticsearch/distribution/rpm/build/distributions
export DEB=/elasticsearch/distribution/deb/build/distributions export DEB=/elasticsearch/distribution/deb/build/distributions
export BATS=/project/build/bats
export BATS_UTILS=/project/build/packaging/bats/utils
export BATS_TESTS=/project/build/packaging/bats/tests
export PACKAGING_ARCHIVES=/project/build/packaging/archives
export PACKAGING_TESTS=/project/build/packaging/tests export PACKAGING_TESTS=/project/build/packaging/tests
VARS VARS
cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
@ -391,11 +397,10 @@ Defaults env_keep += "ZIP"
Defaults env_keep += "TAR" Defaults env_keep += "TAR"
Defaults env_keep += "RPM" Defaults env_keep += "RPM"
Defaults env_keep += "DEB" Defaults env_keep += "DEB"
Defaults env_keep += "BATS"
Defaults env_keep += "BATS_UTILS"
Defaults env_keep += "BATS_TESTS"
Defaults env_keep += "PACKAGING_ARCHIVES" Defaults env_keep += "PACKAGING_ARCHIVES"
Defaults env_keep += "PACKAGING_TESTS" Defaults env_keep += "PACKAGING_TESTS"
Defaults env_keep += "BATS_UTILS"
Defaults env_keep += "BATS_TESTS"
Defaults env_keep += "JAVA_HOME" Defaults env_keep += "JAVA_HOME"
Defaults env_keep += "SYSTEM_JAVA_HOME" Defaults env_keep += "SYSTEM_JAVA_HOME"
SUDOERS_VARS SUDOERS_VARS

300
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java Normal file
View File

@ -0,0 +1,300 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test;
import org.elasticsearch.gradle.BuildPlugin;
import org.elasticsearch.gradle.BwcVersions;
import org.elasticsearch.gradle.DistributionDownloadPlugin;
import org.elasticsearch.gradle.ElasticsearchDistribution;
import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
import org.elasticsearch.gradle.Jdk;
import org.elasticsearch.gradle.JdkDownloadPlugin;
import org.elasticsearch.gradle.Version;
import org.elasticsearch.gradle.VersionProperties;
import org.elasticsearch.gradle.tool.Boilerplate;
import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
import org.elasticsearch.gradle.vagrant.VagrantExtension;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.Directory;
import org.gradle.api.plugins.ExtraPropertiesExtension;
import org.gradle.api.plugins.JavaBasePlugin;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.TaskInputs;
import org.gradle.api.tasks.TaskProvider;
import org.gradle.api.tasks.testing.Test;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
public class DistroTestPlugin implements Plugin<Project> {
private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691";
// all distributions used by distro tests. this is temporary until tests are per distribution
private static final String PACKAGING_DISTRIBUTION = "packaging";
private static final String COPY_PACKAGING_TASK = "copyPackagingArchives";
private static final String IN_VM_SYSPROP = "tests.inVM";
private static Version upgradeVersion;
private Provider<Directory> archivesDir;
private TaskProvider<Copy> copyPackagingArchives;
private Jdk gradleJdk;
@Override
public void apply(Project project) {
project.getPluginManager().apply(JdkDownloadPlugin.class);
project.getPluginManager().apply(DistributionDownloadPlugin.class);
project.getPluginManager().apply(VagrantBasePlugin.class);
project.getPluginManager().apply(JavaPlugin.class);
configureVM(project);
if (upgradeVersion == null) {
// just read this once, since it is the same for all projects. this is safe because gradle configuration is single threaded
upgradeVersion = getUpgradeVersion(project);
}
// setup task to run inside VM
configureDistributions(project);
configureCopyPackagingTask(project);
configureDistroTest(project);
configureBatsTest(project, "oss");
configureBatsTest(project, "default");
}
private static Jdk createJdk(NamedDomainObjectContainer<Jdk> jdksContainer, String name, String version, String platform) {
Jdk jdk = jdksContainer.create(name);
jdk.setVersion(version);
jdk.setPlatform(platform);
return jdk;
}
private static Version getUpgradeVersion(Project project) {
String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion");
if (upgradeFromVersionRaw != null) {
return Version.fromString(upgradeFromVersionRaw);
}
// was not passed in, so randomly choose one from bwc versions
ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class);
if ((boolean) extraProperties.get("bwc_tests_enabled") == false) {
// Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests
return Version.fromString(project.getVersion().toString());
}
ExtraPropertiesExtension rootExtraProperties = project.getRootProject().getExtensions().getByType(ExtraPropertiesExtension.class);
String firstPartOfSeed = rootExtraProperties.get("testSeed").toString().split(":")[0];
final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16);
BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
final List<Version> indexCompatVersions = bwcVersions.getIndexCompatible();
return indexCompatVersions.get(new Random(seed).nextInt(indexCompatVersions.size()));
}
private void configureVM(Project project) {
String box = project.getName();
// setup jdks used by the distro tests, and by gradle executing
NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
String platform = box.contains("windows") ? "windows" : "linux";
this.gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VERSION, platform);
// setup VM used by these tests
VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);
vagrant.setBox(box);
vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdk, "/bin:$PATH", "\\bin;$Env:PATH"));
vagrant.setIsWindowsVM(box.contains("windows"));
}
private static Object convertPath(Project project, VagrantExtension vagrant, Jdk jdk,
String additionaLinux, String additionalWindows) {
return new Object() {
@Override
public String toString() {
if (vagrant.isWindowsVM()) {
return convertWindowsPath(project, jdk.getPath()) + additionalWindows;
}
return convertLinuxPath(project, jdk.getPath()) + additionaLinux;
}
};
}
private void configureCopyPackagingTask(Project project) {
this.archivesDir = project.getParent().getLayout().getBuildDirectory().dir("packaging/archives");
// temporary, until we have tasks per distribution
this.copyPackagingArchives = Boilerplate.maybeRegister(project.getParent().getTasks(), COPY_PACKAGING_TASK, Copy.class,
t -> {
t.into(archivesDir);
t.from(project.getConfigurations().getByName(PACKAGING_DISTRIBUTION));
Path archivesPath = archivesDir.get().getAsFile().toPath();
// write bwc version, and append -SNAPSHOT if it is an unreleased version
ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class);
BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
final String upgradeFromVersion;
if (bwcVersions.unreleasedInfo(upgradeVersion) != null) {
upgradeFromVersion = upgradeVersion.toString() + "-SNAPSHOT";
} else {
upgradeFromVersion = upgradeVersion.toString();
}
TaskInputs inputs = t.getInputs();
inputs.property("version", VersionProperties.getElasticsearch());
inputs.property("upgrade_from_version", upgradeFromVersion);
// TODO: this is serializable, need to think how to represent this as an input
//inputs.property("bwc_versions", bwcVersions);
t.doLast(action -> {
try {
Files.writeString(archivesPath.resolve("version"), VersionProperties.getElasticsearch());
Files.writeString(archivesPath.resolve("upgrade_from_version"), upgradeFromVersion);
// this is always true, but bats tests rely on it. It is just temporary until bats is removed.
Files.writeString(archivesPath.resolve("upgrade_is_oss"), "");
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
});
}
private void configureDistroTest(Project project) {
BuildPlugin.configureCompile(project);
BuildPlugin.configureRepositories(project);
BuildPlugin.configureTestTasks(project);
BuildPlugin.configureInputNormalization(project);
TaskProvider<Test> destructiveTest = project.getTasks().register("destructiveDistroTest", Test.class,
t -> {
t.setMaxParallelForks(1);
t.setWorkingDir(archivesDir.get());
if (System.getProperty(IN_VM_SYSPROP) == null) {
t.dependsOn(copyPackagingArchives, gradleJdk);
}
});
// setup outer task to run
project.getTasks().register("distroTest", GradleDistroTestTask.class,
t -> {
t.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
t.setDescription("Runs distribution tests within vagrant");
t.setTaskName(project.getPath() + ":" + destructiveTest.getName());
t.extraArg("-D'" + IN_VM_SYSPROP + "'");
t.dependsOn(copyPackagingArchives, gradleJdk);
});
}
private void configureBatsTest(Project project, String type) {
// destructive task to run inside
TaskProvider<BatsTestTask> destructiveTest = project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class,
t -> {
// this is hacky for shared source, but bats are a temporary thing we are removing, so it is not worth
// the overhead of a real project dependency
Directory batsDir = project.getParent().getLayout().getProjectDirectory().dir("bats");
t.setTestsDir(batsDir.dir(type));
t.setUtilsDir(batsDir.dir("utils"));
t.setArchivesDir(archivesDir.get());
t.setPackageName("elasticsearch" + (type.equals("oss") ? "-oss" : ""));
if (System.getProperty(IN_VM_SYSPROP) == null) {
t.dependsOn(copyPackagingArchives, gradleJdk);
}
});
VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);
// setup outer task to run
project.getTasks().register("batsTest." + type, GradleDistroTestTask.class,
t -> {
t.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
t.setDescription("Runs bats tests within vagrant");
t.setTaskName(project.getPath() + ":" + destructiveTest.getName());
t.setProgressHandler(new BatsProgressLogger(project.getLogger()));
t.extraArg("-D'" + IN_VM_SYSPROP + "'");
t.dependsOn(copyPackagingArchives, gradleJdk);
t.onlyIf(spec -> vagrant.isWindowsVM() == false); // bats doesn't run on windows
});
}
private void configureDistributions(Project project) {
NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
for (Type type : Arrays.asList(Type.DEB, Type.RPM)) {
for (Flavor flavor : Flavor.values()) {
for (boolean bundledJdk : Arrays.asList(true, false)) {
addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch());
}
}
// upgrade version is always bundled jdk
// NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced
// witha dedicated upgrade test from every bwc version like other bwc tests
addDistro(distributions, type, null, Flavor.DEFAULT, true, upgradeVersion.toString());
if (upgradeVersion.onOrAfter("6.3.0")) {
addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString());
}
}
for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) {
for (Flavor flavor : Flavor.values()) {
for (boolean bundledJdk : Arrays.asList(true, false)) {
addDistro(distributions, Type.ARCHIVE, platform, flavor, bundledJdk, VersionProperties.getElasticsearch());
}
}
}
// temporary until distro tests have one test per distro
Configuration packagingConfig = project.getConfigurations().create(PACKAGING_DISTRIBUTION);
List<Configuration> distroConfigs = distributions.stream().map(ElasticsearchDistribution::getConfiguration)
.collect(Collectors.toList());
packagingConfig.setExtendsFrom(distroConfigs);
}
private static void addDistro(NamedDomainObjectContainer<ElasticsearchDistribution> distributions,
Type type, Platform platform, Flavor flavor, boolean bundledJdk, String version) {
String name = flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") + "-" + version;
if (distributions.findByName(name) != null) {
return;
}
distributions.create(name, d -> {
d.setFlavor(flavor);
d.setType(type);
if (type == Type.ARCHIVE) {
d.setPlatform(platform);
}
d.setBundledJdk(bundledJdk);
d.setVersion(version);
});
}
}

54
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy
View File

@ -1,54 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test
import org.elasticsearch.gradle.vagrant.VagrantCommandTask
import org.gradle.api.Task
/**
* A fixture for integration tests which runs in a virtual machine launched by Vagrant.
*/
class VagrantFixture extends VagrantCommandTask implements Fixture {
private VagrantCommandTask stopTask
public VagrantFixture() {
this.stopTask = project.tasks.create(name: "${name}#stop", type: VagrantCommandTask) {
command 'halt'
}
finalizedBy this.stopTask
}
@Override
void setBoxName(String boxName) {
super.setBoxName(boxName)
this.stopTask.setBoxName(boxName)
}
@Override
void setEnvironmentVars(Map<String, String> environmentVars) {
super.setEnvironmentVars(environmentVars)
this.stopTask.setEnvironmentVars(environmentVars)
}
@Override
public Task getStopTask() {
return this.stopTask
}
}

86
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
View File

@ -1,86 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant
import org.apache.commons.io.output.TeeOutputStream
import org.elasticsearch.gradle.LoggedExec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.internal.logging.progress.ProgressLoggerFactory
import javax.inject.Inject
/**
* Runs a vagrant command. Pretty much like Exec task but with a nicer output
* formatter and defaults to `vagrant` as first part of commandLine.
*/
public class VagrantCommandTask extends LoggedExec {
@Input
String command
@Input @Optional
String subcommand
@Input
String boxName
@Input
Map<String, String> environmentVars
public VagrantCommandTask() {
executable = 'vagrant'
// We're using afterEvaluate here to slot in some logic that captures configurations and
// modifies the command line right before the main execution happens. The reason that we
// call doFirst instead of just doing the work in the afterEvaluate is that the latter
// restricts how subclasses can extend functionality. Calling afterEvaluate is like having
// all the logic of a task happening at construction time, instead of at execution time
// where a subclass can override or extend the logic.
project.afterEvaluate {
doFirst {
if (environmentVars != null) {
environment environmentVars
}
// Build our command line for vagrant
def vagrantCommand = [executable, command]
if (subcommand != null) {
vagrantCommand = vagrantCommand + subcommand
}
commandLine([*vagrantCommand, boxName, *args])
// It'd be nice if --machine-readable were, well, nice
standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream())
}
}
}
@Inject
ProgressLoggerFactory getProgressLoggerFactory() {
throw new UnsupportedOperationException()
}
protected OutputStream createLoggerOutputStream() {
return new VagrantLoggerOutputStream(getProgressLoggerFactory().newOperation(boxName + " " + command).setDescription(boxName),
/* Vagrant tends to output a lot of stuff, but most of the important
stuff starts with ==> $box */
"==> $boxName: ")
}
}

67
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
View File

@ -1,67 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant
import org.elasticsearch.gradle.Version
import org.gradle.api.tasks.Input
class VagrantPropertiesExtension {
@Input
List<String> boxes
@Input
Version upgradeFromVersion
@Input
List<String> upgradeFromVersions
@Input
String batsDir
@Input
Boolean inheritTests
@Input
Boolean inheritTestUtils
@Input
String testClass
VagrantPropertiesExtension(List<String> availableBoxes) {
this.boxes = availableBoxes
this.batsDir = 'src/test/resources/packaging'
}
void boxes(String... boxes) {
this.boxes = Arrays.asList(boxes)
}
void setBatsDir(String batsDir) {
this.batsDir = batsDir
}
void setInheritTests(Boolean inheritTests) {
this.inheritTests = inheritTests
}
void setInheritTestUtils(Boolean inheritTestUtils) {
this.inheritTestUtils = inheritTestUtils
}
}

127
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy
View File

@ -1,127 +0,0 @@
package org.elasticsearch.gradle.vagrant
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.process.ExecResult
import org.gradle.process.internal.ExecException
/**
* Global configuration for if Vagrant tasks are supported in this
* build environment.
*/
class VagrantSupportPlugin implements Plugin<Project> {
@Override
void apply(Project project) {
if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
Map vagrantInstallation = getVagrantInstallation(project)
Map virtualBoxInstallation = getVirtualBoxInstallation(project)
project.rootProject.ext.vagrantInstallation = vagrantInstallation
project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
project.rootProject.ext.vagrantEnvChecksDone = true
// Finding that HOME needs to be set when performing vagrant updates
String homeLocation = System.getenv("HOME")
if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
"and requires \$HOME to be set to function properly.")
}
}
addVerifyInstallationTasks(project)
}
private Map getVagrantInstallation(Project project) {
try {
ByteArrayOutputStream pipe = new ByteArrayOutputStream()
ExecResult runResult = project.exec {
commandLine 'vagrant', '--version'
standardOutput pipe
ignoreExitValue true
}
String version = pipe.toString().trim()
if (runResult.exitValue == 0) {
if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) {
return [ 'supported' : true ]
} else {
return [ 'supported' : false,
'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
}
} else {
return [ 'supported' : false,
'info' : "Could not read installed vagrant version:\n" + version ]
}
} catch (ExecException e) {
// Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
// Swallow error. Vagrant isn't installed. Don't halt the build here.
return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
}
}
private Map getVirtualBoxInstallation(Project project) {
try {
ByteArrayOutputStream pipe = new ByteArrayOutputStream()
ExecResult runResult = project.exec {
commandLine 'vboxmanage', '--version'
standardOutput = pipe
ignoreExitValue true
}
String version = pipe.toString().trim()
if (runResult.exitValue == 0) {
try {
String[] versions = version.split('\\.')
int major = Integer.parseInt(versions[0])
int minor = Integer.parseInt(versions[1])
if ((major < 5) || (major == 5 && minor < 1)) {
return [ 'supported' : false,
'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
} else {
return [ 'supported' : true ]
}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
return [ 'supported' : false,
'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
}
} else {
return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
}
} catch (ExecException e) {
// Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
// Swallow error. VirtualBox isn't installed. Don't halt the build here.
return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
}
}
private void addVerifyInstallationTasks(Project project) {
createCheckVagrantVersionTask(project)
createCheckVirtualBoxVersionTask(project)
}
private void createCheckVagrantVersionTask(Project project) {
project.tasks.create('vagrantCheckVersion') {
description 'Check the Vagrant version'
group 'Verification'
doLast {
if (project.rootProject.vagrantInstallation.supported == false) {
throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
}
}
}
}
private void createCheckVirtualBoxVersionTask(Project project) {
project.tasks.create('virtualboxCheckVersion') {
description 'Check the Virtualbox version'
group 'Verification'
doLast {
if (project.rootProject.virtualBoxInstallation.supported == false) {
throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
}
}
}
}
}

658
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View File

@ -1,658 +0,0 @@
package org.elasticsearch.gradle.vagrant
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BwcVersions
import org.elasticsearch.gradle.FileContentsTask
import org.elasticsearch.gradle.Jdk
import org.elasticsearch.gradle.JdkDownloadPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.execution.TaskExecutionAdapter
import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
import org.gradle.api.tasks.Copy
import org.gradle.api.tasks.Delete
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.StopExecutionException
import org.gradle.api.tasks.TaskState
import java.nio.file.Paths
import static java.util.Collections.unmodifiableList
class VagrantTestPlugin implements Plugin<Project> {
/** All Linux boxes that we test. These are all always supplied **/
static final List<String> LINUX_BOXES = unmodifiableList([
'centos-6',
'centos-7',
'debian-8',
'debian-9',
'fedora-28',
'fedora-29',
'oel-6',
'oel-7',
'opensuse-42',
/* TODO: need a real RHEL license now that it is out of beta 'rhel-8',*/
'sles-12',
'ubuntu-1604',
'ubuntu-1804'
])
/** All Windows boxes that we test, which may or may not be supplied **/
static final List<String> WINDOWS_BOXES = unmodifiableList([
'windows-2012r2',
'windows-2016'
])
/** All boxes that we test, some of which may not be supplied **/
static final List<String> ALL_BOXES = unmodifiableList(LINUX_BOXES + WINDOWS_BOXES)
/** Boxes used when sampling the tests **/
static final List<String> SAMPLE = unmodifiableList([
'centos-7',
'ubuntu-1604'
])
/** All distributions to bring into test VM, whether or not they are used **/
static final List<String> DISTRIBUTIONS = unmodifiableList([
'archives:linux-tar',
'archives:oss-linux-tar',
'archives:windows-zip',
'archives:oss-windows-zip',
'packages:rpm',
'packages:oss-rpm',
'packages:deb',
'packages:oss-deb',
'archives:no-jdk-linux-tar',
'archives:oss-no-jdk-linux-tar',
'archives:no-jdk-windows-zip',
'archives:oss-no-jdk-windows-zip',
'packages:no-jdk-rpm',
'packages:oss-no-jdk-rpm',
'packages:no-jdk-deb',
'packages:oss-no-jdk-deb'
])
/** Packages onboarded for upgrade tests **/
static final List<String> UPGRADE_FROM_ARCHIVES = unmodifiableList(['rpm', 'deb'])
private static final PACKAGING_CONFIGURATION = 'packaging'
private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
private static final BATS = 'bats'
private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
/** Boxes that have been supplied and are available for testing **/
List<String> availableBoxes = []
/** extra env vars to pass to vagrant for box configuration **/
Map<String, String> vagrantBoxEnvVars = [:]
private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691"
private Jdk linuxGradleJdk;
private Jdk windowsGradleJdk;
@Override
void apply(Project project) {
project.pluginManager.apply(JdkDownloadPlugin.class)
NamedDomainObjectContainer<Jdk> jdksContainer = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
linuxGradleJdk = jdksContainer.create("linux_gradle") {
version = GRADLE_JDK_VERSION
platform = "linux"
}
windowsGradleJdk = jdksContainer.create("windows_gradle") {
version = GRADLE_JDK_VERSION
platform = "windows"
}
collectAvailableBoxes(project)
// Creates the Vagrant extension for the project
project.extensions.create('esvagrant', VagrantPropertiesExtension, listSelectedBoxes(project))
// Add required repositories for packaging tests
configurePackagingArchiveRepositories(project)
// Creates custom configurations for Bats testing files (and associated scripts and archives)
createPackagingConfiguration(project)
project.configurations.create(PACKAGING_TEST_CONFIGURATION)
// Creates all the main Vagrant tasks
createVagrantTasks(project)
if (project.extensions.esvagrant.boxes == null || project.extensions.esvagrant.boxes.size() == 0) {
throw new InvalidUserDataException('Must specify at least one vagrant box')
}
for (String box : project.extensions.esvagrant.boxes) {
if (ALL_BOXES.contains(box) == false) {
throw new InvalidUserDataException("Vagrant box [${box}] is unknown to this plugin. Valid boxes are ${ALL_BOXES}")
}
if (availableBoxes.contains(box) == false) {
throw new InvalidUserDataException("Vagrant box [${box}] is not available because an image is not supplied for it. " +
"Available boxes with supplied images are ${availableBoxes}")
}
}
// Creates all tasks related to the Vagrant boxes
createVagrantBoxesTasks(project)
}
/**
* Enumerate all the boxes that we know about and could possibly choose to test
*/
private void collectAvailableBoxes(Project project) {
// these images are hardcoded in the Vagrantfile and are always available
availableBoxes.addAll(LINUX_BOXES)
// these images need to be provided at runtime
String windows_2012r2_box = project.getProperties().get('vagrant.windows-2012r2.id')
if (windows_2012r2_box != null && windows_2012r2_box.isEmpty() == false) {
availableBoxes.add('windows-2012r2')
vagrantBoxEnvVars['VAGRANT_WINDOWS_2012R2_BOX'] = windows_2012r2_box
}
String windows_2016_box = project.getProperties().get('vagrant.windows-2016.id')
if (windows_2016_box != null && windows_2016_box.isEmpty() == false) {
availableBoxes.add('windows-2016')
vagrantBoxEnvVars['VAGRANT_WINDOWS_2016_BOX'] = windows_2016_box
}
}
/**
* Enumerate all the boxes that we have chosen to test
*/
private static List<String> listSelectedBoxes(Project project) {
String vagrantBoxes = project.getProperties().get('vagrant.boxes', 'sample')
switch (vagrantBoxes) {
case 'sample':
return SAMPLE
case 'linux-all':
return LINUX_BOXES
case 'windows-all':
return WINDOWS_BOXES
case 'all':
return ALL_BOXES
case '':
return []
default:
return vagrantBoxes.split(',')
}
}
private static void configurePackagingArchiveRepositories(Project project) {
RepositoryHandler repos = project.repositories
repos.jcenter() // will have releases before 5.0.0
/* Setup a repository that tries to download from
https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]
which should work for 5.0.0+. This isn't a real ivy repository but gradle
is fine with that */
repos.ivy {
name "elasticsearch"
artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]"
}
}
private static void createPackagingConfiguration(Project project) {
project.configurations.create(PACKAGING_CONFIGURATION)
String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion");
Version upgradeFromVersion
if (upgradeFromVersionRaw == null) {
String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16)
final def indexCompatVersions = project.bwcVersions.indexCompatible
upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())]
} else {
upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
}
List<Object> dependencies = new ArrayList<>()
DISTRIBUTIONS.each {
// Adds a dependency for the current version
dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
}
if (project.ext.bwc_tests_enabled) {
// The version of elasticsearch that we upgrade *from*
// we only add them as dependencies if the bwc tests are enabled, so we don't trigger builds otherwise
BwcVersions.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
if (unreleasedInfo != null) {
// handle snapshots pointing to bwc build
UPGRADE_FROM_ARCHIVES.each {
dependencies.add(project.dependencies.project(
path: "${unreleasedInfo.gradleProjectPath}", configuration: it))
if (upgradeFromVersion.onOrAfter('6.3.0')) {
dependencies.add(project.dependencies.project(
path: "${unreleasedInfo.gradleProjectPath}", configuration: "oss-${it}"))
}
}
} else {
UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from*
if (upgradeFromVersion.onOrAfter('7.0.0')) {
String arch = it == "rpm" ? "x86_64" : "amd64"
dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}-${arch}@${it}")
dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}-${arch}@${it}")
} else {
dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
if (upgradeFromVersion.onOrAfter('6.3.0')) {
dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
}
}
}
}
} else {
// Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests.
upgradeFromVersion = Version.fromString(project.version)
}
for (Object dependency : dependencies) {
project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
}
project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
}
private static void createCleanTask(Project project) {
if (project.tasks.findByName('clean') == null) {
project.tasks.create('clean', Delete.class) {
description 'Clean the project build directory'
group 'Build'
delete project.buildDir
}
}
}
private static void createStopTask(Project project) {
project.tasks.create('stop') {
description 'Stop any tasks from tests that still may be running'
group 'Verification'
}
}
private static void createSmokeTestTask(Project project) {
project.tasks.create('vagrantSmokeTest') {
description 'Smoke test the specified vagrant boxes'
group 'Verification'
}
}
private void createPrepareVagrantTestEnvTask(Project project) {
File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION)
File archivesDir = new File(packagingDir, 'archives')
Copy copyPackagingArchives = project.tasks.create('copyPackagingArchives', Copy) {
into archivesDir
from project.configurations[PACKAGING_CONFIGURATION]
}
File testsDir = new File(packagingDir, 'tests')
Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) {
into testsDir
from project.configurations[PACKAGING_TEST_CONFIGURATION]
}
Task createLinuxRunnerScript = project.tasks.create('createLinuxRunnerScript', FileContentsTask) {
dependsOn copyPackagingTests, linuxGradleJdk
file "${testsDir}/run-tests.sh"
contents """\
if [ "\$#" -eq 0 ]; then
test_args=( "${-> project.extensions.esvagrant.testClass}" )
else
test_args=( "\$@" )
fi
"${-> convertLinuxPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
"""
}
Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) {
dependsOn copyPackagingTests, windowsGradleJdk
file "${testsDir}/run-tests.ps1"
// the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely
// a little trappy for those unfamiliar with powershell
contents """\
try {
if (\$args.Count -eq 0) {
\$testArgs = @("${-> project.extensions.esvagrant.testClass}")
} else {
\$testArgs = \$args
}
& "${-> convertWindowsPath(project, windowsGradleJdk.toString()) }/bin/java" -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
exit \$LASTEXITCODE
} catch {
# catch if we have a failure to even run the script at all above, equivalent to set -e, sort of
echo "\$_.Exception.Message"
exit 1
}
"""
}
Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
dependsOn copyPackagingArchives
file "${archivesDir}/version"
contents project.version
}
Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
String version = project.extensions.esvagrant.upgradeFromVersion
if (project.bwcVersions.unreleased.contains(project.extensions.esvagrant.upgradeFromVersion)) {
version += "-SNAPSHOT"
}
dependsOn copyPackagingArchives
file "${archivesDir}/upgrade_from_version"
contents version
}
Task createUpgradeIsOssFile = project.tasks.create('createUpgradeIsOssFile', FileContentsTask) {
dependsOn copyPackagingArchives
doFirst {
project.delete("${archivesDir}/upgrade_is_oss")
if (project.extensions.esvagrant.upgradeFromVersion.before('6.3.0')) {
throw new StopExecutionException("upgrade version is before 6.3.0")
}
}
file "${archivesDir}/upgrade_is_oss"
contents ''
}
File batsDir = new File(packagingDir, BATS)
Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) {
into "${batsDir}/tests"
from {
"${project.extensions.esvagrant.batsDir}/tests"
}
}
Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) {
into "${batsDir}/utils"
from {
"${project.extensions.esvagrant.batsDir}/utils"
}
}
// Now we iterate over dependencies of the bats configuration. When a project dependency is found,
// we bring back its test files or test utils.
project.afterEvaluate {
project.configurations[PACKAGING_CONFIGURATION].dependencies
.findAll {it.targetConfiguration == PACKAGING_CONFIGURATION }
.each { d ->
if (d instanceof DefaultProjectDependency) {
DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
Project externalBatsProject = externalBatsDependency.dependencyProject
String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir
if (project.extensions.esvagrant.inheritTests) {
copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
}
if (project.extensions.esvagrant.inheritTestUtils) {
copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
}
}
}
}
Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
vagrantSetUpTask.dependsOn(
'vagrantCheckVersion',
copyPackagingArchives,
copyPackagingTests,
createLinuxRunnerScript,
createWindowsRunnerScript,
createVersionFile,
createUpgradeFromFile,
createUpgradeIsOssFile,
copyBatsTests,
copyBatsUtils
)
}
private static void createPackagingTestTask(Project project) {
project.tasks.create('packagingTest') {
group 'Verification'
description "Tests distribution installation on different platforms using vagrant. See TESTING.asciidoc for details."
dependsOn 'vagrantCheckVersion'
}
}
private void createBoxListTasks(Project project) {
project.tasks.create('listAllBoxes') {
group 'Verification'
description 'List all vagrant boxes which can be tested by this plugin'
doLast {
println("All vagrant boxes supported by ${project.path}")
for (String box : ALL_BOXES) {
println(box)
}
}
dependsOn 'vagrantCheckVersion'
}
project.tasks.create('listAvailableBoxes') {
group 'Verification'
description 'List all vagrant boxes which are available for testing'
doLast {
println("All vagrant boxes available to ${project.path}")
for (String box : availableBoxes) {
println(box)
}
}
dependsOn 'vagrantCheckVersion'
}
}
private void createVagrantTasks(Project project) {
createCleanTask(project)
createStopTask(project)
createSmokeTestTask(project)
createPrepareVagrantTestEnvTask(project)
createPackagingTestTask(project)
createBoxListTasks(project)
}
private void createVagrantBoxesTasks(Project project) {
assert project.extensions.esvagrant.boxes != null
assert project.tasks.stop != null
Task stop = project.tasks.stop
assert project.tasks.vagrantSmokeTest != null
Task vagrantSmokeTest = project.tasks.vagrantSmokeTest
assert project.tasks.vagrantCheckVersion != null
Task vagrantCheckVersion = project.tasks.vagrantCheckVersion
assert project.tasks.virtualboxCheckVersion != null
Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion
assert project.tasks.setupPackagingTest != null
Task setupPackagingTest = project.tasks.setupPackagingTest
assert project.tasks.packagingTest != null
Task packagingTest = project.tasks.packagingTest
/*
* We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
* so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
*/
def vagrantEnvVars = [
'VAGRANT_CWD' : "${project.rootDir.absolutePath}",
'VAGRANT_VAGRANTFILE' : 'Vagrantfile',
'VAGRANT_PROJECT_DIR' : "${project.projectDir.absolutePath}"
]
vagrantEnvVars.putAll(vagrantBoxEnvVars)
// Each box gets it own set of tasks
for (String box : availableBoxes) {
String boxTask = box.capitalize().replace('-', '')
// always add a halt task for all boxes, so clean makes sure they are all shutdown
Task halt = project.tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
command 'halt'
boxName box
environmentVars vagrantEnvVars
}
stop.dependsOn(halt)
Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
command 'box'
subcommand 'update'
boxName box
environmentVars vagrantEnvVars
dependsOn vagrantCheckVersion, virtualboxCheckVersion
}
update.mustRunAfter(setupPackagingTest)
/*
* Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag
* vagrant.destroy that defaults to true that can be used to control whether or not to destroy any test boxes before test
* execution.
*/
final String vagrantDestroyProperty = project.getProperties().get('vagrant.destroy', 'true')
boolean vagrantDestroy
if ("true".equals(vagrantDestroyProperty)) {
vagrantDestroy = true
} else if ("false".equals(vagrantDestroyProperty)) {
vagrantDestroy = false
} else {
throw new GradleException("[vagrant.destroy] must be [true] or [false] but was [" + vagrantDestroyProperty + "]")
}
/*
* Some versions of Vagrant will fail destroy if the box does not exist. Therefore we check if the box exists before attempting
* to destroy the box.
*/
final Task destroy = project.tasks.create("vagrant${boxTask}#destroy", LoggedExec) {
commandLine "bash", "-c", "vagrant status ${box} | grep -q \"${box}\\s\\+not created\" || vagrant destroy ${box} --force"
workingDir project.rootProject.rootDir
environment vagrantEnvVars
}
destroy.onlyIf { vagrantDestroy }
update.mustRunAfter(destroy)
Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
command 'up'
boxName box
environmentVars vagrantEnvVars
/* We lock the provider to virtualbox because the Vagrantfile specifies
lots of boxes that only work properly in virtualbox. Virtualbox is
vagrant's default but its possible to change that default and folks do.
But the boxes that we use are unlikely to work properly with other
virtualization providers. Thus the lock. */
args '--provision', '--provider', 'virtualbox'
/* It'd be possible to check if the box is already up here and output
SKIPPED but that would require running vagrant status which is slow! */
dependsOn destroy, update
}
Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) {
environment vagrantEnvVars
dependsOn up
finalizedBy halt
}
vagrantSmokeTest.dependsOn(smoke)
if (LINUX_BOXES.contains(box)) {
smoke.commandLine = ['vagrant', 'ssh', box, '--command',
"set -o pipefail && echo 'Hello from ${project.path}' | sed -ue 's/^/ ${box}: /'"]
} else {
smoke.commandLine = ['vagrant', 'winrm', box, '--command',
"Write-Host ' ${box}: Hello from ${project.path}'"]
}
if (LINUX_BOXES.contains(box)) {
Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
remoteCommand BATS_TEST_COMMAND
boxName box
environmentVars vagrantEnvVars
dependsOn up, setupPackagingTest
finalizedBy halt
}
TaskExecutionAdapter batsPackagingReproListener = createReproListener(project, batsPackagingTest.path)
batsPackagingTest.doFirst {
project.gradle.addListener(batsPackagingReproListener)
}
batsPackagingTest.doLast {
project.gradle.removeListener(batsPackagingReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {
// these tests are temporarily disabled for suse boxes while we debug an issue
// https://github.com/elastic/elasticsearch/issues/30295
if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
packagingTest.dependsOn(batsPackagingTest)
}
}
}
Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) {
boxName box
environmentVars vagrantEnvVars
dependsOn up, setupPackagingTest
finalizedBy halt
}
// todo remove this onlyIf after all packaging tests are consolidated
javaPackagingTest.onlyIf {
project.extensions.esvagrant.testClass != null
}
if (LINUX_BOXES.contains(box)) {
javaPackagingTest.command = 'ssh'
javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
} else {
// powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
// remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
// without administrator elevation), which we need to do for these tests. passing --elevated runs the session
// as a scheduled job locally on the vm as a true administrator to get around this limitation
//
// https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
// https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
javaPackagingTest.command = 'winrm'
javaPackagingTest.args = ['--elevated', '--command', '& "$Env:PACKAGING_TESTS/run-tests.ps1"; exit $LASTEXITCODE']
}
TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
javaPackagingTest.doFirst {
project.gradle.addListener(javaPackagingReproListener)
}
javaPackagingTest.doLast {
project.gradle.removeListener(javaPackagingReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {
// these tests are temporarily disabled for suse boxes while we debug an issue
// https://github.com/elastic/elasticsearch/issues/30295
if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
packagingTest.dependsOn(javaPackagingTest)
}
}
}
}
private static TaskExecutionAdapter createReproListener(Project project, String reproTaskPath) {
return new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew"
if (state.failure != null) {
println "REPRODUCE WITH: ${gradlew} \"${reproTaskPath}\" -Dtests.seed=${project.testSeed} "
}
}
}
}
// convert the given path from an elasticsearch repo path to a VM path
private String convertLinuxPath(Project project, String path) {
return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path));
}
private String convertWindowsPath(Project project, String path) {
return "C:\\elasticsearch\\" + project.rootDir.toPath().relativize(Paths.get(path)).toString().replace('/', '\\');
}
}

8
buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
View File

@ -203,12 +203,16 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
String extension = distribution.getType().toString(); String extension = distribution.getType().toString();
String classifier = "x86_64"; String classifier = "x86_64";
if (distribution.getType() == Type.ARCHIVE) { if (distribution.getVersion().before("7.0.0")) {
classifier = null; // no platform specific distros before 7.0
} else if (distribution.getType() == Type.ARCHIVE) {
extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
classifier = distribution.getPlatform() + "-" + classifier; classifier = distribution.getPlatform() + "-" + classifier;
} else if (distribution.getType() == Type.DEB) {
classifier = "amd64";
} }
return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":") return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
+ distribution.getVersion() + ":" + classifier + "@" + extension; + distribution.getVersion() + (classifier == null ? "" : ":" + classifier) + "@" + extension;
} }
private static Dependency projectDependency(Project project, String projectPath, String projectConfig) { private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {

19
buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
View File

@ -20,9 +20,7 @@
package org.elasticsearch.gradle; package org.elasticsearch.gradle;
import org.gradle.api.Buildable; import org.gradle.api.Buildable;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileTree;
import org.gradle.api.model.ObjectFactory; import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.Property; import org.gradle.api.provider.Property;
import org.gradle.api.tasks.TaskDependency; import org.gradle.api.tasks.TaskDependency;
@ -30,9 +28,8 @@ import org.gradle.api.tasks.TaskDependency;
import java.io.File; import java.io.File;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.Callable;
public class ElasticsearchDistribution implements Buildable { public class ElasticsearchDistribution implements Buildable, Iterable<File> {
public enum Platform { public enum Platform {
LINUX, LINUX,
@ -93,10 +90,6 @@ public class ElasticsearchDistribution implements Buildable {
return configuration.getBuildDependencies(); return configuration.getBuildDependencies();
} }
public FileTree getFileTree(Project project) {
return project.fileTree((Callable<File>) configuration::getSingleFile);
}
@Override @Override
public String toString() { public String toString() {
return configuration.getSingleFile().toString(); return configuration.getSingleFile().toString();
@ -190,6 +183,16 @@ public class ElasticsearchDistribution implements Buildable {
return configuration.getBuildDependencies(); return configuration.getBuildDependencies();
} }
@Override
public Iterator<File> iterator() {
return configuration.iterator();
}
// TODO: remove this when distro tests are per distribution
public Configuration getConfiguration() {
return configuration;
}
// internal, make this distribution's configuration unmodifiable // internal, make this distribution's configuration unmodifiable
void finalizeValues() { void finalizeValues() {

6
buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
View File

@ -83,9 +83,13 @@ public class Jdk implements Buildable, Iterable<File> {
return configuration; return configuration;
} }
public String getPath() {
return configuration.getSingleFile().toString();
}
@Override @Override
public String toString() { public String toString() {
return configuration.getSingleFile().toString(); return getPath();
} }
@Override @Override

8
buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
View File

@ -48,13 +48,14 @@ import java.util.regex.Matcher;
public class JdkDownloadPlugin implements Plugin<Project> { public class JdkDownloadPlugin implements Plugin<Project> {
private static final String REPO_NAME_PREFIX = "jdk_repo_"; private static final String REPO_NAME_PREFIX = "jdk_repo_";
private static final String CONTAINER_NAME = "jdks";
@Override @Override
public void apply(Project project) { public void apply(Project project) {
NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name -> NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name ->
new Jdk(name, project) new Jdk(name, project)
); );
project.getExtensions().add("jdks", jdksContainer); project.getExtensions().add(CONTAINER_NAME, jdksContainer);
project.afterEvaluate(p -> { project.afterEvaluate(p -> {
for (Jdk jdk : jdksContainer) { for (Jdk jdk : jdksContainer) {
@ -82,6 +83,11 @@ public class JdkDownloadPlugin implements Plugin<Project> {
}); });
} }
@SuppressWarnings("unchecked")
public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
return (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName(CONTAINER_NAME);
}
private static void setupRootJdkDownload(Project rootProject, String platform, String version) { private static void setupRootJdkDownload(Project rootProject, String platform, String version) {
String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version; String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version;
// NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task

34
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy → buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
View File

@ -16,30 +16,24 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.gradle.vagrant
import org.gradle.api.tasks.Input package org.elasticsearch.gradle;
/** import org.gradle.api.GradleException;
* Runs bats over vagrant. Pretty much like running it using Exec but with a
* nicer output formatter.
*/
public class BatsOverVagrantTask extends VagrantCommandTask {
@Input public class Util {
Object remoteCommand
BatsOverVagrantTask() { public static boolean getBooleanProperty(String property, boolean defaultValue) {
command = 'ssh' String propertyValue = System.getProperty(property);
if (propertyValue == null) {
return defaultValue;
}
if ("true".equals(propertyValue)) {
return true;
} else if ("false".equals(propertyValue)) {
return false;
} else {
throw new GradleException("Sysprop [" + property + "] must be [true] or [false] but was [" + propertyValue + "]");
} }
void setRemoteCommand(Object remoteCommand) {
this.remoteCommand = Objects.requireNonNull(remoteCommand)
setArgs((Iterable<?>) ['--command', remoteCommand])
}
@Override
protected OutputStream createLoggerOutputStream() {
return new TapLoggerOutputStream(logger, getProgressLoggerFactory().newOperation(boxName).setDescription(boxName));
} }
} }

92
buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.Directory;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.TaskAction;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class BatsTestTask extends DefaultTask {
private Directory testsDir;
private Directory utilsDir;
private Directory archivesDir;
private String packageName;
@InputDirectory
public Directory getTestsDir() {
return testsDir;
}
public void setTestsDir(Directory testsDir) {
this.testsDir = testsDir;
}
@InputDirectory
public Directory getUtilsDir() {
return utilsDir;
}
public void setUtilsDir(Directory utilsDir) {
this.utilsDir = utilsDir;
}
@InputDirectory
public Directory getArchivesDir() {
return archivesDir;
}
public void setArchivesDir(Directory archivesDir) {
this.archivesDir = archivesDir;
}
@Input
public String getPackageName() {
return packageName;
}
public void setPackageName(String packageName) {
this.packageName = packageName;
}
@TaskAction
public void runBats() {
List<Object> command = new ArrayList<>();
command.add("bats");
command.add("--tap");
command.addAll(testsDir.getAsFileTree().getFiles().stream()
.filter(f -> f.getName().endsWith(".bats"))
.sorted().collect(Collectors.toList()));
getProject().exec(spec -> {
spec.setWorkingDir(archivesDir.getAsFile());
spec.environment(System.getenv());
spec.environment("BATS_TESTS", testsDir.getAsFile().toString());
spec.environment("BATS_UTILS", utilsDir.getAsFile().toString());
spec.environment("PACKAGE_NAME", packageName);
spec.setCommandLine(command);
});
}
}

91
buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test;
import org.elasticsearch.gradle.vagrant.VagrantShellTask;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.options.Option;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
/**
* Run a gradle task of the current build, within the configured vagrant VM.
*/
public class GradleDistroTestTask extends VagrantShellTask {
private String taskName;
private String testClass;
private List<String> extraArgs = new ArrayList<>();
public void setTaskName(String taskName) {
this.taskName = taskName;
}
@Input
public String getTaskName() {
return taskName;
}
@Option(option = "tests", description = "Sets test class or method name to be included, '*' is supported.")
public void setTestClass(String testClass) {
this.testClass = testClass;
}
@Input
public List<String> getExtraArgs() {
return extraArgs;
}
public void extraArg(String arg) {
this.extraArgs.add(arg);
}
@Override
protected List<String> getWindowsScript() {
return getScript(true);
}
@Override
protected List<String> getLinuxScript() {
return getScript(false);
}
private List<String> getScript(boolean isWindows) {
String cacheDir = getProject().getBuildDir() + "/gradle-cache";
StringBuilder line = new StringBuilder();
line.append(isWindows ? "& .\\gradlew " : "./gradlew ");
line.append(taskName);
line.append(" --project-cache-dir ");
line.append(isWindows ? convertWindowsPath(getProject(), cacheDir) : convertLinuxPath(getProject(), cacheDir));
line.append(" -S");
line.append(" -D'org.gradle.logging.level'=" + getProject().getGradle().getStartParameter().getLogLevel());
if (testClass != null) {
line.append(" --tests=");
line.append(testClass);
}
extraArgs.stream().map(s -> " " + s).forEach(line::append);
return Collections.singletonList(line.toString());
}
}

8
buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
View File

@ -52,6 +52,14 @@ public abstract class Boilerplate {
} }
public static <T extends Task> TaskProvider<T> maybeRegister(TaskContainer tasks, String name, Class<T> clazz, Action<T> action) {
try {
return tasks.named(name, clazz);
} catch (UnknownTaskException e) {
return tasks.register(name, clazz, action);
}
}
public static void maybeConfigure(TaskContainer tasks, String name, Action<? super Task> config) { public static void maybeConfigure(TaskContainer tasks, String name, Action<? super Task> config) {
TaskProvider<?> task; TaskProvider<?> task;
try { try {

42
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java → buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java
View File

@ -19,12 +19,10 @@
package org.elasticsearch.gradle.vagrant; package org.elasticsearch.gradle.vagrant;
import org.elasticsearch.gradle.LoggingOutputStream;
import org.gradle.api.GradleScriptException;
import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logger;
import org.gradle.internal.logging.progress.ProgressLogger;
import java.util.Formatter; import java.util.Formatter;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -41,49 +39,43 @@ import java.util.regex.Pattern;
* There is a Tap4j project but we can't use it because it wants to parse the * There is a Tap4j project but we can't use it because it wants to parse the
* entire TAP stream at once and won't parse it stream-wise. * entire TAP stream at once and won't parse it stream-wise.
*/ */
public class TapLoggerOutputStream extends LoggingOutputStream { public class BatsProgressLogger implements UnaryOperator<String> {
private static final Pattern lineRegex = private static final Pattern lineRegex =
Pattern.compile("(?<status>ok|not ok) \\d+(?<skip> # skip (?<skipReason>\\(.+\\))?)? \\[(?<suite>.+)\\] (?<test>.+)"); Pattern.compile("(?<status>ok|not ok) \\d+(?<skip> # skip (?<skipReason>\\(.+\\))?)? \\[(?<suite>.+)\\] (?<test>.+)");
private static final Pattern startRegex = Pattern.compile("1..(\\d+)");
private final Logger logger; private final Logger logger;
private final ProgressLogger progressLogger;
private boolean isStarted = false;
private int testsCompleted = 0; private int testsCompleted = 0;
private int testsFailed = 0; private int testsFailed = 0;
private int testsSkipped = 0; private int testsSkipped = 0;
private Integer testCount; private Integer testCount;
private String countsFormat; private String countsFormat;
TapLoggerOutputStream(Logger logger, ProgressLogger progressLogger) { public BatsProgressLogger(Logger logger) {
this.logger = logger; this.logger = logger;
this.progressLogger = progressLogger;
} }
@Override @Override
public void logLine(String line) { public String apply(String line) {
if (isStarted == false) {
progressLogger.started("started");
isStarted = true;
}
if (testCount == null) { if (testCount == null) {
try { Matcher m = startRegex.matcher(line);
int lastDot = line.lastIndexOf('.'); if (m.matches() == false) {
testCount = Integer.parseInt(line.substring(lastDot + 1)); // haven't reached start of bats test yet, pass through whatever we see
return line;
}
testCount = Integer.parseInt(m.group(1));
int length = String.valueOf(testCount).length(); int length = String.valueOf(testCount).length();
String count = "%0" + length + "d"; String count = "%0" + length + "d";
countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]";
return; return null;
} catch (Exception e) {
throw new GradleScriptException("Error parsing first line of TAP stream!!", e);
}
} }
Matcher m = lineRegex.matcher(line); Matcher m = lineRegex.matcher(line);
if (m.matches() == false) { if (m.matches() == false) {
/* These might be failure report lines or comments or whatever. Its hard /* These might be failure report lines or comments or whatever. Its hard
to tell and it doesn't matter. */ to tell and it doesn't matter. */
logger.warn(line); logger.warn(line);
return; return null;
} }
boolean skipped = m.group("skip") != null; boolean skipped = m.group("skip") != null;
boolean success = skipped == false && m.group("status").equals("ok"); boolean success = skipped == false && m.group("status").equals("ok");
@ -104,15 +96,9 @@ public class TapLoggerOutputStream extends LoggingOutputStream {
} }
String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString(); String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString();
progressLogger.progress("BATS " + counts + ", " + status + " [" + suiteName + "] " + testName);
if (success == false) { if (success == false) {
logger.warn(line); logger.warn(line);
} }
} return "BATS " + counts + ", " + status + " [" + suiteName + "] " + testName;
@Override
public void close() {
flush();
progressLogger.completed();
} }
} }

147
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java Normal file
View File

@ -0,0 +1,147 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant;
import org.elasticsearch.gradle.ReaperPlugin;
import org.elasticsearch.gradle.ReaperService;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.execution.TaskActionListener;
import org.gradle.api.execution.TaskExecutionListener;
import org.gradle.api.tasks.TaskState;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class VagrantBasePlugin implements Plugin<Project> {
@Override
public void apply(Project project) {
project.getRootProject().getPluginManager().apply(VagrantSetupCheckerPlugin.class);
project.getRootProject().getPluginManager().apply(VagrantManagerPlugin.class);
project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class);
VagrantExtension extension = project.getExtensions().create("vagrant", VagrantExtension.class, project);
VagrantMachine service = project.getExtensions().create("vagrantService", VagrantMachine.class, project, extension, reaper);
project.getGradle().getTaskGraph().whenReady(graph ->
service.refs = graph.getAllTasks().stream()
.filter(t -> t instanceof VagrantShellTask)
.filter(t -> t.getProject() == project)
.count());
}
/**
* Check vagrant and virtualbox versions, if any vagrant test tasks will be run.
*/
static class VagrantSetupCheckerPlugin implements Plugin<Project> {
private static final Pattern VAGRANT_VERSION = Pattern.compile("Vagrant (\\d+\\.\\d+\\.\\d+)");
private static final Pattern VIRTUAL_BOX_VERSION = Pattern.compile("(\\d+\\.\\d+)");
@Override
public void apply(Project project) {
if (project != project.getRootProject()) {
throw new IllegalArgumentException("VagrantSetupCheckerPlugin can only be applied to the root project of a build");
}
project.getGradle().getTaskGraph().whenReady(graph -> {
boolean needsVagrant = graph.getAllTasks().stream().anyMatch(t -> t instanceof VagrantShellTask);
if (needsVagrant) {
checkVersion(project, "vagrant", VAGRANT_VERSION, 1, 8, 6);
checkVersion(project, "vboxmanage", VIRTUAL_BOX_VERSION, 5, 1);
}
});
}
void checkVersion(Project project, String tool, Pattern versionRegex, int... minVersion) {
ByteArrayOutputStream pipe = new ByteArrayOutputStream();
project.exec(spec -> {
spec.setCommandLine(tool, "--version");
spec.setStandardOutput(pipe);
});
String output = pipe.toString(StandardCharsets.UTF_8).trim();
Matcher matcher = versionRegex.matcher(output);
if (matcher.find() == false) {
throw new IllegalStateException(tool +
" version output [" + output + "] did not match regex [" + versionRegex.pattern() + "]");
}
String version = matcher.group(1);
List<Integer> versionParts = Stream.of(version.split("\\.")).map(Integer::parseInt).collect(Collectors.toList());
for (int i = 0; i < minVersion.length; ++i) {
int found = versionParts.get(i);
if (found > minVersion[i]) {
break; // most significant version is good
} else if (found < minVersion[i]) {
throw new IllegalStateException("Unsupported version of " + tool + ". Found [" + version + "], expected [" +
Stream.of(minVersion).map(String::valueOf).collect(Collectors.joining(".")) + "+");
} // else equal, so check next element
}
}
}
/**
* Adds global hooks to manage destroying, starting and updating VMs.
*/
static class VagrantManagerPlugin implements Plugin<Project>, TaskActionListener, TaskExecutionListener {
@Override
public void apply(Project project) {
if (project != project.getRootProject()) {
throw new IllegalArgumentException("VagrantManagerPlugin can only be applied to the root project of a build");
}
project.getGradle().addListener(this);
}
private void callIfVagrantTask(Task task, Consumer<VagrantMachine> method) {
if (task instanceof VagrantShellTask) {
VagrantMachine service = task.getProject().getExtensions().getByType(VagrantMachine.class);
method.accept(service);
}
}
@Override
public void beforeExecute(Task task) { /* nothing to do */}
@Override
public void afterActions(Task task) { /* nothing to do */ }
@Override
public void beforeActions(Task task) {
callIfVagrantTask(task, VagrantMachine::maybeStartVM);
}
@Override
public void afterExecute(Task task, TaskState state) {
callIfVagrantTask(task, service -> service.maybeStopVM(state.getFailure() != null));
}
}
}

93
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant;
import org.gradle.api.Project;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import java.io.File;
import java.util.Map;
public class VagrantExtension {
private final Property<String> box;
private final MapProperty<String, Object> hostEnv;
private final MapProperty<String, Object> vmEnv;
private final RegularFileProperty vagrantfile;
private boolean isWindowsVM;
public VagrantExtension(Project project) {
this.box = project.getObjects().property(String.class);
this.hostEnv = project.getObjects().mapProperty(String.class, Object.class);
this.vmEnv = project.getObjects().mapProperty(String.class, Object.class);
this.vagrantfile = project.getObjects().fileProperty();
this.vagrantfile.convention(project.getRootProject().getLayout().getProjectDirectory().file("Vagrantfile"));
this.isWindowsVM = false;
}
@Input
public String getBox() {
return box.get();
}
public void setBox(String box) {
// TODO: should verify this against the Vagrantfile, but would need to do so in afterEvaluate once vagrantfile is unmodifiable
this.box.set(box);
}
@Input
public Map<String, Object> getHostEnv() {
return hostEnv.get();
}
public void hostEnv(String name, Object value) {
hostEnv.put(name, value);
}
@Input
public Map<String, Object> getVmEnv() {
return vmEnv.get();
}
public void vmEnv(String name, Object value) {
vmEnv.put(name, value);
}
@Input
public boolean isWindowsVM() {
return isWindowsVM;
}
public void setIsWindowsVM(boolean isWindowsVM) {
this.isWindowsVM = isWindowsVM;
}
@Input
public File getVagrantfile() {
return this.vagrantfile.get().getAsFile();
}
public void setVagrantfile(File file) {
vagrantfile.set(file);
}
}

210
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java Normal file
View File

@ -0,0 +1,210 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant;
import org.apache.commons.io.output.TeeOutputStream;
import org.elasticsearch.gradle.LoggedExec;
import org.elasticsearch.gradle.LoggingOutputStream;
import org.elasticsearch.gradle.ReaperService;
import org.elasticsearch.gradle.Util;
import org.gradle.api.Action;
import org.gradle.api.Project;
import org.gradle.internal.logging.progress.ProgressLogger;
import org.gradle.internal.logging.progress.ProgressLoggerFactory;
import javax.inject.Inject;
import java.io.File;
import java.io.OutputStream;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.UnaryOperator;
/**
* An helper to manage a vagrant box.
*
* This is created alongside a {@link VagrantExtension} for a project to manage starting and
* stopping a single vagrant box.
*/
public class VagrantMachine {
private final Project project;
private final VagrantExtension extension;
private final ReaperService reaper;
// pkg private so plugin can set this after construction
long refs;
private boolean isVMStarted = false;
public VagrantMachine(Project project, VagrantExtension extension, ReaperService reaper) {
this.project = project;
this.extension = extension;
this.reaper = reaper;
}
@Inject
protected ProgressLoggerFactory getProgressLoggerFactory() {
throw new UnsupportedOperationException();
}
public void execute(Action<VagrantExecSpec> action) {
VagrantExecSpec vagrantSpec = new VagrantExecSpec();
action.execute(vagrantSpec);
Objects.requireNonNull(vagrantSpec.command);
LoggedExec.exec(project, execSpec -> {
execSpec.setExecutable("vagrant");
File vagrantfile = extension.getVagrantfile();
execSpec.setEnvironment(System.getenv()); // pass through env
execSpec.environment("VAGRANT_CWD", vagrantfile.getParentFile().toString());
execSpec.environment("VAGRANT_VAGRANTFILE", vagrantfile.getName());
execSpec.environment("VAGRANT_LOG", "debug");
extension.getHostEnv().forEach(execSpec::environment);
execSpec.args(vagrantSpec.command);
if (vagrantSpec.subcommand != null) {
execSpec.args(vagrantSpec.subcommand);
}
execSpec.args(extension.getBox());
if (vagrantSpec.args != null) {
execSpec.args(Arrays.asList(vagrantSpec.args));
}
UnaryOperator<String> progressHandler = vagrantSpec.progressHandler;
if (progressHandler == null) {
progressHandler = new VagrantProgressLogger("==> " + extension.getBox() + ": ");
}
OutputStream output = execSpec.getStandardOutput();
// output from vagrant needs to be manually curated because --machine-readable isn't actually "readable"
OutputStream progressStream = new ProgressOutputStream(vagrantSpec.command, progressHandler);
execSpec.setStandardOutput(new TeeOutputStream(output, progressStream));
});
}
// start the configuration VM if it hasn't been started yet
void maybeStartVM() {
if (isVMStarted) {
return;
}
execute(spec -> {
spec.setCommand("box");
spec.setSubcommand("update");
});
// Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag that defaults
// to true that can be used to control whether or not to destroy any test boxes before test execution.
boolean destroyVM = Util.getBooleanProperty("vagrant.destroy", true);
if (destroyVM) {
execute(spec -> {
spec.setCommand("destroy");
spec.setArgs("--force");
});
}
// register box to be shutdown if gradle dies
reaper.registerCommand(extension.getBox(), "vagrant", "halt", "-f", extension.getBox());
// We lock the provider to virtualbox because the Vagrantfile specifies lots of boxes that only work
// properly in virtualbox. Virtualbox is vagrant's default but its possible to change that default and folks do.
execute(spec -> {
spec.setCommand("up");
spec.setArgs("--provision", "--provider", "virtualbox");
});
isVMStarted = true;
}
// stops the VM if refs are down to 0, or force was called
void maybeStopVM(boolean force) {
assert refs >= 1;
this.refs--;
if ((refs == 0 || force) && isVMStarted) {
execute(spec -> spec.setCommand("halt"));
reaper.unregister(extension.getBox());
}
}
// convert the given path from an elasticsearch repo path to a VM path
public static String convertLinuxPath(Project project, String path) {
return "/elasticsearch/" + project.getRootDir().toPath().relativize(Paths.get(path));
}
public static String convertWindowsPath(Project project, String path) {
return "C:\\elasticsearch\\" + project.getRootDir().toPath().relativize(Paths.get(path)).toString().replace('/', '\\');
}
public static class VagrantExecSpec {
private String command;
private String subcommand;
private String[] args;
private UnaryOperator<String> progressHandler;
private VagrantExecSpec() {}
public void setCommand(String command) {
this.command = command;
}
public void setSubcommand(String subcommand) {
this.subcommand = subcommand;
}
public void setArgs(String... args) {
this.args = args;
}
/**
* A function to translate output from the vagrant command execution to the progress line.
*
* The function takes the current line of output from vagrant, and returns a new
* progress line, or {@code null} if there is no update.
*/
public void setProgressHandler(UnaryOperator<String> progressHandler) {
this.progressHandler = progressHandler;
}
}
private class ProgressOutputStream extends LoggingOutputStream {
private ProgressLogger progressLogger;
private UnaryOperator<String> progressHandler;
ProgressOutputStream(String command, UnaryOperator<String> progressHandler) {
this.progressHandler = progressHandler;
this.progressLogger = getProgressLoggerFactory().newOperation("vagrant");
progressLogger.start(extension.getBox() + "> " + command, "hello");
}
@Override
protected void logLine(String line) {
String progress = progressHandler.apply(line);
if (progress != null) {
progressLogger.progress(progress);
}
System.out.println(line);
}
@Override
public void close() {
progressLogger.completed();
}
}
}

31
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java → buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java
View File

@ -19,30 +19,23 @@
package org.elasticsearch.gradle.vagrant; package org.elasticsearch.gradle.vagrant;
import org.elasticsearch.gradle.LoggingOutputStream; import java.util.function.UnaryOperator;
import org.gradle.internal.logging.progress.ProgressLogger;
public class VagrantProgressLogger implements UnaryOperator<String> {
public class VagrantLoggerOutputStream extends LoggingOutputStream {
private static final String HEADING_PREFIX = "==> "; private static final String HEADING_PREFIX = "==> ";
private final ProgressLogger progressLogger;
private final String squashedPrefix; private final String squashedPrefix;
private boolean isStarted = false;
private String lastLine = ""; private String lastLine = "";
private boolean inProgressReport = false;
private String heading = ""; private String heading = "";
private boolean inProgressReport = false;
VagrantLoggerOutputStream(ProgressLogger progressLogger, String squashedPrefix) { public VagrantProgressLogger(String squashedPrefix) {
this.progressLogger = progressLogger;
this.squashedPrefix = squashedPrefix; this.squashedPrefix = squashedPrefix;
} }
@Override @Override
protected void logLine(String line) { public String apply(String line) {
if (isStarted == false) {
progressLogger.started("started");
isStarted = true;
}
if (line.startsWith("\r\u001b")) { if (line.startsWith("\r\u001b")) {
/* We don't want to try to be a full terminal emulator but we want to /* We don't want to try to be a full terminal emulator but we want to
keep the escape sequences from leaking and catch _some_ of the keep the escape sequences from leaking and catch _some_ of the
@ -51,7 +44,7 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream {
if ("[K".equals(line)) { if ("[K".equals(line)) {
inProgressReport = true; inProgressReport = true;
} }
return; return null;
} }
if (line.startsWith(squashedPrefix)) { if (line.startsWith(squashedPrefix)) {
line = line.substring(squashedPrefix.length()); line = line.substring(squashedPrefix.length());
@ -67,14 +60,8 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream {
inProgressReport = false; inProgressReport = false;
line = lastLine + line; line = lastLine + line;
} else { } else {
return; return null;
} }
progressLogger.progress(line); return line;
}
@Override
public void close() {
flush();
progressLogger.completed();
} }
} }

109
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.TaskAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
/**
* A shell script to run within a vagrant VM.
*
* The script is run as root within the VM.
*/
public abstract class VagrantShellTask extends DefaultTask {
private final VagrantExtension extension;
private final VagrantMachine service;
private UnaryOperator<String> progressHandler = UnaryOperator.identity();
public VagrantShellTask() {
extension = getProject().getExtensions().findByType(VagrantExtension.class);
if (extension == null) {
throw new IllegalStateException("elasticsearch.vagrant-base must be applied to create " + getClass().getName());
}
service = getProject().getExtensions().getByType(VagrantMachine.class);
}
@Input
protected abstract List<String> getWindowsScript();
@Input
protected abstract List<String> getLinuxScript();
@Input
public UnaryOperator<String> getProgressHandler() {
return progressHandler;
}
public void setProgressHandler(UnaryOperator<String> progressHandler) {
this.progressHandler = progressHandler;
}
@TaskAction
public void runScript() {
String rootDir = getProject().getRootDir().toString();
if (extension.isWindowsVM()) {
service.execute(spec -> {
spec.setCommand("winrm");
List<String> script = new ArrayList<>();
script.add("try {");
script.add("cd " + convertWindowsPath(getProject(), rootDir));
extension.getVmEnv().forEach((k, v) -> script.add("$Env:" + k + " = \"" + v + "\""));
script.addAll(getWindowsScript().stream().map(s -> " " + s).collect(Collectors.toList()));
script.addAll(Arrays.asList(
" exit $LASTEXITCODE",
"} catch {",
// catch if we have a failure to even run the script at all above, equivalent to set -e, sort of
" echo $_.Exception.Message",
" exit 1",
"}"));
spec.setArgs("--elevated", "--command", String.join("\n", script));
spec.setProgressHandler(progressHandler);
});
} else {
service.execute(spec -> {
spec.setCommand("ssh");
List<String> script = new ArrayList<>();
script.add("sudo bash -c '"); // start inline bash script
script.add("pwd");
script.add("cd " + convertLinuxPath(getProject(), rootDir));
extension.getVmEnv().forEach((k, v) -> script.add("export " + k + "=" + v));
script.addAll(getLinuxScript());
script.add("'"); // end inline bash script
spec.setArgs("--command", String.join("\n", script));
spec.setProgressHandler(progressHandler);
});
}
}
}

20
buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
implementation-class=org.elasticsearch.gradle.test.DistroTestPlugin

1
buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties
View File

@ -1 +0,0 @@
implementation-class=org.elasticsearch.gradle.vagrant.VagrantTestPlugin

1
buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties
View File

@ -1 +0,0 @@
implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin

18
buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java
View File

@ -16,10 +16,13 @@ import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.regex.Pattern;
/** /**
* A wrapper around gradle's Exec task to capture output and log on error. * A wrapper around gradle's Exec task to capture output and log on error.
@ -98,6 +101,8 @@ public class LoggedExec extends Exec {
return genericExec(project, project::javaexec, action); return genericExec(project, project::javaexec, action);
} }
private static final Pattern NEWLINE = Pattern.compile(System.lineSeparator());
private static <T extends BaseExecSpec> ExecResult genericExec( private static <T extends BaseExecSpec> ExecResult genericExec(
Project project, Project project,
Function<Action<T>,ExecResult> function, Function<Action<T>,ExecResult> function,
@ -107,19 +112,20 @@ public class LoggedExec extends Exec {
return function.apply(action); return function.apply(action);
} }
ByteArrayOutputStream output = new ByteArrayOutputStream(); ByteArrayOutputStream output = new ByteArrayOutputStream();
ByteArrayOutputStream error = new ByteArrayOutputStream();
try { try {
return function.apply(spec -> { return function.apply(spec -> {
spec.setStandardOutput(output); spec.setStandardOutput(output);
spec.setErrorOutput(error); spec.setErrorOutput(output);
action.execute(spec); action.execute(spec);
try {
output.write(("Output for " + spec.getExecutable() + ":").getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}); });
} catch (Exception e) { } catch (Exception e) {
try { try {
project.getLogger().error("Standard output:"); NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> project.getLogger().error("| " + s));
project.getLogger().error(output.toString("UTF-8"));
project.getLogger().error("Standard error:");
project.getLogger().error(error.toString("UTF-8"));
} catch (UnsupportedEncodingException ue) { } catch (UnsupportedEncodingException ue) {
throw new GradleException("Failed to read exec output", ue); throw new GradleException("Failed to read exec output", ue);
} }

4
buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java
View File

@ -63,12 +63,12 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) { Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
filebytes = stream.readAllBytes(); filebytes = stream.readAllBytes();
} }
String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip"; String urlPath = "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip";
wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200))); wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes))); wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
wireMock.start(); wireMock.start();
assertExtractedDistro("1.0.0", "archive", "windows", null, null, assertExtractedDistro("7.0.0", "archive", "windows", null, null,
"tests.download_service", wireMock.baseUrl()); "tests.download_service", wireMock.baseUrl());
} catch (Exception e) { } catch (Exception e) {
// for debugging // for debugging

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/10_basic.bats → qa/vagrant/bats/default/10_basic.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats → qa/vagrant/bats/default/20_tar_bootstrap_password.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats → qa/vagrant/bats/default/25_package_bootstrap_password.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats → qa/vagrant/bats/default/30_tar_setup_passwords.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats → qa/vagrant/bats/default/35_package_setup_passwords.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats → qa/vagrant/bats/default/40_tar_certgen.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats → qa/vagrant/bats/default/45_package_certgen.bats
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash → qa/vagrant/bats/default/bootstrap_password.bash
View File

7
x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash → qa/vagrant/bats/default/certgen.bash
View File

@ -15,7 +15,6 @@ instances="/tmp/instances.yml"
certificates="/tmp/certificates.zip" certificates="/tmp/certificates.zip"
setup() { setup() {
export PACKAGE_NAME="elasticsearch"
if [ $BATS_TEST_NUMBER == 1 ]; then if [ $BATS_TEST_NUMBER == 1 ]; then
clean_before_test clean_before_test
fi fi
@ -176,6 +175,7 @@ NEW_PASS
} }
@test "[$GROUP] create instances file" { @test "[$GROUP] create instances file" {
rm -f /tmp/instances.yml
run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE" run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE"
cat > /tmp/instances.yml <<- EOF cat > /tmp/instances.yml <<- EOF
instances: instances:
@ -426,3 +426,8 @@ DATA_SETTINGS
false false
} }
} }
@test "[$GROUP] remove Elasticsearch" {
# NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed
clean_before_test
}

0
x-pack/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash → qa/vagrant/bats/default/setup_passwords.bash
View File

0
qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats → qa/vagrant/bats/oss/25_tar_plugins.bats
View File

0
qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats → qa/vagrant/bats/oss/50_modules_and_plugins.bats
View File

0
qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats → qa/vagrant/bats/oss/70_sysv_initd.bats
View File

3
qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats → qa/vagrant/bats/oss/80_upgrade.bats
View File

@ -122,6 +122,7 @@ setup() {
curl -s localhost:9200/library2/book/1?pretty | grep Darkness curl -s localhost:9200/library2/book/1?pretty | grep Darkness
} }
@test "[UPGRADE] stop version under test" { @test "[UPGRADE] cleanup version under test" {
stop_elasticsearch_service stop_elasticsearch_service
clean_before_test
} }

0
qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash → qa/vagrant/bats/oss/module_and_plugin_test_cases.bash
View File

0
qa/vagrant/src/test/resources/packaging/utils/modules.bash → qa/vagrant/bats/utils/modules.bash
View File

2
qa/vagrant/src/test/resources/packaging/utils/packages.bash → qa/vagrant/bats/utils/packages.bash
View File

@ -48,7 +48,7 @@ export_elasticsearch_paths() {
export ESDATA="/var/lib/elasticsearch" export ESDATA="/var/lib/elasticsearch"
export ESLOG="/var/log/elasticsearch" export ESLOG="/var/log/elasticsearch"
export ESENVFILE=$(env_file) export ESENVFILE=$(env_file)
export PACKAGE_NAME=${PACKAGE_NAME:-"elasticsearch-oss"} export PACKAGE_NAME
} }

0
qa/vagrant/src/test/resources/packaging/utils/plugins.bash → qa/vagrant/bats/utils/plugins.bash
View File

0
qa/vagrant/src/test/resources/packaging/utils/tar.bash → qa/vagrant/bats/utils/tar.bash
View File

0
qa/vagrant/src/test/resources/packaging/utils/utils.bash → qa/vagrant/bats/utils/utils.bash
View File

0
x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash → qa/vagrant/bats/utils/xpack.bash
View File

89
qa/vagrant/build.gradle
View File

@ -18,10 +18,7 @@
*/ */
plugins { plugins {
id 'java'
id 'elasticsearch.build' id 'elasticsearch.build'
id 'elasticsearch.vagrantsupport'
id 'elasticsearch.vagrant'
} }
dependencies { dependencies {
@ -36,33 +33,13 @@ dependencies {
compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-logging:commons-logging:${versions.commonslogging}"
compile project(':libs:elasticsearch-core') compile project(':libs:elasticsearch-core')
// pulls in the jar built by this project and its dependencies
packagingTest project(path: project.path, configuration: 'runtime')
} }
List<String> plugins = [] configurations.create('testClasses')
for (Project subproj : project.rootProject.subprojects) {
if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) {
// add plugin as a dep
dependencies {
packaging project(path: "${subproj.path}", configuration: 'zip')
}
plugins.add(subproj.name)
}
}
plugins = plugins.toSorted()
setupPackagingTest { String classesDir = project.file(project.sourceSets.main.output.classesDirs.singleFile).toString()
doFirst { artifacts.add('testClasses', project.layout.projectDirectory.dir(classesDir)) {
File expectedPlugins = file('build/plugins/expected') builtBy tasks.named('testClasses')
expectedPlugins.parentFile.mkdirs()
expectedPlugins.setText(plugins.join('\n'), 'UTF-8')
}
}
esvagrant {
testClass 'org.elasticsearch.packaging.PackagingTests'
} }
forbiddenApisMain { forbiddenApisMain {
@ -91,3 +68,61 @@ tasks.thirdPartyAudit.ignoreMissingClasses (
'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener' 'javax.servlet.ServletContextListener'
) )
boolean sample = project.properties.get('vagrant.boxes') != 'all'
subprojects { Project platformProject ->
apply plugin: 'elasticsearch.distro-test'
apply plugin: 'java'
configurations.create('testClasses')
dependencies {
testClasses project(path: ':qa:vagrant', configuration: 'testClasses')
testRuntime project(path: ':qa:vagrant', configuration: 'runtime')
}
tasks.named('destructiveDistroTest') {
testClassesDirs += project.files(configurations.testClasses.singleFile)
}
// TODO: remove this property lookup once CI is switched to use an explicit task for the sample tests
boolean allBoxes = project.properties.get('vagrant.boxes', '') == 'all'
if (allBoxes || ['centos-7', 'ubuntu-1604'].contains(platformProject.name)) {
tasks.register('packagingTest') {
dependsOn 'distroTest', 'batsTest.oss', 'batsTest.default'
}
}
vagrant {
hostEnv 'VAGRANT_PROJECT_DIR', platformProject.projectDir.absolutePath
}
}
configurations {
allPlugins
}
List<String> plugins = []
for (Project subproj : project.rootProject.subprojects) {
if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) {
// add plugin as a dep
dependencies {
allPlugins project(path: "${subproj.path}", configuration: 'zip')
}
plugins.add(subproj.name)
}
}
plugins = plugins.toSorted()
copyPackagingArchives {
from configurations.allPlugins
doLast {
// TODO: this was copied from the old way bats tests get the plugins list. we should pass
// this in differently when converting to java tests
File expectedPlugins = file('build/plugins/expected')
expectedPlugins.parentFile.mkdirs()
expectedPlugins.setText(plugins.join('\n'), 'UTF-8')
}
}

0
qa/vagrant/centos-6/build.gradle Normal file
View File

0
qa/vagrant/centos-7/build.gradle Normal file
View File

0
qa/vagrant/debian-8/build.gradle Normal file
View File

0
qa/vagrant/debian-9/build.gradle Normal file
View File

0
qa/vagrant/fedora-28/build.gradle Normal file
View File

0
qa/vagrant/fedora-29/build.gradle Normal file
View File

0
qa/vagrant/oel-6/build.gradle Normal file
View File

0
qa/vagrant/oel-7/build.gradle Normal file
View File

0
qa/vagrant/opensuse-42/build.gradle Normal file
View File

0
qa/vagrant/sles-12/build.gradle Normal file
View File

73
qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java
View File

@ -1,73 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.packaging;
import org.elasticsearch.packaging.test.DefaultDebBasicTests;
import org.elasticsearch.packaging.test.DefaultDebPreservationTests;
import org.elasticsearch.packaging.test.DefaultLinuxTarTests;
import org.elasticsearch.packaging.test.DefaultNoJdkDebBasicTests;
import org.elasticsearch.packaging.test.DefaultNoJdkLinuxTarTests;
import org.elasticsearch.packaging.test.DefaultNoJdkRpmBasicTests;
import org.elasticsearch.packaging.test.DefaultNoJdkWindowsZipTests;
import org.elasticsearch.packaging.test.DefaultRpmBasicTests;
import org.elasticsearch.packaging.test.DefaultRpmPreservationTests;
import org.elasticsearch.packaging.test.DefaultWindowsServiceTests;
import org.elasticsearch.packaging.test.DefaultWindowsZipTests;
import org.elasticsearch.packaging.test.OssDebBasicTests;
import org.elasticsearch.packaging.test.OssDebPreservationTests;
import org.elasticsearch.packaging.test.OssLinuxTarTests;
import org.elasticsearch.packaging.test.OssNoJdkDebBasicTests;
import org.elasticsearch.packaging.test.OssNoJdkLinuxTarTests;
import org.elasticsearch.packaging.test.OssNoJdkRpmBasicTests;
import org.elasticsearch.packaging.test.OssNoJdkWindowsZipTests;
import org.elasticsearch.packaging.test.OssRpmBasicTests;
import org.elasticsearch.packaging.test.OssRpmPreservationTests;
import org.elasticsearch.packaging.test.OssWindowsServiceTests;
import org.elasticsearch.packaging.test.OssWindowsZipTests;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
DefaultLinuxTarTests.class,
OssLinuxTarTests.class,
DefaultWindowsZipTests.class,
OssWindowsZipTests.class,
DefaultRpmBasicTests.class,
OssRpmBasicTests.class,
DefaultDebBasicTests.class,
OssDebBasicTests.class,
DefaultDebPreservationTests.class,
OssDebPreservationTests.class,
DefaultRpmPreservationTests.class,
OssRpmPreservationTests.class,
DefaultWindowsServiceTests.class,
OssWindowsServiceTests.class,
DefaultNoJdkLinuxTarTests.class,
OssNoJdkLinuxTarTests.class,
DefaultNoJdkWindowsZipTests.class,
OssNoJdkWindowsZipTests.class,
DefaultNoJdkRpmBasicTests.class,
OssNoJdkRpmBasicTests.class,
DefaultNoJdkDebBasicTests.class,
OssNoJdkDebBasicTests.class
})
public class PackagingTests {}

15
qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.packaging.test; package org.elasticsearch.packaging.test;
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.http.client.fluent.Request; import org.apache.http.client.fluent.Request;
import org.elasticsearch.packaging.util.Archives; import org.elasticsearch.packaging.util.Archives;
import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Distribution;
@ -36,7 +35,6 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.stream.Stream; import java.util.stream.Stream;
import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER;
import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.installArchive;
import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
@ -225,17 +223,16 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
final Shell sh = new Shell(); final Shell sh = new Shell();
// Create temporary directory with a space and link to java binary. // Create temporary directory with a space and link to java binary.
// Use it as java_home // Use it as java_home
String nameWithSpace = RandomStrings.randomAsciiAlphanumOfLength(getRandom(), 10) + "java home"; String testJavaHome = FileUtils.mkdir(Paths.get("/home", ARCHIVE_OWNER, "java home")).toAbsolutePath().toString();
String test_java_home = FileUtils.mkdir(Paths.get("/home",ARCHIVE_OWNER, nameWithSpace)).toAbsolutePath().toString();
try { try {
final String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); final String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
final String java = systemJavaHome + "/bin/java"; final String java = systemJavaHome + "/bin/java";
sh.run("mkdir -p \"" + test_java_home + "/bin\""); sh.run("mkdir -p \"" + testJavaHome + "/bin\"");
sh.run("ln -s \"" + java + "\" \"" + test_java_home + "/bin/java\""); sh.run("ln -s \"" + java + "\" \"" + testJavaHome + "/bin/java\"");
sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + test_java_home + "\""); sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + testJavaHome + "\"");
sh.getEnv().put("JAVA_HOME", test_java_home); sh.getEnv().put("JAVA_HOME", testJavaHome);
//verify ES can start, stop and run plugin list //verify ES can start, stop and run plugin list
Archives.runElasticsearch(installation, sh); Archives.runElasticsearch(installation, sh);
@ -246,7 +243,7 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
Result result = sh.run(pluginListCommand); Result result = sh.run(pluginListCommand);
assertThat(result.exitCode, equalTo(0)); assertThat(result.exitCode, equalTo(0));
} finally { } finally {
FileUtils.rm(Paths.get("\"" + test_java_home + "\"")); FileUtils.rm(Paths.get(testJavaHome));
} }
}); });
} }

6
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java
View File

@ -45,10 +45,8 @@ import java.util.StringJoiner;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.ZipException; import java.util.zip.ZipException;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.core.IsNot.not; import static org.hamcrest.core.IsNot.not;
import static org.hamcrest.text.IsEmptyString.isEmptyOrNullString;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -228,9 +226,7 @@ public class FileUtils {
} }
public static Path getPackagingArchivesDir() { public static Path getPackagingArchivesDir() {
String fromEnv = System.getenv("PACKAGING_ARCHIVES"); return Paths.get(""); // tests are started in the packaging archives dir, ie the empty relative path
assertThat(fromEnv, not(isEmptyOrNullString()));
return Paths.get(fromEnv);
} }
public static Path getDistributionFile(Distribution distribution) { public static Path getDistributionFile(Distribution distribution) {

0
qa/vagrant/ubuntu-1604/build.gradle Normal file
View File

0
qa/vagrant/ubuntu-1804/build.gradle Normal file
View File

11
qa/vagrant/windows-2012r2/build.gradle Normal file
View File

@ -0,0 +1,11 @@
String boxId = project.properties.get('vagrant.windows-2012r2.id')
if (boxId != null) {
vagrant {
hostEnv 'VAGRANT_WINDOWS_2012R2_BOX', boxId
}
} else {
tasks.named('distroTest').configure {
onlyIf { false }
}
}

11
qa/vagrant/windows-2016/build.gradle Normal file
View File

@ -0,0 +1,11 @@
String boxId = project.properties.get('vagrant.windows-2016.id')
if (boxId != null) {
vagrant {
hostEnv 'VAGRANT_WINDOWS_2016_BOX', boxId
}
} else {
tasks.named('distroTest').configure {
onlyIf { true }
}
}

11
x-pack/qa/vagrant/build.gradle
View File

@ -1,11 +0,0 @@
apply plugin: 'elasticsearch.vagrantsupport'
apply plugin: 'elasticsearch.vagrant'
esvagrant {
inheritTestUtils true
}
dependencies {
// Inherit Bats test utils from :qa:vagrant project
packaging project(path: ':qa:vagrant', configuration: 'packaging')
}