diff --git a/Vagrantfile b/Vagrantfile index 14f6ad00f3a..ed572aba8a8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -41,6 +41,16 @@ Vagrant.configure(2) do |config| # the elasticsearch project called vagrant.... config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.synced_folder '.', '/elasticsearch' + # TODO: make these syncs work for windows!!! + config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/jars-3", "/root/.gradle/caches/jars-3", + create: true, + owner: "vagrant" + config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/modules-2", "/root/.gradle/caches/modules-2", + create: true, + owner: "vagrant" + config.vm.synced_folder "#{Dir.home}/.gradle/wrapper", "/root/.gradle/wrapper", + create: true, + owner: "vagrant" # Expose project directory. Note that VAGRANT_CWD may not be the same as Dir.pwd PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd @@ -380,10 +390,6 @@ export ZIP=/elasticsearch/distribution/zip/build/distributions export TAR=/elasticsearch/distribution/tar/build/distributions export RPM=/elasticsearch/distribution/rpm/build/distributions export DEB=/elasticsearch/distribution/deb/build/distributions -export BATS=/project/build/bats -export BATS_UTILS=/project/build/packaging/bats/utils -export BATS_TESTS=/project/build/packaging/bats/tests -export PACKAGING_ARCHIVES=/project/build/packaging/archives export PACKAGING_TESTS=/project/build/packaging/tests VARS cat \<\ /etc/sudoers.d/elasticsearch_vars @@ -391,11 +397,10 @@ Defaults env_keep += "ZIP" Defaults env_keep += "TAR" Defaults env_keep += "RPM" Defaults env_keep += "DEB" -Defaults env_keep += "BATS" -Defaults env_keep += "BATS_UTILS" -Defaults env_keep += "BATS_TESTS" Defaults env_keep += "PACKAGING_ARCHIVES" Defaults env_keep += "PACKAGING_TESTS" +Defaults env_keep += "BATS_UTILS" +Defaults env_keep += "BATS_TESTS" Defaults env_keep += "JAVA_HOME" Defaults env_keep += "SYSTEM_JAVA_HOME" SUDOERS_VARS diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java new file mode 100644 index 00000000000..d78dba6b47c --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -0,0 +1,300 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.BuildPlugin; +import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.DistributionDownloadPlugin; +import org.elasticsearch.gradle.ElasticsearchDistribution; +import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; +import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; +import org.elasticsearch.gradle.ElasticsearchDistribution.Type; +import org.elasticsearch.gradle.Jdk; +import org.elasticsearch.gradle.JdkDownloadPlugin; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.elasticsearch.gradle.vagrant.BatsProgressLogger; +import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; +import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.Directory; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.TaskInputs; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.testing.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +public class DistroTestPlugin implements Plugin { + + private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691"; + + // all distributions used by distro tests. this is temporary until tests are per distribution + private static final String PACKAGING_DISTRIBUTION = "packaging"; + private static final String COPY_PACKAGING_TASK = "copyPackagingArchives"; + private static final String IN_VM_SYSPROP = "tests.inVM"; + + private static Version upgradeVersion; + private Provider archivesDir; + private TaskProvider copyPackagingArchives; + private Jdk gradleJdk; + + @Override + public void apply(Project project) { + project.getPluginManager().apply(JdkDownloadPlugin.class); + project.getPluginManager().apply(DistributionDownloadPlugin.class); + project.getPluginManager().apply(VagrantBasePlugin.class); + project.getPluginManager().apply(JavaPlugin.class); + + configureVM(project); + + if (upgradeVersion == null) { + // just read this once, since it is the same for all projects. this is safe because gradle configuration is single threaded + upgradeVersion = getUpgradeVersion(project); + } + + // setup task to run inside VM + configureDistributions(project); + configureCopyPackagingTask(project); + configureDistroTest(project); + configureBatsTest(project, "oss"); + configureBatsTest(project, "default"); + } + + private static Jdk createJdk(NamedDomainObjectContainer jdksContainer, String name, String version, String platform) { + Jdk jdk = jdksContainer.create(name); + jdk.setVersion(version); + jdk.setPlatform(platform); + return jdk; + } + + private static Version getUpgradeVersion(Project project) { + String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion"); + if (upgradeFromVersionRaw != null) { + return Version.fromString(upgradeFromVersionRaw); + } + + // was not passed in, so randomly choose one from bwc versions + ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class); + + if ((boolean) extraProperties.get("bwc_tests_enabled") == false) { + // Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests + return Version.fromString(project.getVersion().toString()); + } + + ExtraPropertiesExtension rootExtraProperties = project.getRootProject().getExtensions().getByType(ExtraPropertiesExtension.class); + String firstPartOfSeed = rootExtraProperties.get("testSeed").toString().split(":")[0]; + final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16); + BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions"); + final List indexCompatVersions = bwcVersions.getIndexCompatible(); + return indexCompatVersions.get(new Random(seed).nextInt(indexCompatVersions.size())); + } + + private void configureVM(Project project) { + String box = project.getName(); + + // setup jdks used by the distro tests, and by gradle executing + + NamedDomainObjectContainer jdksContainer = JdkDownloadPlugin.getContainer(project); + String platform = box.contains("windows") ? "windows" : "linux"; + this.gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VERSION, platform); + + // setup VM used by these tests + VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); + vagrant.setBox(box); + vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdk, "/bin:$PATH", "\\bin;$Env:PATH")); + vagrant.setIsWindowsVM(box.contains("windows")); + } + + private static Object convertPath(Project project, VagrantExtension vagrant, Jdk jdk, + String additionaLinux, String additionalWindows) { + return new Object() { + @Override + public String toString() { + if (vagrant.isWindowsVM()) { + return convertWindowsPath(project, jdk.getPath()) + additionalWindows; + } + return convertLinuxPath(project, jdk.getPath()) + additionaLinux; + } + }; + } + + private void configureCopyPackagingTask(Project project) { + this.archivesDir = project.getParent().getLayout().getBuildDirectory().dir("packaging/archives"); + // temporary, until we have tasks per distribution + this.copyPackagingArchives = Boilerplate.maybeRegister(project.getParent().getTasks(), COPY_PACKAGING_TASK, Copy.class, + t -> { + t.into(archivesDir); + t.from(project.getConfigurations().getByName(PACKAGING_DISTRIBUTION)); + + Path archivesPath = archivesDir.get().getAsFile().toPath(); + + // write bwc version, and append -SNAPSHOT if it is an unreleased version + ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class); + BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions"); + final String upgradeFromVersion; + if (bwcVersions.unreleasedInfo(upgradeVersion) != null) { + upgradeFromVersion = upgradeVersion.toString() + "-SNAPSHOT"; + } else { + upgradeFromVersion = upgradeVersion.toString(); + } + TaskInputs inputs = t.getInputs(); + inputs.property("version", VersionProperties.getElasticsearch()); + inputs.property("upgrade_from_version", upgradeFromVersion); + // TODO: this is serializable, need to think how to represent this as an input + //inputs.property("bwc_versions", bwcVersions); + t.doLast(action -> { + try { + Files.writeString(archivesPath.resolve("version"), VersionProperties.getElasticsearch()); + Files.writeString(archivesPath.resolve("upgrade_from_version"), upgradeFromVersion); + // this is always true, but bats tests rely on it. It is just temporary until bats is removed. + Files.writeString(archivesPath.resolve("upgrade_is_oss"), ""); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + }); + } + + private void configureDistroTest(Project project) { + BuildPlugin.configureCompile(project); + BuildPlugin.configureRepositories(project); + BuildPlugin.configureTestTasks(project); + BuildPlugin.configureInputNormalization(project); + + TaskProvider destructiveTest = project.getTasks().register("destructiveDistroTest", Test.class, + t -> { + t.setMaxParallelForks(1); + t.setWorkingDir(archivesDir.get()); + if (System.getProperty(IN_VM_SYSPROP) == null) { + t.dependsOn(copyPackagingArchives, gradleJdk); + } + }); + + // setup outer task to run + project.getTasks().register("distroTest", GradleDistroTestTask.class, + t -> { + t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + t.setDescription("Runs distribution tests within vagrant"); + t.setTaskName(project.getPath() + ":" + destructiveTest.getName()); + t.extraArg("-D'" + IN_VM_SYSPROP + "'"); + t.dependsOn(copyPackagingArchives, gradleJdk); + }); + } + + private void configureBatsTest(Project project, String type) { + + // destructive task to run inside + TaskProvider destructiveTest = project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class, + t -> { + // this is hacky for shared source, but bats are a temporary thing we are removing, so it is not worth + // the overhead of a real project dependency + Directory batsDir = project.getParent().getLayout().getProjectDirectory().dir("bats"); + t.setTestsDir(batsDir.dir(type)); + t.setUtilsDir(batsDir.dir("utils")); + t.setArchivesDir(archivesDir.get()); + t.setPackageName("elasticsearch" + (type.equals("oss") ? "-oss" : "")); + if (System.getProperty(IN_VM_SYSPROP) == null) { + t.dependsOn(copyPackagingArchives, gradleJdk); + } + }); + + VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); + // setup outer task to run + project.getTasks().register("batsTest." + type, GradleDistroTestTask.class, + t -> { + t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + t.setDescription("Runs bats tests within vagrant"); + t.setTaskName(project.getPath() + ":" + destructiveTest.getName()); + t.setProgressHandler(new BatsProgressLogger(project.getLogger())); + t.extraArg("-D'" + IN_VM_SYSPROP + "'"); + t.dependsOn(copyPackagingArchives, gradleJdk); + t.onlyIf(spec -> vagrant.isWindowsVM() == false); // bats doesn't run on windows + }); + } + + private void configureDistributions(Project project) { + NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); + + for (Type type : Arrays.asList(Type.DEB, Type.RPM)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch()); + } + } + // upgrade version is always bundled jdk + // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced + // witha dedicated upgrade test from every bwc version like other bwc tests + addDistro(distributions, type, null, Flavor.DEFAULT, true, upgradeVersion.toString()); + if (upgradeVersion.onOrAfter("6.3.0")) { + addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString()); + } + } + for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + addDistro(distributions, Type.ARCHIVE, platform, flavor, bundledJdk, VersionProperties.getElasticsearch()); + } + } + } + + // temporary until distro tests have one test per distro + Configuration packagingConfig = project.getConfigurations().create(PACKAGING_DISTRIBUTION); + List distroConfigs = distributions.stream().map(ElasticsearchDistribution::getConfiguration) + .collect(Collectors.toList()); + packagingConfig.setExtendsFrom(distroConfigs); + } + + private static void addDistro(NamedDomainObjectContainer distributions, + Type type, Platform platform, Flavor flavor, boolean bundledJdk, String version) { + + String name = flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") + "-" + version; + if (distributions.findByName(name) != null) { + return; + } + distributions.create(name, d -> { + d.setFlavor(flavor); + d.setType(type); + if (type == Type.ARCHIVE) { + d.setPlatform(platform); + } + d.setBundledJdk(bundledJdk); + d.setVersion(version); + }); + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy deleted file mode 100644 index fa08a8f9c66..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.test - -import org.elasticsearch.gradle.vagrant.VagrantCommandTask -import org.gradle.api.Task - -/** - * A fixture for integration tests which runs in a virtual machine launched by Vagrant. - */ -class VagrantFixture extends VagrantCommandTask implements Fixture { - - private VagrantCommandTask stopTask - - public VagrantFixture() { - this.stopTask = project.tasks.create(name: "${name}#stop", type: VagrantCommandTask) { - command 'halt' - } - finalizedBy this.stopTask - } - - @Override - void setBoxName(String boxName) { - super.setBoxName(boxName) - this.stopTask.setBoxName(boxName) - } - - @Override - void setEnvironmentVars(Map environmentVars) { - super.setEnvironmentVars(environmentVars) - this.stopTask.setEnvironmentVars(environmentVars) - } - - @Override - public Task getStopTask() { - return this.stopTask - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy deleted file mode 100644 index bcc612c7afa..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.apache.commons.io.output.TeeOutputStream -import org.elasticsearch.gradle.LoggedExec -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.Optional -import org.gradle.internal.logging.progress.ProgressLoggerFactory - -import javax.inject.Inject - -/** - * Runs a vagrant command. Pretty much like Exec task but with a nicer output - * formatter and defaults to `vagrant` as first part of commandLine. - */ -public class VagrantCommandTask extends LoggedExec { - - @Input - String command - - @Input @Optional - String subcommand - - @Input - String boxName - - @Input - Map environmentVars - - public VagrantCommandTask() { - executable = 'vagrant' - - // We're using afterEvaluate here to slot in some logic that captures configurations and - // modifies the command line right before the main execution happens. The reason that we - // call doFirst instead of just doing the work in the afterEvaluate is that the latter - // restricts how subclasses can extend functionality. Calling afterEvaluate is like having - // all the logic of a task happening at construction time, instead of at execution time - // where a subclass can override or extend the logic. - project.afterEvaluate { - doFirst { - if (environmentVars != null) { - environment environmentVars - } - - // Build our command line for vagrant - def vagrantCommand = [executable, command] - if (subcommand != null) { - vagrantCommand = vagrantCommand + subcommand - } - commandLine([*vagrantCommand, boxName, *args]) - - // It'd be nice if --machine-readable were, well, nice - standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream()) - } - } - } - - @Inject - ProgressLoggerFactory getProgressLoggerFactory() { - throw new UnsupportedOperationException() - } - - protected OutputStream createLoggerOutputStream() { - return new VagrantLoggerOutputStream(getProgressLoggerFactory().newOperation(boxName + " " + command).setDescription(boxName), - /* Vagrant tends to output a lot of stuff, but most of the important - stuff starts with ==> $box */ - "==> $boxName: ") - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy deleted file mode 100644 index e9b664a5a31..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.elasticsearch.gradle.Version -import org.gradle.api.tasks.Input - -class VagrantPropertiesExtension { - - @Input - List boxes - - @Input - Version upgradeFromVersion - - @Input - List upgradeFromVersions - - @Input - String batsDir - - @Input - Boolean inheritTests - - @Input - Boolean inheritTestUtils - - @Input - String testClass - - VagrantPropertiesExtension(List availableBoxes) { - this.boxes = availableBoxes - this.batsDir = 'src/test/resources/packaging' - } - - void boxes(String... boxes) { - this.boxes = Arrays.asList(boxes) - } - - void setBatsDir(String batsDir) { - this.batsDir = batsDir - } - - void setInheritTests(Boolean inheritTests) { - this.inheritTests = inheritTests - } - - void setInheritTestUtils(Boolean inheritTestUtils) { - this.inheritTestUtils = inheritTestUtils - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy deleted file mode 100644 index 9dfe487e830..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy +++ /dev/null @@ -1,127 +0,0 @@ -package org.elasticsearch.gradle.vagrant - -import org.gradle.api.GradleException -import org.gradle.api.InvalidUserDataException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.process.ExecResult -import org.gradle.process.internal.ExecException - -/** - * Global configuration for if Vagrant tasks are supported in this - * build environment. - */ -class VagrantSupportPlugin implements Plugin { - - @Override - void apply(Project project) { - if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) { - Map vagrantInstallation = getVagrantInstallation(project) - Map virtualBoxInstallation = getVirtualBoxInstallation(project) - - project.rootProject.ext.vagrantInstallation = vagrantInstallation - project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation - project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported - project.rootProject.ext.vagrantEnvChecksDone = true - - // Finding that HOME needs to be set when performing vagrant updates - String homeLocation = System.getenv("HOME") - if (project.rootProject.ext.vagrantSupported && homeLocation == null) { - throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " + - "and requires \$HOME to be set to function properly.") - } - } - - addVerifyInstallationTasks(project) - } - - private Map getVagrantInstallation(Project project) { - try { - ByteArrayOutputStream pipe = new ByteArrayOutputStream() - ExecResult runResult = project.exec { - commandLine 'vagrant', '--version' - standardOutput pipe - ignoreExitValue true - } - String version = pipe.toString().trim() - if (runResult.exitValue == 0) { - if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) { - return [ 'supported' : true ] - } else { - return [ 'supported' : false, - 'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ] - } - } else { - return [ 'supported' : false, - 'info' : "Could not read installed vagrant version:\n" + version ] - } - } catch (ExecException e) { - // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set. - // Swallow error. Vagrant isn't installed. Don't halt the build here. - return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ] - } - } - - private Map getVirtualBoxInstallation(Project project) { - try { - ByteArrayOutputStream pipe = new ByteArrayOutputStream() - ExecResult runResult = project.exec { - commandLine 'vboxmanage', '--version' - standardOutput = pipe - ignoreExitValue true - } - String version = pipe.toString().trim() - if (runResult.exitValue == 0) { - try { - String[] versions = version.split('\\.') - int major = Integer.parseInt(versions[0]) - int minor = Integer.parseInt(versions[1]) - if ((major < 5) || (major == 5 && minor < 1)) { - return [ 'supported' : false, - 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ] - } else { - return [ 'supported' : true ] - } - } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { - return [ 'supported' : false, - 'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ] - } - } else { - return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ] - } - } catch (ExecException e) { - // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set. - // Swallow error. VirtualBox isn't installed. Don't halt the build here. - return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ] - } - } - - private void addVerifyInstallationTasks(Project project) { - createCheckVagrantVersionTask(project) - createCheckVirtualBoxVersionTask(project) - } - - private void createCheckVagrantVersionTask(Project project) { - project.tasks.create('vagrantCheckVersion') { - description 'Check the Vagrant version' - group 'Verification' - doLast { - if (project.rootProject.vagrantInstallation.supported == false) { - throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info) - } - } - } - } - - private void createCheckVirtualBoxVersionTask(Project project) { - project.tasks.create('virtualboxCheckVersion') { - description 'Check the Virtualbox version' - group 'Verification' - doLast { - if (project.rootProject.virtualBoxInstallation.supported == false) { - throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info) - } - } - } - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy deleted file mode 100644 index 3868e0417f4..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ /dev/null @@ -1,658 +0,0 @@ -package org.elasticsearch.gradle.vagrant - -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.BwcVersions -import org.elasticsearch.gradle.FileContentsTask -import org.elasticsearch.gradle.Jdk -import org.elasticsearch.gradle.JdkDownloadPlugin -import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.Version -import org.gradle.api.GradleException -import org.gradle.api.InvalidUserDataException -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.artifacts.dsl.RepositoryHandler -import org.gradle.api.execution.TaskExecutionAdapter -import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Delete -import org.gradle.api.tasks.Exec -import org.gradle.api.tasks.StopExecutionException -import org.gradle.api.tasks.TaskState - -import java.nio.file.Paths - -import static java.util.Collections.unmodifiableList - -class VagrantTestPlugin implements Plugin { - - /** All Linux boxes that we test. These are all always supplied **/ - static final List LINUX_BOXES = unmodifiableList([ - 'centos-6', - 'centos-7', - 'debian-8', - 'debian-9', - 'fedora-28', - 'fedora-29', - 'oel-6', - 'oel-7', - 'opensuse-42', - /* TODO: need a real RHEL license now that it is out of beta 'rhel-8',*/ - 'sles-12', - 'ubuntu-1604', - 'ubuntu-1804' - ]) - - /** All Windows boxes that we test, which may or may not be supplied **/ - static final List WINDOWS_BOXES = unmodifiableList([ - 'windows-2012r2', - 'windows-2016' - ]) - - /** All boxes that we test, some of which may not be supplied **/ - static final List ALL_BOXES = unmodifiableList(LINUX_BOXES + WINDOWS_BOXES) - - /** Boxes used when sampling the tests **/ - static final List SAMPLE = unmodifiableList([ - 'centos-7', - 'ubuntu-1604' - ]) - - /** All distributions to bring into test VM, whether or not they are used **/ - static final List DISTRIBUTIONS = unmodifiableList([ - 'archives:linux-tar', - 'archives:oss-linux-tar', - 'archives:windows-zip', - 'archives:oss-windows-zip', - 'packages:rpm', - 'packages:oss-rpm', - 'packages:deb', - 'packages:oss-deb', - 'archives:no-jdk-linux-tar', - 'archives:oss-no-jdk-linux-tar', - 'archives:no-jdk-windows-zip', - 'archives:oss-no-jdk-windows-zip', - 'packages:no-jdk-rpm', - 'packages:oss-no-jdk-rpm', - 'packages:no-jdk-deb', - 'packages:oss-no-jdk-deb' - ]) - - /** Packages onboarded for upgrade tests **/ - static final List UPGRADE_FROM_ARCHIVES = unmodifiableList(['rpm', 'deb']) - - private static final PACKAGING_CONFIGURATION = 'packaging' - private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest' - private static final BATS = 'bats' - private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS" - - /** Boxes that have been supplied and are available for testing **/ - List availableBoxes = [] - - /** extra env vars to pass to vagrant for box configuration **/ - Map vagrantBoxEnvVars = [:] - - private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691" - private Jdk linuxGradleJdk; - private Jdk windowsGradleJdk; - - @Override - void apply(Project project) { - project.pluginManager.apply(JdkDownloadPlugin.class) - NamedDomainObjectContainer jdksContainer = (NamedDomainObjectContainer) project.getExtensions().getByName("jdks"); - linuxGradleJdk = jdksContainer.create("linux_gradle") { - version = GRADLE_JDK_VERSION - platform = "linux" - } - windowsGradleJdk = jdksContainer.create("windows_gradle") { - version = GRADLE_JDK_VERSION - platform = "windows" - } - - collectAvailableBoxes(project) - - // Creates the Vagrant extension for the project - project.extensions.create('esvagrant', VagrantPropertiesExtension, listSelectedBoxes(project)) - - // Add required repositories for packaging tests - configurePackagingArchiveRepositories(project) - - // Creates custom configurations for Bats testing files (and associated scripts and archives) - createPackagingConfiguration(project) - project.configurations.create(PACKAGING_TEST_CONFIGURATION) - - // Creates all the main Vagrant tasks - createVagrantTasks(project) - - if (project.extensions.esvagrant.boxes == null || project.extensions.esvagrant.boxes.size() == 0) { - throw new InvalidUserDataException('Must specify at least one vagrant box') - } - - for (String box : project.extensions.esvagrant.boxes) { - if (ALL_BOXES.contains(box) == false) { - throw new InvalidUserDataException("Vagrant box [${box}] is unknown to this plugin. Valid boxes are ${ALL_BOXES}") - } - - if (availableBoxes.contains(box) == false) { - throw new InvalidUserDataException("Vagrant box [${box}] is not available because an image is not supplied for it. " + - "Available boxes with supplied images are ${availableBoxes}") - } - } - - // Creates all tasks related to the Vagrant boxes - createVagrantBoxesTasks(project) - } - - /** - * Enumerate all the boxes that we know about and could possibly choose to test - */ - private void collectAvailableBoxes(Project project) { - // these images are hardcoded in the Vagrantfile and are always available - availableBoxes.addAll(LINUX_BOXES) - - // these images need to be provided at runtime - String windows_2012r2_box = project.getProperties().get('vagrant.windows-2012r2.id') - if (windows_2012r2_box != null && windows_2012r2_box.isEmpty() == false) { - availableBoxes.add('windows-2012r2') - vagrantBoxEnvVars['VAGRANT_WINDOWS_2012R2_BOX'] = windows_2012r2_box - } - - String windows_2016_box = project.getProperties().get('vagrant.windows-2016.id') - if (windows_2016_box != null && windows_2016_box.isEmpty() == false) { - availableBoxes.add('windows-2016') - vagrantBoxEnvVars['VAGRANT_WINDOWS_2016_BOX'] = windows_2016_box - } - } - - /** - * Enumerate all the boxes that we have chosen to test - */ - private static List listSelectedBoxes(Project project) { - String vagrantBoxes = project.getProperties().get('vagrant.boxes', 'sample') - switch (vagrantBoxes) { - case 'sample': - return SAMPLE - case 'linux-all': - return LINUX_BOXES - case 'windows-all': - return WINDOWS_BOXES - case 'all': - return ALL_BOXES - case '': - return [] - default: - return vagrantBoxes.split(',') - } - } - - private static void configurePackagingArchiveRepositories(Project project) { - RepositoryHandler repos = project.repositories - - repos.jcenter() // will have releases before 5.0.0 - - /* Setup a repository that tries to download from - https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext] - which should work for 5.0.0+. This isn't a real ivy repository but gradle - is fine with that */ - repos.ivy { - name "elasticsearch" - artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]" - } - } - - private static void createPackagingConfiguration(Project project) { - project.configurations.create(PACKAGING_CONFIGURATION) - - String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion"); - Version upgradeFromVersion - if (upgradeFromVersionRaw == null) { - String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0) - final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16) - final def indexCompatVersions = project.bwcVersions.indexCompatible - upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())] - } else { - upgradeFromVersion = Version.fromString(upgradeFromVersionRaw) - } - - List dependencies = new ArrayList<>() - DISTRIBUTIONS.each { - // Adds a dependency for the current version - dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) - } - - if (project.ext.bwc_tests_enabled) { - // The version of elasticsearch that we upgrade *from* - // we only add them as dependencies if the bwc tests are enabled, so we don't trigger builds otherwise - BwcVersions.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion) - if (unreleasedInfo != null) { - // handle snapshots pointing to bwc build - UPGRADE_FROM_ARCHIVES.each { - dependencies.add(project.dependencies.project( - path: "${unreleasedInfo.gradleProjectPath}", configuration: it)) - if (upgradeFromVersion.onOrAfter('6.3.0')) { - dependencies.add(project.dependencies.project( - path: "${unreleasedInfo.gradleProjectPath}", configuration: "oss-${it}")) - } - } - } else { - UPGRADE_FROM_ARCHIVES.each { - // The version of elasticsearch that we upgrade *from* - if (upgradeFromVersion.onOrAfter('7.0.0')) { - String arch = it == "rpm" ? "x86_64" : "amd64" - dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}-${arch}@${it}") - dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}-${arch}@${it}") - } else { - dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") - if (upgradeFromVersion.onOrAfter('6.3.0')) { - dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") - } - } - } - } - } else { - // Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests. - upgradeFromVersion = Version.fromString(project.version) - } - - for (Object dependency : dependencies) { - project.dependencies.add(PACKAGING_CONFIGURATION, dependency) - } - - project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion - } - - private static void createCleanTask(Project project) { - if (project.tasks.findByName('clean') == null) { - project.tasks.create('clean', Delete.class) { - description 'Clean the project build directory' - group 'Build' - delete project.buildDir - } - } - } - - private static void createStopTask(Project project) { - project.tasks.create('stop') { - description 'Stop any tasks from tests that still may be running' - group 'Verification' - } - } - - private static void createSmokeTestTask(Project project) { - project.tasks.create('vagrantSmokeTest') { - description 'Smoke test the specified vagrant boxes' - group 'Verification' - } - } - - private void createPrepareVagrantTestEnvTask(Project project) { - File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION) - - File archivesDir = new File(packagingDir, 'archives') - Copy copyPackagingArchives = project.tasks.create('copyPackagingArchives', Copy) { - into archivesDir - from project.configurations[PACKAGING_CONFIGURATION] - } - - File testsDir = new File(packagingDir, 'tests') - Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) { - into testsDir - from project.configurations[PACKAGING_TEST_CONFIGURATION] - } - - Task createLinuxRunnerScript = project.tasks.create('createLinuxRunnerScript', FileContentsTask) { - dependsOn copyPackagingTests, linuxGradleJdk - file "${testsDir}/run-tests.sh" - contents """\ - if [ "\$#" -eq 0 ]; then - test_args=( "${-> project.extensions.esvagrant.testClass}" ) - else - test_args=( "\$@" ) - fi - - "${-> convertLinuxPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}" - """ - } - Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) { - dependsOn copyPackagingTests, windowsGradleJdk - file "${testsDir}/run-tests.ps1" - // the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely - // a little trappy for those unfamiliar with powershell - contents """\ - try { - if (\$args.Count -eq 0) { - \$testArgs = @("${-> project.extensions.esvagrant.testClass}") - } else { - \$testArgs = \$args - } - & "${-> convertWindowsPath(project, windowsGradleJdk.toString()) }/bin/java" -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs - exit \$LASTEXITCODE - } catch { - # catch if we have a failure to even run the script at all above, equivalent to set -e, sort of - echo "\$_.Exception.Message" - exit 1 - } - """ - } - - Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) { - dependsOn copyPackagingArchives - file "${archivesDir}/version" - contents project.version - } - - Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) { - String version = project.extensions.esvagrant.upgradeFromVersion - if (project.bwcVersions.unreleased.contains(project.extensions.esvagrant.upgradeFromVersion)) { - version += "-SNAPSHOT" - } - dependsOn copyPackagingArchives - file "${archivesDir}/upgrade_from_version" - contents version - } - - Task createUpgradeIsOssFile = project.tasks.create('createUpgradeIsOssFile', FileContentsTask) { - dependsOn copyPackagingArchives - doFirst { - project.delete("${archivesDir}/upgrade_is_oss") - if (project.extensions.esvagrant.upgradeFromVersion.before('6.3.0')) { - throw new StopExecutionException("upgrade version is before 6.3.0") - } - } - file "${archivesDir}/upgrade_is_oss" - contents '' - } - - File batsDir = new File(packagingDir, BATS) - Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) { - into "${batsDir}/tests" - from { - "${project.extensions.esvagrant.batsDir}/tests" - } - } - - Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) { - into "${batsDir}/utils" - from { - "${project.extensions.esvagrant.batsDir}/utils" - } - } - - // Now we iterate over dependencies of the bats configuration. When a project dependency is found, - // we bring back its test files or test utils. - project.afterEvaluate { - project.configurations[PACKAGING_CONFIGURATION].dependencies - .findAll {it.targetConfiguration == PACKAGING_CONFIGURATION } - .each { d -> - if (d instanceof DefaultProjectDependency) { - DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d - Project externalBatsProject = externalBatsDependency.dependencyProject - String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir - - if (project.extensions.esvagrant.inheritTests) { - copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests")) - } - if (project.extensions.esvagrant.inheritTestUtils) { - copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils")) - } - } - } - } - - Task vagrantSetUpTask = project.tasks.create('setupPackagingTest') - vagrantSetUpTask.dependsOn( - 'vagrantCheckVersion', - copyPackagingArchives, - copyPackagingTests, - createLinuxRunnerScript, - createWindowsRunnerScript, - createVersionFile, - createUpgradeFromFile, - createUpgradeIsOssFile, - copyBatsTests, - copyBatsUtils - ) - } - - private static void createPackagingTestTask(Project project) { - project.tasks.create('packagingTest') { - group 'Verification' - description "Tests distribution installation on different platforms using vagrant. See TESTING.asciidoc for details." - dependsOn 'vagrantCheckVersion' - } - } - - private void createBoxListTasks(Project project) { - project.tasks.create('listAllBoxes') { - group 'Verification' - description 'List all vagrant boxes which can be tested by this plugin' - doLast { - println("All vagrant boxes supported by ${project.path}") - for (String box : ALL_BOXES) { - println(box) - } - } - dependsOn 'vagrantCheckVersion' - } - - project.tasks.create('listAvailableBoxes') { - group 'Verification' - description 'List all vagrant boxes which are available for testing' - doLast { - println("All vagrant boxes available to ${project.path}") - for (String box : availableBoxes) { - println(box) - } - } - dependsOn 'vagrantCheckVersion' - } - } - - private void createVagrantTasks(Project project) { - createCleanTask(project) - createStopTask(project) - createSmokeTestTask(project) - createPrepareVagrantTestEnvTask(project) - createPackagingTestTask(project) - createBoxListTasks(project) - } - - private void createVagrantBoxesTasks(Project project) { - assert project.extensions.esvagrant.boxes != null - - assert project.tasks.stop != null - Task stop = project.tasks.stop - - assert project.tasks.vagrantSmokeTest != null - Task vagrantSmokeTest = project.tasks.vagrantSmokeTest - - assert project.tasks.vagrantCheckVersion != null - Task vagrantCheckVersion = project.tasks.vagrantCheckVersion - - assert project.tasks.virtualboxCheckVersion != null - Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion - - assert project.tasks.setupPackagingTest != null - Task setupPackagingTest = project.tasks.setupPackagingTest - - assert project.tasks.packagingTest != null - Task packagingTest = project.tasks.packagingTest - - /* - * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD) - * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin. - */ - def vagrantEnvVars = [ - 'VAGRANT_CWD' : "${project.rootDir.absolutePath}", - 'VAGRANT_VAGRANTFILE' : 'Vagrantfile', - 'VAGRANT_PROJECT_DIR' : "${project.projectDir.absolutePath}" - ] - vagrantEnvVars.putAll(vagrantBoxEnvVars) - - // Each box gets it own set of tasks - for (String box : availableBoxes) { - String boxTask = box.capitalize().replace('-', '') - - // always add a halt task for all boxes, so clean makes sure they are all shutdown - Task halt = project.tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) { - command 'halt' - boxName box - environmentVars vagrantEnvVars - } - stop.dependsOn(halt) - - Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) { - command 'box' - subcommand 'update' - boxName box - environmentVars vagrantEnvVars - dependsOn vagrantCheckVersion, virtualboxCheckVersion - } - update.mustRunAfter(setupPackagingTest) - - /* - * Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag - * vagrant.destroy that defaults to true that can be used to control whether or not to destroy any test boxes before test - * execution. - */ - final String vagrantDestroyProperty = project.getProperties().get('vagrant.destroy', 'true') - boolean vagrantDestroy - if ("true".equals(vagrantDestroyProperty)) { - vagrantDestroy = true - } else if ("false".equals(vagrantDestroyProperty)) { - vagrantDestroy = false - } else { - throw new GradleException("[vagrant.destroy] must be [true] or [false] but was [" + vagrantDestroyProperty + "]") - } - /* - * Some versions of Vagrant will fail destroy if the box does not exist. Therefore we check if the box exists before attempting - * to destroy the box. - */ - final Task destroy = project.tasks.create("vagrant${boxTask}#destroy", LoggedExec) { - commandLine "bash", "-c", "vagrant status ${box} | grep -q \"${box}\\s\\+not created\" || vagrant destroy ${box} --force" - workingDir project.rootProject.rootDir - environment vagrantEnvVars - } - destroy.onlyIf { vagrantDestroy } - update.mustRunAfter(destroy) - - Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) { - command 'up' - boxName box - environmentVars vagrantEnvVars - /* We lock the provider to virtualbox because the Vagrantfile specifies - lots of boxes that only work properly in virtualbox. Virtualbox is - vagrant's default but its possible to change that default and folks do. - But the boxes that we use are unlikely to work properly with other - virtualization providers. Thus the lock. */ - args '--provision', '--provider', 'virtualbox' - /* It'd be possible to check if the box is already up here and output - SKIPPED but that would require running vagrant status which is slow! */ - dependsOn destroy, update - } - - Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) { - environment vagrantEnvVars - dependsOn up - finalizedBy halt - } - vagrantSmokeTest.dependsOn(smoke) - if (LINUX_BOXES.contains(box)) { - smoke.commandLine = ['vagrant', 'ssh', box, '--command', - "set -o pipefail && echo 'Hello from ${project.path}' | sed -ue 's/^/ ${box}: /'"] - } else { - smoke.commandLine = ['vagrant', 'winrm', box, '--command', - "Write-Host ' ${box}: Hello from ${project.path}'"] - } - - if (LINUX_BOXES.contains(box)) { - Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) { - remoteCommand BATS_TEST_COMMAND - boxName box - environmentVars vagrantEnvVars - dependsOn up, setupPackagingTest - finalizedBy halt - } - - TaskExecutionAdapter batsPackagingReproListener = createReproListener(project, batsPackagingTest.path) - batsPackagingTest.doFirst { - project.gradle.addListener(batsPackagingReproListener) - } - batsPackagingTest.doLast { - project.gradle.removeListener(batsPackagingReproListener) - } - if (project.extensions.esvagrant.boxes.contains(box)) { - // these tests are temporarily disabled for suse boxes while we debug an issue - // https://github.com/elastic/elasticsearch/issues/30295 - if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { - packagingTest.dependsOn(batsPackagingTest) - } - } - } - - Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) { - boxName box - environmentVars vagrantEnvVars - dependsOn up, setupPackagingTest - finalizedBy halt - } - - // todo remove this onlyIf after all packaging tests are consolidated - javaPackagingTest.onlyIf { - project.extensions.esvagrant.testClass != null - } - - if (LINUX_BOXES.contains(box)) { - javaPackagingTest.command = 'ssh' - javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"'] - } else { - // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however - // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user - // without administrator elevation), which we need to do for these tests. passing --elevated runs the session - // as a scheduled job locally on the vm as a true administrator to get around this limitation - // - // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225 - // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html - javaPackagingTest.command = 'winrm' - javaPackagingTest.args = ['--elevated', '--command', '& "$Env:PACKAGING_TESTS/run-tests.ps1"; exit $LASTEXITCODE'] - } - - TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path) - javaPackagingTest.doFirst { - project.gradle.addListener(javaPackagingReproListener) - } - javaPackagingTest.doLast { - project.gradle.removeListener(javaPackagingReproListener) - } - if (project.extensions.esvagrant.boxes.contains(box)) { - // these tests are temporarily disabled for suse boxes while we debug an issue - // https://github.com/elastic/elasticsearch/issues/30295 - if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { - packagingTest.dependsOn(javaPackagingTest) - } - } - } - } - - private static TaskExecutionAdapter createReproListener(Project project, String reproTaskPath) { - return new TaskExecutionAdapter() { - @Override - void afterExecute(Task task, TaskState state) { - final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew" - if (state.failure != null) { - println "REPRODUCE WITH: ${gradlew} \"${reproTaskPath}\" -Dtests.seed=${project.testSeed} " - } - } - } - } - - // convert the given path from an elasticsearch repo path to a VM path - private String convertLinuxPath(Project project, String path) { - return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path)); - } - private String convertWindowsPath(Project project, String path) { - return "C:\\elasticsearch\\" + project.rootDir.toPath().relativize(Paths.get(path)).toString().replace('/', '\\'); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 5a3a4a277dd..d8c693b77d2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -203,12 +203,16 @@ public class DistributionDownloadPlugin implements Plugin { String extension = distribution.getType().toString(); String classifier = "x86_64"; - if (distribution.getType() == Type.ARCHIVE) { + if (distribution.getVersion().before("7.0.0")) { + classifier = null; // no platform specific distros before 7.0 + } else if (distribution.getType() == Type.ARCHIVE) { extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; classifier = distribution.getPlatform() + "-" + classifier; + } else if (distribution.getType() == Type.DEB) { + classifier = "amd64"; } return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":") - + distribution.getVersion() + ":" + classifier + "@" + extension; + + distribution.getVersion() + (classifier == null ? "" : ":" + classifier) + "@" + extension; } private static Dependency projectDependency(Project project, String projectPath, String projectConfig) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 53089f9b3d7..815da77a154 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -20,9 +20,7 @@ package org.elasticsearch.gradle; import org.gradle.api.Buildable; -import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.file.FileTree; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskDependency; @@ -30,9 +28,8 @@ import org.gradle.api.tasks.TaskDependency; import java.io.File; import java.util.Iterator; import java.util.Locale; -import java.util.concurrent.Callable; -public class ElasticsearchDistribution implements Buildable { +public class ElasticsearchDistribution implements Buildable, Iterable { public enum Platform { LINUX, @@ -93,10 +90,6 @@ public class ElasticsearchDistribution implements Buildable { return configuration.getBuildDependencies(); } - public FileTree getFileTree(Project project) { - return project.fileTree((Callable) configuration::getSingleFile); - } - @Override public String toString() { return configuration.getSingleFile().toString(); @@ -190,6 +183,16 @@ public class ElasticsearchDistribution implements Buildable { return configuration.getBuildDependencies(); } + @Override + public Iterator iterator() { + return configuration.iterator(); + } + + // TODO: remove this when distro tests are per distribution + public Configuration getConfiguration() { + return configuration; + } + // internal, make this distribution's configuration unmodifiable void finalizeValues() { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java index aa26f398e8b..91516e26af9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java @@ -83,9 +83,13 @@ public class Jdk implements Buildable, Iterable { return configuration; } + public String getPath() { + return configuration.getSingleFile().toString(); + } + @Override public String toString() { - return configuration.getSingleFile().toString(); + return getPath(); } @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java index d4f0d9941da..7c57af701fe 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java @@ -48,13 +48,14 @@ import java.util.regex.Matcher; public class JdkDownloadPlugin implements Plugin { private static final String REPO_NAME_PREFIX = "jdk_repo_"; + private static final String CONTAINER_NAME = "jdks"; @Override public void apply(Project project) { NamedDomainObjectContainer jdksContainer = project.container(Jdk.class, name -> new Jdk(name, project) ); - project.getExtensions().add("jdks", jdksContainer); + project.getExtensions().add(CONTAINER_NAME, jdksContainer); project.afterEvaluate(p -> { for (Jdk jdk : jdksContainer) { @@ -82,6 +83,11 @@ public class JdkDownloadPlugin implements Plugin { }); } + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(CONTAINER_NAME); + } + private static void setupRootJdkDownload(Project rootProject, String platform, String version) { String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version; // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java similarity index 52% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy rename to buildSrc/src/main/java/org/elasticsearch/gradle/Util.java index 1d85d8584bb..ffc5ce353d2 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java @@ -16,30 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.gradle.vagrant -import org.gradle.api.tasks.Input +package org.elasticsearch.gradle; -/** - * Runs bats over vagrant. Pretty much like running it using Exec but with a - * nicer output formatter. - */ -public class BatsOverVagrantTask extends VagrantCommandTask { +import org.gradle.api.GradleException; - @Input - Object remoteCommand +public class Util { - BatsOverVagrantTask() { - command = 'ssh' - } - - void setRemoteCommand(Object remoteCommand) { - this.remoteCommand = Objects.requireNonNull(remoteCommand) - setArgs((Iterable) ['--command', remoteCommand]) - } - - @Override - protected OutputStream createLoggerOutputStream() { - return new TapLoggerOutputStream(logger, getProgressLoggerFactory().newOperation(boxName).setDescription(boxName)); + public static boolean getBooleanProperty(String property, boolean defaultValue) { + String propertyValue = System.getProperty(property); + if (propertyValue == null) { + return defaultValue; + } + if ("true".equals(propertyValue)) { + return true; + } else if ("false".equals(propertyValue)) { + return false; + } else { + throw new GradleException("Sysprop [" + property + "] must be [true] or [false] but was [" + propertyValue + "]"); + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java new file mode 100644 index 00000000000..c3d79f44ae3 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.Directory; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class BatsTestTask extends DefaultTask { + + private Directory testsDir; + private Directory utilsDir; + private Directory archivesDir; + private String packageName; + + @InputDirectory + public Directory getTestsDir() { + return testsDir; + } + + public void setTestsDir(Directory testsDir) { + this.testsDir = testsDir; + } + + @InputDirectory + public Directory getUtilsDir() { + return utilsDir; + } + + public void setUtilsDir(Directory utilsDir) { + this.utilsDir = utilsDir; + } + + @InputDirectory + public Directory getArchivesDir() { + return archivesDir; + } + + public void setArchivesDir(Directory archivesDir) { + this.archivesDir = archivesDir; + } + + @Input + public String getPackageName() { + return packageName; + } + + public void setPackageName(String packageName) { + this.packageName = packageName; + } + + @TaskAction + public void runBats() { + List command = new ArrayList<>(); + command.add("bats"); + command.add("--tap"); + command.addAll(testsDir.getAsFileTree().getFiles().stream() + .filter(f -> f.getName().endsWith(".bats")) + .sorted().collect(Collectors.toList())); + getProject().exec(spec -> { + spec.setWorkingDir(archivesDir.getAsFile()); + spec.environment(System.getenv()); + spec.environment("BATS_TESTS", testsDir.getAsFile().toString()); + spec.environment("BATS_UTILS", utilsDir.getAsFile().toString()); + spec.environment("PACKAGE_NAME", packageName); + spec.setCommandLine(command); + }); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java new file mode 100644 index 00000000000..cfb960e4e36 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.vagrant.VagrantShellTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.options.Option; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +/** + * Run a gradle task of the current build, within the configured vagrant VM. + */ +public class GradleDistroTestTask extends VagrantShellTask { + + private String taskName; + private String testClass; + private List extraArgs = new ArrayList<>(); + + public void setTaskName(String taskName) { + this.taskName = taskName; + } + + @Input + public String getTaskName() { + return taskName; + } + + @Option(option = "tests", description = "Sets test class or method name to be included, '*' is supported.") + public void setTestClass(String testClass) { + this.testClass = testClass; + } + + @Input + public List getExtraArgs() { + return extraArgs; + } + + public void extraArg(String arg) { + this.extraArgs.add(arg); + } + + @Override + protected List getWindowsScript() { + return getScript(true); + } + + @Override + protected List getLinuxScript() { + return getScript(false); + } + + private List getScript(boolean isWindows) { + String cacheDir = getProject().getBuildDir() + "/gradle-cache"; + StringBuilder line = new StringBuilder(); + line.append(isWindows ? "& .\\gradlew " : "./gradlew "); + line.append(taskName); + line.append(" --project-cache-dir "); + line.append(isWindows ? convertWindowsPath(getProject(), cacheDir) : convertLinuxPath(getProject(), cacheDir)); + line.append(" -S"); + line.append(" -D'org.gradle.logging.level'=" + getProject().getGradle().getStartParameter().getLogLevel()); + if (testClass != null) { + line.append(" --tests="); + line.append(testClass); + } + extraArgs.stream().map(s -> " " + s).forEach(line::append); + return Collections.singletonList(line.toString()); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java index 70926c8982d..760e5f60f1c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java @@ -52,6 +52,14 @@ public abstract class Boilerplate { } + public static TaskProvider maybeRegister(TaskContainer tasks, String name, Class clazz, Action action) { + try { + return tasks.named(name, clazz); + } catch (UnknownTaskException e) { + return tasks.register(name, clazz, action); + } + } + public static void maybeConfigure(TaskContainer tasks, String name, Action config) { TaskProvider task; try { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java similarity index 69% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java index 353b2687ad1..8db4e704fb4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java @@ -19,12 +19,10 @@ package org.elasticsearch.gradle.vagrant; -import org.elasticsearch.gradle.LoggingOutputStream; -import org.gradle.api.GradleScriptException; import org.gradle.api.logging.Logger; -import org.gradle.internal.logging.progress.ProgressLogger; import java.util.Formatter; +import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,49 +39,43 @@ import java.util.regex.Pattern; * There is a Tap4j project but we can't use it because it wants to parse the * entire TAP stream at once and won't parse it stream-wise. */ -public class TapLoggerOutputStream extends LoggingOutputStream { +public class BatsProgressLogger implements UnaryOperator { private static final Pattern lineRegex = Pattern.compile("(?ok|not ok) \\d+(? # skip (?\\(.+\\))?)? \\[(?.+)\\] (?.+)"); + private static final Pattern startRegex = Pattern.compile("1..(\\d+)"); private final Logger logger; - private final ProgressLogger progressLogger; - private boolean isStarted = false; private int testsCompleted = 0; private int testsFailed = 0; private int testsSkipped = 0; private Integer testCount; private String countsFormat; - TapLoggerOutputStream(Logger logger, ProgressLogger progressLogger) { + public BatsProgressLogger(Logger logger) { this.logger = logger; - this.progressLogger = progressLogger; } @Override - public void logLine(String line) { - if (isStarted == false) { - progressLogger.started("started"); - isStarted = true; - } + public String apply(String line) { if (testCount == null) { - try { - int lastDot = line.lastIndexOf('.'); - testCount = Integer.parseInt(line.substring(lastDot + 1)); - int length = String.valueOf(testCount).length(); - String count = "%0" + length + "d"; - countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; - return; - } catch (Exception e) { - throw new GradleScriptException("Error parsing first line of TAP stream!!", e); + Matcher m = startRegex.matcher(line); + if (m.matches() == false) { + // haven't reached start of bats test yet, pass through whatever we see + return line; } + testCount = Integer.parseInt(m.group(1)); + int length = String.valueOf(testCount).length(); + String count = "%0" + length + "d"; + countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; + return null; } Matcher m = lineRegex.matcher(line); if (m.matches() == false) { /* These might be failure report lines or comments or whatever. Its hard to tell and it doesn't matter. */ logger.warn(line); - return; + return null; } boolean skipped = m.group("skip") != null; boolean success = skipped == false && m.group("status").equals("ok"); @@ -104,15 +96,9 @@ public class TapLoggerOutputStream extends LoggingOutputStream { } String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString(); - progressLogger.progress("BATS " + counts + ", " + status + " [" + suiteName + "] " + testName); if (success == false) { logger.warn(line); } - } - - @Override - public void close() { - flush(); - progressLogger.completed(); + return "BATS " + counts + ", " + status + " [" + suiteName + "] " + testName; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java new file mode 100644 index 00000000000..f77fe982f74 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.elasticsearch.gradle.ReaperPlugin; +import org.elasticsearch.gradle.ReaperService; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.execution.TaskActionListener; +import org.gradle.api.execution.TaskExecutionListener; +import org.gradle.api.tasks.TaskState; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class VagrantBasePlugin implements Plugin { + + @Override + public void apply(Project project) { + project.getRootProject().getPluginManager().apply(VagrantSetupCheckerPlugin.class); + project.getRootProject().getPluginManager().apply(VagrantManagerPlugin.class); + project.getRootProject().getPluginManager().apply(ReaperPlugin.class); + + ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class); + VagrantExtension extension = project.getExtensions().create("vagrant", VagrantExtension.class, project); + VagrantMachine service = project.getExtensions().create("vagrantService", VagrantMachine.class, project, extension, reaper); + + project.getGradle().getTaskGraph().whenReady(graph -> + service.refs = graph.getAllTasks().stream() + .filter(t -> t instanceof VagrantShellTask) + .filter(t -> t.getProject() == project) + .count()); + } + + + /** + * Check vagrant and virtualbox versions, if any vagrant test tasks will be run. + */ + static class VagrantSetupCheckerPlugin implements Plugin { + + private static final Pattern VAGRANT_VERSION = Pattern.compile("Vagrant (\\d+\\.\\d+\\.\\d+)"); + private static final Pattern VIRTUAL_BOX_VERSION = Pattern.compile("(\\d+\\.\\d+)"); + + @Override + public void apply(Project project) { + if (project != project.getRootProject()) { + throw new IllegalArgumentException("VagrantSetupCheckerPlugin can only be applied to the root project of a build"); + } + + project.getGradle().getTaskGraph().whenReady(graph -> { + boolean needsVagrant = graph.getAllTasks().stream().anyMatch(t -> t instanceof VagrantShellTask); + if (needsVagrant) { + checkVersion(project, "vagrant", VAGRANT_VERSION, 1, 8, 6); + checkVersion(project, "vboxmanage", VIRTUAL_BOX_VERSION, 5, 1); + } + }); + } + + void checkVersion(Project project, String tool, Pattern versionRegex, int... minVersion) { + ByteArrayOutputStream pipe = new ByteArrayOutputStream(); + project.exec(spec -> { + spec.setCommandLine(tool, "--version"); + spec.setStandardOutput(pipe); + }); + String output = pipe.toString(StandardCharsets.UTF_8).trim(); + Matcher matcher = versionRegex.matcher(output); + if (matcher.find() == false) { + throw new IllegalStateException(tool + + " version output [" + output + "] did not match regex [" + versionRegex.pattern() + "]"); + } + + String version = matcher.group(1); + List versionParts = Stream.of(version.split("\\.")).map(Integer::parseInt).collect(Collectors.toList()); + for (int i = 0; i < minVersion.length; ++i) { + int found = versionParts.get(i); + if (found > minVersion[i]) { + break; // most significant version is good + } else if (found < minVersion[i]) { + throw new IllegalStateException("Unsupported version of " + tool + ". Found [" + version + "], expected [" + + Stream.of(minVersion).map(String::valueOf).collect(Collectors.joining(".")) + "+"); + } // else equal, so check next element + } + } + } + + /** + * Adds global hooks to manage destroying, starting and updating VMs. + */ + static class VagrantManagerPlugin implements Plugin, TaskActionListener, TaskExecutionListener { + + @Override + public void apply(Project project) { + if (project != project.getRootProject()) { + throw new IllegalArgumentException("VagrantManagerPlugin can only be applied to the root project of a build"); + } + project.getGradle().addListener(this); + } + + private void callIfVagrantTask(Task task, Consumer method) { + if (task instanceof VagrantShellTask) { + VagrantMachine service = task.getProject().getExtensions().getByType(VagrantMachine.class); + method.accept(service); + } + } + + @Override + public void beforeExecute(Task task) { /* nothing to do */} + + @Override + public void afterActions(Task task) { /* nothing to do */ } + + @Override + public void beforeActions(Task task) { + callIfVagrantTask(task, VagrantMachine::maybeStartVM); + } + + @Override + public void afterExecute(Task task, TaskState state) { + callIfVagrantTask(task, service -> service.maybeStopVM(state.getFailure() != null)); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java new file mode 100644 index 00000000000..10ec03f7f10 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.gradle.api.Project; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; + +import java.io.File; +import java.util.Map; + +public class VagrantExtension { + + private final Property box; + private final MapProperty hostEnv; + private final MapProperty vmEnv; + private final RegularFileProperty vagrantfile; + private boolean isWindowsVM; + + public VagrantExtension(Project project) { + this.box = project.getObjects().property(String.class); + this.hostEnv = project.getObjects().mapProperty(String.class, Object.class); + this.vmEnv = project.getObjects().mapProperty(String.class, Object.class); + this.vagrantfile = project.getObjects().fileProperty(); + this.vagrantfile.convention(project.getRootProject().getLayout().getProjectDirectory().file("Vagrantfile")); + this.isWindowsVM = false; + } + + @Input + public String getBox() { + return box.get(); + } + + public void setBox(String box) { + // TODO: should verify this against the Vagrantfile, but would need to do so in afterEvaluate once vagrantfile is unmodifiable + this.box.set(box); + } + + @Input + public Map getHostEnv() { + return hostEnv.get(); + } + + public void hostEnv(String name, Object value) { + hostEnv.put(name, value); + } + + @Input + public Map getVmEnv() { + return vmEnv.get(); + } + + public void vmEnv(String name, Object value) { + vmEnv.put(name, value); + } + + @Input + public boolean isWindowsVM() { + return isWindowsVM; + } + + public void setIsWindowsVM(boolean isWindowsVM) { + this.isWindowsVM = isWindowsVM; + } + + @Input + public File getVagrantfile() { + return this.vagrantfile.get().getAsFile(); + } + + public void setVagrantfile(File file) { + vagrantfile.set(file); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java new file mode 100644 index 00000000000..aa89658d9a9 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java @@ -0,0 +1,210 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.apache.commons.io.output.TeeOutputStream; +import org.elasticsearch.gradle.LoggedExec; +import org.elasticsearch.gradle.LoggingOutputStream; +import org.elasticsearch.gradle.ReaperService; +import org.elasticsearch.gradle.Util; +import org.gradle.api.Action; +import org.gradle.api.Project; +import org.gradle.internal.logging.progress.ProgressLogger; +import org.gradle.internal.logging.progress.ProgressLoggerFactory; + +import javax.inject.Inject; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Objects; +import java.util.function.UnaryOperator; + +/** + * An helper to manage a vagrant box. + * + * This is created alongside a {@link VagrantExtension} for a project to manage starting and + * stopping a single vagrant box. + */ +public class VagrantMachine { + + private final Project project; + private final VagrantExtension extension; + private final ReaperService reaper; + // pkg private so plugin can set this after construction + long refs; + private boolean isVMStarted = false; + + public VagrantMachine(Project project, VagrantExtension extension, ReaperService reaper) { + this.project = project; + this.extension = extension; + this.reaper = reaper; + } + + @Inject + protected ProgressLoggerFactory getProgressLoggerFactory() { + throw new UnsupportedOperationException(); + } + + public void execute(Action action) { + VagrantExecSpec vagrantSpec = new VagrantExecSpec(); + action.execute(vagrantSpec); + + Objects.requireNonNull(vagrantSpec.command); + + LoggedExec.exec(project, execSpec -> { + execSpec.setExecutable("vagrant"); + File vagrantfile = extension.getVagrantfile(); + execSpec.setEnvironment(System.getenv()); // pass through env + execSpec.environment("VAGRANT_CWD", vagrantfile.getParentFile().toString()); + execSpec.environment("VAGRANT_VAGRANTFILE", vagrantfile.getName()); + execSpec.environment("VAGRANT_LOG", "debug"); + extension.getHostEnv().forEach(execSpec::environment); + + execSpec.args(vagrantSpec.command); + if (vagrantSpec.subcommand != null) { + execSpec.args(vagrantSpec.subcommand); + } + execSpec.args(extension.getBox()); + if (vagrantSpec.args != null) { + execSpec.args(Arrays.asList(vagrantSpec.args)); + } + + UnaryOperator progressHandler = vagrantSpec.progressHandler; + if (progressHandler == null) { + progressHandler = new VagrantProgressLogger("==> " + extension.getBox() + ": "); + } + OutputStream output = execSpec.getStandardOutput(); + // output from vagrant needs to be manually curated because --machine-readable isn't actually "readable" + OutputStream progressStream = new ProgressOutputStream(vagrantSpec.command, progressHandler); + execSpec.setStandardOutput(new TeeOutputStream(output, progressStream)); + }); + } + + // start the configuration VM if it hasn't been started yet + void maybeStartVM() { + if (isVMStarted) { + return; + } + + execute(spec -> { + spec.setCommand("box"); + spec.setSubcommand("update"); + }); + + // Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag that defaults + // to true that can be used to control whether or not to destroy any test boxes before test execution. + boolean destroyVM = Util.getBooleanProperty("vagrant.destroy", true); + if (destroyVM) { + execute(spec -> { + spec.setCommand("destroy"); + spec.setArgs("--force"); + }); + } + + // register box to be shutdown if gradle dies + reaper.registerCommand(extension.getBox(), "vagrant", "halt", "-f", extension.getBox()); + + // We lock the provider to virtualbox because the Vagrantfile specifies lots of boxes that only work + // properly in virtualbox. Virtualbox is vagrant's default but its possible to change that default and folks do. + execute(spec -> { + spec.setCommand("up"); + spec.setArgs("--provision", "--provider", "virtualbox"); + }); + isVMStarted = true; + } + + // stops the VM if refs are down to 0, or force was called + void maybeStopVM(boolean force) { + assert refs >= 1; + this.refs--; + if ((refs == 0 || force) && isVMStarted) { + execute(spec -> spec.setCommand("halt")); + reaper.unregister(extension.getBox()); + } + } + + // convert the given path from an elasticsearch repo path to a VM path + public static String convertLinuxPath(Project project, String path) { + return "/elasticsearch/" + project.getRootDir().toPath().relativize(Paths.get(path)); + } + + public static String convertWindowsPath(Project project, String path) { + return "C:\\elasticsearch\\" + project.getRootDir().toPath().relativize(Paths.get(path)).toString().replace('/', '\\'); + } + + public static class VagrantExecSpec { + private String command; + private String subcommand; + private String[] args; + private UnaryOperator progressHandler; + + private VagrantExecSpec() {} + + public void setCommand(String command) { + this.command = command; + } + + public void setSubcommand(String subcommand) { + this.subcommand = subcommand; + } + + public void setArgs(String... args) { + this.args = args; + } + + /** + * A function to translate output from the vagrant command execution to the progress line. + * + * The function takes the current line of output from vagrant, and returns a new + * progress line, or {@code null} if there is no update. + */ + public void setProgressHandler(UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + } + } + + private class ProgressOutputStream extends LoggingOutputStream { + + private ProgressLogger progressLogger; + private UnaryOperator progressHandler; + + ProgressOutputStream(String command, UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + this.progressLogger = getProgressLoggerFactory().newOperation("vagrant"); + progressLogger.start(extension.getBox() + "> " + command, "hello"); + } + + @Override + protected void logLine(String line) { + String progress = progressHandler.apply(line); + if (progress != null) { + progressLogger.progress(progress); + } + System.out.println(line); + } + + @Override + public void close() { + progressLogger.completed(); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java similarity index 73% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java index 2e4a6123556..d041ebbda92 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java @@ -19,30 +19,23 @@ package org.elasticsearch.gradle.vagrant; -import org.elasticsearch.gradle.LoggingOutputStream; -import org.gradle.internal.logging.progress.ProgressLogger; +import java.util.function.UnaryOperator; + +public class VagrantProgressLogger implements UnaryOperator { -public class VagrantLoggerOutputStream extends LoggingOutputStream { private static final String HEADING_PREFIX = "==> "; - private final ProgressLogger progressLogger; private final String squashedPrefix; - private boolean isStarted = false; private String lastLine = ""; - private boolean inProgressReport = false; private String heading = ""; + private boolean inProgressReport = false; - VagrantLoggerOutputStream(ProgressLogger progressLogger, String squashedPrefix) { - this.progressLogger = progressLogger; + public VagrantProgressLogger(String squashedPrefix) { this.squashedPrefix = squashedPrefix; } @Override - protected void logLine(String line) { - if (isStarted == false) { - progressLogger.started("started"); - isStarted = true; - } + public String apply(String line) { if (line.startsWith("\r\u001b")) { /* We don't want to try to be a full terminal emulator but we want to keep the escape sequences from leaking and catch _some_ of the @@ -51,7 +44,7 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream { if ("[K".equals(line)) { inProgressReport = true; } - return; + return null; } if (line.startsWith(squashedPrefix)) { line = line.substring(squashedPrefix.length()); @@ -67,14 +60,8 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream { inProgressReport = false; line = lastLine + line; } else { - return; + return null; } - progressLogger.progress(line); - } - - @Override - public void close() { - flush(); - progressLogger.completed(); + return line; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java new file mode 100644 index 00000000000..12561712ccc --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.TaskAction; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +/** + * A shell script to run within a vagrant VM. + * + * The script is run as root within the VM. + */ +public abstract class VagrantShellTask extends DefaultTask { + + private final VagrantExtension extension; + private final VagrantMachine service; + private UnaryOperator progressHandler = UnaryOperator.identity(); + + public VagrantShellTask() { + extension = getProject().getExtensions().findByType(VagrantExtension.class); + if (extension == null) { + throw new IllegalStateException("elasticsearch.vagrant-base must be applied to create " + getClass().getName()); + } + service = getProject().getExtensions().getByType(VagrantMachine.class); + } + + @Input + protected abstract List getWindowsScript(); + + @Input + protected abstract List getLinuxScript(); + + @Input + public UnaryOperator getProgressHandler() { + return progressHandler; + } + + public void setProgressHandler(UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + } + + @TaskAction + public void runScript() { + String rootDir = getProject().getRootDir().toString(); + if (extension.isWindowsVM()) { + service.execute(spec -> { + spec.setCommand("winrm"); + + List script = new ArrayList<>(); + script.add("try {"); + script.add("cd " + convertWindowsPath(getProject(), rootDir)); + extension.getVmEnv().forEach((k, v) -> script.add("$Env:" + k + " = \"" + v + "\"")); + script.addAll(getWindowsScript().stream().map(s -> " " + s).collect(Collectors.toList())); + script.addAll(Arrays.asList( + " exit $LASTEXITCODE", + "} catch {", + // catch if we have a failure to even run the script at all above, equivalent to set -e, sort of + " echo $_.Exception.Message", + " exit 1", + "}")); + spec.setArgs("--elevated", "--command", String.join("\n", script)); + spec.setProgressHandler(progressHandler); + }); + } else { + service.execute(spec -> { + spec.setCommand("ssh"); + + List script = new ArrayList<>(); + script.add("sudo bash -c '"); // start inline bash script + script.add("pwd"); + script.add("cd " + convertLinuxPath(getProject(), rootDir)); + extension.getVmEnv().forEach((k, v) -> script.add("export " + k + "=" + v)); + script.addAll(getLinuxScript()); + script.add("'"); // end inline bash script + spec.setArgs("--command", String.join("\n", script)); + spec.setProgressHandler(progressHandler); + }); + } + } + + +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties new file mode 100644 index 00000000000..90b2914e452 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.DistroTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties deleted file mode 100644 index 844310fa9d7..00000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.vagrant.VagrantTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties deleted file mode 100644 index 73a3f412349..00000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin \ No newline at end of file diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java index 873cdc7d7bf..343ead0bdf6 100644 --- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java @@ -16,10 +16,13 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Pattern; /** * A wrapper around gradle's Exec task to capture output and log on error. @@ -98,6 +101,8 @@ public class LoggedExec extends Exec { return genericExec(project, project::javaexec, action); } + private static final Pattern NEWLINE = Pattern.compile(System.lineSeparator()); + private static ExecResult genericExec( Project project, Function,ExecResult> function, @@ -107,19 +112,20 @@ public class LoggedExec extends Exec { return function.apply(action); } ByteArrayOutputStream output = new ByteArrayOutputStream(); - ByteArrayOutputStream error = new ByteArrayOutputStream(); try { return function.apply(spec -> { spec.setStandardOutput(output); - spec.setErrorOutput(error); + spec.setErrorOutput(output); action.execute(spec); + try { + output.write(("Output for " + spec.getExecutable() + ":").getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } }); } catch (Exception e) { try { - project.getLogger().error("Standard output:"); - project.getLogger().error(output.toString("UTF-8")); - project.getLogger().error("Standard error:"); - project.getLogger().error(error.toString("UTF-8")); + NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> project.getLogger().error("| " + s)); } catch (UnsupportedEncodingException ue) { throw new GradleException("Failed to read exec output", ue); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java index d83de5f2173..5f728392313 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java @@ -63,12 +63,12 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase { Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) { filebytes = stream.readAllBytes(); } - String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip"; + String urlPath = "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip"; wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200))); wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes))); wireMock.start(); - assertExtractedDistro("1.0.0", "archive", "windows", null, null, + assertExtractedDistro("7.0.0", "archive", "windows", null, null, "tests.download_service", wireMock.baseUrl()); } catch (Exception e) { // for debugging diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/10_basic.bats b/qa/vagrant/bats/default/10_basic.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/10_basic.bats rename to qa/vagrant/bats/default/10_basic.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats b/qa/vagrant/bats/default/20_tar_bootstrap_password.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats rename to qa/vagrant/bats/default/20_tar_bootstrap_password.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats b/qa/vagrant/bats/default/25_package_bootstrap_password.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats rename to qa/vagrant/bats/default/25_package_bootstrap_password.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats b/qa/vagrant/bats/default/30_tar_setup_passwords.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats rename to qa/vagrant/bats/default/30_tar_setup_passwords.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats b/qa/vagrant/bats/default/35_package_setup_passwords.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats rename to qa/vagrant/bats/default/35_package_setup_passwords.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats b/qa/vagrant/bats/default/40_tar_certgen.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats rename to qa/vagrant/bats/default/40_tar_certgen.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats b/qa/vagrant/bats/default/45_package_certgen.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats rename to qa/vagrant/bats/default/45_package_certgen.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash b/qa/vagrant/bats/default/bootstrap_password.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash rename to qa/vagrant/bats/default/bootstrap_password.bash diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/qa/vagrant/bats/default/certgen.bash similarity index 98% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash rename to qa/vagrant/bats/default/certgen.bash index 13aed28b4c1..b52e286742c 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash +++ b/qa/vagrant/bats/default/certgen.bash @@ -15,7 +15,6 @@ instances="/tmp/instances.yml" certificates="/tmp/certificates.zip" setup() { - export PACKAGE_NAME="elasticsearch" if [ $BATS_TEST_NUMBER == 1 ]; then clean_before_test fi @@ -176,6 +175,7 @@ NEW_PASS } @test "[$GROUP] create instances file" { + rm -f /tmp/instances.yml run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE" cat > /tmp/instances.yml <<- EOF instances: @@ -426,3 +426,8 @@ DATA_SETTINGS false } } + +@test "[$GROUP] remove Elasticsearch" { + # NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed + clean_before_test +} diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash b/qa/vagrant/bats/default/setup_passwords.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash rename to qa/vagrant/bats/default/setup_passwords.bash diff --git a/qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats b/qa/vagrant/bats/oss/25_tar_plugins.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats rename to qa/vagrant/bats/oss/25_tar_plugins.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats b/qa/vagrant/bats/oss/50_modules_and_plugins.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats rename to qa/vagrant/bats/oss/50_modules_and_plugins.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats b/qa/vagrant/bats/oss/70_sysv_initd.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats rename to qa/vagrant/bats/oss/70_sysv_initd.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats b/qa/vagrant/bats/oss/80_upgrade.bats similarity index 98% rename from qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats rename to qa/vagrant/bats/oss/80_upgrade.bats index 697e6456d1f..0c80751a58f 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats +++ b/qa/vagrant/bats/oss/80_upgrade.bats @@ -122,6 +122,7 @@ setup() { curl -s localhost:9200/library2/book/1?pretty | grep Darkness } -@test "[UPGRADE] stop version under test" { +@test "[UPGRADE] cleanup version under test" { stop_elasticsearch_service + clean_before_test } diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/bats/oss/module_and_plugin_test_cases.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash rename to qa/vagrant/bats/oss/module_and_plugin_test_cases.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/modules.bash b/qa/vagrant/bats/utils/modules.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/modules.bash rename to qa/vagrant/bats/utils/modules.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/bats/utils/packages.bash similarity index 99% rename from qa/vagrant/src/test/resources/packaging/utils/packages.bash rename to qa/vagrant/bats/utils/packages.bash index 5df432c35b3..2da3a02c543 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/bats/utils/packages.bash @@ -48,7 +48,7 @@ export_elasticsearch_paths() { export ESDATA="/var/lib/elasticsearch" export ESLOG="/var/log/elasticsearch" export ESENVFILE=$(env_file) - export PACKAGE_NAME=${PACKAGE_NAME:-"elasticsearch-oss"} + export PACKAGE_NAME } diff --git a/qa/vagrant/src/test/resources/packaging/utils/plugins.bash b/qa/vagrant/bats/utils/plugins.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/plugins.bash rename to qa/vagrant/bats/utils/plugins.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/bats/utils/tar.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/tar.bash rename to qa/vagrant/bats/utils/tar.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/bats/utils/utils.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/utils.bash rename to qa/vagrant/bats/utils/utils.bash diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash b/qa/vagrant/bats/utils/xpack.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash rename to qa/vagrant/bats/utils/xpack.bash diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index f5cfcdda03c..7cbc475e933 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -18,10 +18,7 @@ */ plugins { - id 'java' id 'elasticsearch.build' - id 'elasticsearch.vagrantsupport' - id 'elasticsearch.vagrant' } dependencies { @@ -36,34 +33,14 @@ dependencies { compile "commons-logging:commons-logging:${versions.commonslogging}" compile project(':libs:elasticsearch-core') - - // pulls in the jar built by this project and its dependencies - packagingTest project(path: project.path, configuration: 'runtime') } -List plugins = [] -for (Project subproj : project.rootProject.subprojects) { - if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) { - // add plugin as a dep - dependencies { - packaging project(path: "${subproj.path}", configuration: 'zip') - } - plugins.add(subproj.name) - } -} -plugins = plugins.toSorted() +configurations.create('testClasses') -setupPackagingTest { - doFirst { - File expectedPlugins = file('build/plugins/expected') - expectedPlugins.parentFile.mkdirs() - expectedPlugins.setText(plugins.join('\n'), 'UTF-8') - } -} - -esvagrant { - testClass 'org.elasticsearch.packaging.PackagingTests' -} +String classesDir = project.file(project.sourceSets.main.output.classesDirs.singleFile).toString() +artifacts.add('testClasses', project.layout.projectDirectory.dir(classesDir)) { + builtBy tasks.named('testClasses') +} forbiddenApisMain { replaceSignatureFiles 'jdk-signatures' @@ -91,3 +68,61 @@ tasks.thirdPartyAudit.ignoreMissingClasses ( 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener' ) + +boolean sample = project.properties.get('vagrant.boxes') != 'all' + +subprojects { Project platformProject -> + apply plugin: 'elasticsearch.distro-test' + apply plugin: 'java' + + configurations.create('testClasses') + dependencies { + testClasses project(path: ':qa:vagrant', configuration: 'testClasses') + testRuntime project(path: ':qa:vagrant', configuration: 'runtime') + } + + tasks.named('destructiveDistroTest') { + testClassesDirs += project.files(configurations.testClasses.singleFile) + } + + // TODO: remove this property lookup once CI is switched to use an explicit task for the sample tests + boolean allBoxes = project.properties.get('vagrant.boxes', '') == 'all' + if (allBoxes || ['centos-7', 'ubuntu-1604'].contains(platformProject.name)) { + tasks.register('packagingTest') { + dependsOn 'distroTest', 'batsTest.oss', 'batsTest.default' + } + } + + vagrant { + hostEnv 'VAGRANT_PROJECT_DIR', platformProject.projectDir.absolutePath + } + +} + +configurations { + allPlugins +} + +List plugins = [] +for (Project subproj : project.rootProject.subprojects) { + if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) { + // add plugin as a dep + dependencies { + allPlugins project(path: "${subproj.path}", configuration: 'zip') + } + plugins.add(subproj.name) + } +} +plugins = plugins.toSorted() + +copyPackagingArchives { + from configurations.allPlugins + doLast { + // TODO: this was copied from the old way bats tests get the plugins list. we should pass + // this in differently when converting to java tests + File expectedPlugins = file('build/plugins/expected') + expectedPlugins.parentFile.mkdirs() + expectedPlugins.setText(plugins.join('\n'), 'UTF-8') + } +} + diff --git a/qa/vagrant/centos-6/build.gradle b/qa/vagrant/centos-6/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/centos-7/build.gradle b/qa/vagrant/centos-7/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/debian-8/build.gradle b/qa/vagrant/debian-8/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/debian-9/build.gradle b/qa/vagrant/debian-9/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/fedora-28/build.gradle b/qa/vagrant/fedora-28/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/fedora-29/build.gradle b/qa/vagrant/fedora-29/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/oel-6/build.gradle b/qa/vagrant/oel-6/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/oel-7/build.gradle b/qa/vagrant/oel-7/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/opensuse-42/build.gradle b/qa/vagrant/opensuse-42/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/sles-12/build.gradle b/qa/vagrant/sles-12/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java deleted file mode 100644 index 06c978b823a..00000000000 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.packaging; - -import org.elasticsearch.packaging.test.DefaultDebBasicTests; -import org.elasticsearch.packaging.test.DefaultDebPreservationTests; -import org.elasticsearch.packaging.test.DefaultLinuxTarTests; -import org.elasticsearch.packaging.test.DefaultNoJdkDebBasicTests; -import org.elasticsearch.packaging.test.DefaultNoJdkLinuxTarTests; -import org.elasticsearch.packaging.test.DefaultNoJdkRpmBasicTests; -import org.elasticsearch.packaging.test.DefaultNoJdkWindowsZipTests; -import org.elasticsearch.packaging.test.DefaultRpmBasicTests; -import org.elasticsearch.packaging.test.DefaultRpmPreservationTests; -import org.elasticsearch.packaging.test.DefaultWindowsServiceTests; -import org.elasticsearch.packaging.test.DefaultWindowsZipTests; -import org.elasticsearch.packaging.test.OssDebBasicTests; -import org.elasticsearch.packaging.test.OssDebPreservationTests; -import org.elasticsearch.packaging.test.OssLinuxTarTests; -import org.elasticsearch.packaging.test.OssNoJdkDebBasicTests; -import org.elasticsearch.packaging.test.OssNoJdkLinuxTarTests; -import org.elasticsearch.packaging.test.OssNoJdkRpmBasicTests; -import org.elasticsearch.packaging.test.OssNoJdkWindowsZipTests; -import org.elasticsearch.packaging.test.OssRpmBasicTests; -import org.elasticsearch.packaging.test.OssRpmPreservationTests; -import org.elasticsearch.packaging.test.OssWindowsServiceTests; -import org.elasticsearch.packaging.test.OssWindowsZipTests; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.junit.runners.Suite.SuiteClasses; - -@RunWith(Suite.class) -@SuiteClasses({ - DefaultLinuxTarTests.class, - OssLinuxTarTests.class, - DefaultWindowsZipTests.class, - OssWindowsZipTests.class, - DefaultRpmBasicTests.class, - OssRpmBasicTests.class, - DefaultDebBasicTests.class, - OssDebBasicTests.class, - DefaultDebPreservationTests.class, - OssDebPreservationTests.class, - DefaultRpmPreservationTests.class, - OssRpmPreservationTests.class, - DefaultWindowsServiceTests.class, - OssWindowsServiceTests.class, - DefaultNoJdkLinuxTarTests.class, - OssNoJdkLinuxTarTests.class, - DefaultNoJdkWindowsZipTests.class, - OssNoJdkWindowsZipTests.class, - DefaultNoJdkRpmBasicTests.class, - OssNoJdkRpmBasicTests.class, - DefaultNoJdkDebBasicTests.class, - OssNoJdkDebBasicTests.class -}) -public class PackagingTests {} diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index e5cbbcc60cc..d6ba57ba075 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -20,7 +20,6 @@ package org.elasticsearch.packaging.test; import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.http.client.fluent.Request; import org.elasticsearch.packaging.util.Archives; import org.elasticsearch.packaging.util.Distribution; @@ -36,7 +35,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.Stream; -import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; @@ -225,17 +223,16 @@ public abstract class ArchiveTestCase extends PackagingTestCase { final Shell sh = new Shell(); // Create temporary directory with a space and link to java binary. // Use it as java_home - String nameWithSpace = RandomStrings.randomAsciiAlphanumOfLength(getRandom(), 10) + "java home"; - String test_java_home = FileUtils.mkdir(Paths.get("/home",ARCHIVE_OWNER, nameWithSpace)).toAbsolutePath().toString(); + String testJavaHome = FileUtils.mkdir(Paths.get("/home", ARCHIVE_OWNER, "java home")).toAbsolutePath().toString(); try { final String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); final String java = systemJavaHome + "/bin/java"; - sh.run("mkdir -p \"" + test_java_home + "/bin\""); - sh.run("ln -s \"" + java + "\" \"" + test_java_home + "/bin/java\""); - sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + test_java_home + "\""); + sh.run("mkdir -p \"" + testJavaHome + "/bin\""); + sh.run("ln -s \"" + java + "\" \"" + testJavaHome + "/bin/java\""); + sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + testJavaHome + "\""); - sh.getEnv().put("JAVA_HOME", test_java_home); + sh.getEnv().put("JAVA_HOME", testJavaHome); //verify ES can start, stop and run plugin list Archives.runElasticsearch(installation, sh); @@ -246,7 +243,7 @@ public abstract class ArchiveTestCase extends PackagingTestCase { Result result = sh.run(pluginListCommand); assertThat(result.exitCode, equalTo(0)); } finally { - FileUtils.rm(Paths.get("\"" + test_java_home + "\"")); + FileUtils.rm(Paths.get(testJavaHome)); } }); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java index ca6c3e48d41..857fad55eea 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java @@ -45,10 +45,8 @@ import java.util.StringJoiner; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.core.IsNot.not; -import static org.hamcrest.text.IsEmptyString.isEmptyOrNullString; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -228,9 +226,7 @@ public class FileUtils { } public static Path getPackagingArchivesDir() { - String fromEnv = System.getenv("PACKAGING_ARCHIVES"); - assertThat(fromEnv, not(isEmptyOrNullString())); - return Paths.get(fromEnv); + return Paths.get(""); // tests are started in the packaging archives dir, ie the empty relative path } public static Path getDistributionFile(Distribution distribution) { diff --git a/qa/vagrant/ubuntu-1604/build.gradle b/qa/vagrant/ubuntu-1604/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/ubuntu-1804/build.gradle b/qa/vagrant/ubuntu-1804/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/windows-2012r2/build.gradle b/qa/vagrant/windows-2012r2/build.gradle new file mode 100644 index 00000000000..f49de70eae7 --- /dev/null +++ b/qa/vagrant/windows-2012r2/build.gradle @@ -0,0 +1,11 @@ + +String boxId = project.properties.get('vagrant.windows-2012r2.id') +if (boxId != null) { + vagrant { + hostEnv 'VAGRANT_WINDOWS_2012R2_BOX', boxId + } +} else { + tasks.named('distroTest').configure { + onlyIf { false } + } +} diff --git a/qa/vagrant/windows-2016/build.gradle b/qa/vagrant/windows-2016/build.gradle new file mode 100644 index 00000000000..e0cfa1c6875 --- /dev/null +++ b/qa/vagrant/windows-2016/build.gradle @@ -0,0 +1,11 @@ + +String boxId = project.properties.get('vagrant.windows-2016.id') +if (boxId != null) { + vagrant { + hostEnv 'VAGRANT_WINDOWS_2016_BOX', boxId + } +} else { + tasks.named('distroTest').configure { + onlyIf { true } + } +} diff --git a/x-pack/qa/vagrant/build.gradle b/x-pack/qa/vagrant/build.gradle deleted file mode 100644 index 411b8d90c6d..00000000000 --- a/x-pack/qa/vagrant/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -apply plugin: 'elasticsearch.vagrantsupport' -apply plugin: 'elasticsearch.vagrant' - -esvagrant { - inheritTestUtils true -} - -dependencies { - // Inherit Bats test utils from :qa:vagrant project - packaging project(path: ':qa:vagrant', configuration: 'packaging') -}