From 97efb6a403f4ae69c113af17c12dd98691602dde Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 12 Aug 2019 16:01:53 -0700 Subject: [PATCH] Convert vagrant tests to per platform projects (#45064) The vagrant based tests currently reside in a single project, creating dozens of tasks to manage starting and stopping the vagrant VM along with running java and bats tests within each image. This all-in-one pattern makes parallelizing packaging tests difficult. This commit rewrites the vagrant testing infrastructure to be independent of the actual test runners, thus allowing each platform to be handled in a separate subproject. Additionally, the java and bats tests are changed to be run through a "destructive" gradle task, which is run inside the VM. The combination of these will allow parallelization both locally (through running several VMs at once) as well as running the destructive tasks in CI machines dedicated to each platform (thus removing the need for vagrant in CI). --- Vagrantfile | 19 +- .../gradle/test/DistroTestPlugin.java | 300 ++++++++ .../gradle/test/VagrantFixture.groovy | 54 -- .../gradle/vagrant/VagrantCommandTask.groovy | 86 --- .../vagrant/VagrantPropertiesExtension.groovy | 67 -- .../vagrant/VagrantSupportPlugin.groovy | 127 ---- .../gradle/vagrant/VagrantTestPlugin.groovy | 658 ------------------ .../gradle/DistributionDownloadPlugin.java | 8 +- .../gradle/ElasticsearchDistribution.java | 19 +- .../java/org/elasticsearch/gradle/Jdk.java | 6 +- .../gradle/JdkDownloadPlugin.java | 8 +- .../org/elasticsearch/gradle/Util.java} | 36 +- .../gradle/test/BatsTestTask.java | 92 +++ .../gradle/test/GradleDistroTestTask.java | 91 +++ .../gradle/tool/Boilerplate.java | 8 + ...putStream.java => BatsProgressLogger.java} | 46 +- .../gradle/vagrant/VagrantBasePlugin.java | 147 ++++ .../gradle/vagrant/VagrantExtension.java | 93 +++ .../gradle/vagrant/VagrantMachine.java | 210 ++++++ ...Stream.java => VagrantProgressLogger.java} | 31 +- .../gradle/vagrant/VagrantShellTask.java | 109 +++ .../elasticsearch.distro-test.properties | 20 + .../elasticsearch.vagrant.properties | 1 - .../elasticsearch.vagrantsupport.properties | 1 - .../org/elasticsearch/gradle/LoggedExec.java | 18 +- .../gradle/DistributionDownloadPluginIT.java | 4 +- .../vagrant/bats/default}/10_basic.bats | 0 .../default}/20_tar_bootstrap_password.bats | 0 .../25_package_bootstrap_password.bats | 0 .../bats/default}/30_tar_setup_passwords.bats | 0 .../default}/35_package_setup_passwords.bats | 0 .../vagrant/bats/default}/40_tar_certgen.bats | 0 .../bats/default}/45_package_certgen.bats | 0 .../bats/default}/bootstrap_password.bash | 0 .../vagrant/bats/default}/certgen.bash | 7 +- .../bats/default}/setup_passwords.bash | 0 .../tests => bats/oss}/25_tar_plugins.bats | 0 .../oss}/50_modules_and_plugins.bats | 0 .../tests => bats/oss}/70_sysv_initd.bats | 0 .../tests => bats/oss}/80_upgrade.bats | 3 +- .../oss}/module_and_plugin_test_cases.bash | 0 .../packaging => bats}/utils/modules.bash | 0 .../packaging => bats}/utils/packages.bash | 2 +- .../packaging => bats}/utils/plugins.bash | 0 .../packaging => bats}/utils/tar.bash | 0 .../packaging => bats}/utils/utils.bash | 0 .../vagrant/bats}/utils/xpack.bash | 0 qa/vagrant/build.gradle | 91 ++- qa/vagrant/centos-6/build.gradle | 0 qa/vagrant/centos-7/build.gradle | 0 qa/vagrant/debian-8/build.gradle | 0 qa/vagrant/debian-9/build.gradle | 0 qa/vagrant/fedora-28/build.gradle | 0 qa/vagrant/fedora-29/build.gradle | 0 qa/vagrant/oel-6/build.gradle | 0 qa/vagrant/oel-7/build.gradle | 0 qa/vagrant/opensuse-42/build.gradle | 0 qa/vagrant/sles-12/build.gradle | 0 .../packaging/PackagingTests.java | 73 -- .../packaging/test/ArchiveTestCase.java | 15 +- .../packaging/util/FileUtils.java | 6 +- qa/vagrant/ubuntu-1604/build.gradle | 0 qa/vagrant/ubuntu-1804/build.gradle | 0 qa/vagrant/windows-2012r2/build.gradle | 11 + qa/vagrant/windows-2016/build.gradle | 11 + x-pack/qa/vagrant/build.gradle | 11 - 66 files changed, 1266 insertions(+), 1223 deletions(-) create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy rename buildSrc/src/main/{groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy => java/org/elasticsearch/gradle/Util.java} (52%) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java rename buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/{TapLoggerOutputStream.java => BatsProgressLogger.java} (69%) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java rename buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/{VagrantLoggerOutputStream.java => VagrantProgressLogger.java} (73%) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties delete mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties delete mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/10_basic.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/20_tar_bootstrap_password.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/25_package_bootstrap_password.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/30_tar_setup_passwords.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/35_package_setup_passwords.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/40_tar_certgen.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/45_package_certgen.bats (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/bootstrap_password.bash (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/certgen.bash (98%) rename {x-pack/qa/vagrant/src/test/resources/packaging/tests => qa/vagrant/bats/default}/setup_passwords.bash (100%) rename qa/vagrant/{src/test/resources/packaging/tests => bats/oss}/25_tar_plugins.bats (100%) rename qa/vagrant/{src/test/resources/packaging/tests => bats/oss}/50_modules_and_plugins.bats (100%) rename qa/vagrant/{src/test/resources/packaging/tests => bats/oss}/70_sysv_initd.bats (100%) rename qa/vagrant/{src/test/resources/packaging/tests => bats/oss}/80_upgrade.bats (98%) rename qa/vagrant/{src/test/resources/packaging/tests => bats/oss}/module_and_plugin_test_cases.bash (100%) rename qa/vagrant/{src/test/resources/packaging => bats}/utils/modules.bash (100%) rename qa/vagrant/{src/test/resources/packaging => bats}/utils/packages.bash (99%) rename qa/vagrant/{src/test/resources/packaging => bats}/utils/plugins.bash (100%) rename qa/vagrant/{src/test/resources/packaging => bats}/utils/tar.bash (100%) rename qa/vagrant/{src/test/resources/packaging => bats}/utils/utils.bash (100%) rename {x-pack/qa/vagrant/src/test/resources/packaging => qa/vagrant/bats}/utils/xpack.bash (100%) create mode 100644 qa/vagrant/centos-6/build.gradle create mode 100644 qa/vagrant/centos-7/build.gradle create mode 100644 qa/vagrant/debian-8/build.gradle create mode 100644 qa/vagrant/debian-9/build.gradle create mode 100644 qa/vagrant/fedora-28/build.gradle create mode 100644 qa/vagrant/fedora-29/build.gradle create mode 100644 qa/vagrant/oel-6/build.gradle create mode 100644 qa/vagrant/oel-7/build.gradle create mode 100644 qa/vagrant/opensuse-42/build.gradle create mode 100644 qa/vagrant/sles-12/build.gradle delete mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java create mode 100644 qa/vagrant/ubuntu-1604/build.gradle create mode 100644 qa/vagrant/ubuntu-1804/build.gradle create mode 100644 qa/vagrant/windows-2012r2/build.gradle create mode 100644 qa/vagrant/windows-2016/build.gradle delete mode 100644 x-pack/qa/vagrant/build.gradle diff --git a/Vagrantfile b/Vagrantfile index 14f6ad00f3a..ed572aba8a8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -41,6 +41,16 @@ Vagrant.configure(2) do |config| # the elasticsearch project called vagrant.... config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.synced_folder '.', '/elasticsearch' + # TODO: make these syncs work for windows!!! + config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/jars-3", "/root/.gradle/caches/jars-3", + create: true, + owner: "vagrant" + config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/modules-2", "/root/.gradle/caches/modules-2", + create: true, + owner: "vagrant" + config.vm.synced_folder "#{Dir.home}/.gradle/wrapper", "/root/.gradle/wrapper", + create: true, + owner: "vagrant" # Expose project directory. Note that VAGRANT_CWD may not be the same as Dir.pwd PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd @@ -380,10 +390,6 @@ export ZIP=/elasticsearch/distribution/zip/build/distributions export TAR=/elasticsearch/distribution/tar/build/distributions export RPM=/elasticsearch/distribution/rpm/build/distributions export DEB=/elasticsearch/distribution/deb/build/distributions -export BATS=/project/build/bats -export BATS_UTILS=/project/build/packaging/bats/utils -export BATS_TESTS=/project/build/packaging/bats/tests -export PACKAGING_ARCHIVES=/project/build/packaging/archives export PACKAGING_TESTS=/project/build/packaging/tests VARS cat \<\ /etc/sudoers.d/elasticsearch_vars @@ -391,11 +397,10 @@ Defaults env_keep += "ZIP" Defaults env_keep += "TAR" Defaults env_keep += "RPM" Defaults env_keep += "DEB" -Defaults env_keep += "BATS" -Defaults env_keep += "BATS_UTILS" -Defaults env_keep += "BATS_TESTS" Defaults env_keep += "PACKAGING_ARCHIVES" Defaults env_keep += "PACKAGING_TESTS" +Defaults env_keep += "BATS_UTILS" +Defaults env_keep += "BATS_TESTS" Defaults env_keep += "JAVA_HOME" Defaults env_keep += "SYSTEM_JAVA_HOME" SUDOERS_VARS diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java new file mode 100644 index 00000000000..d78dba6b47c --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -0,0 +1,300 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.BuildPlugin; +import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.DistributionDownloadPlugin; +import org.elasticsearch.gradle.ElasticsearchDistribution; +import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; +import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; +import org.elasticsearch.gradle.ElasticsearchDistribution.Type; +import org.elasticsearch.gradle.Jdk; +import org.elasticsearch.gradle.JdkDownloadPlugin; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.elasticsearch.gradle.vagrant.BatsProgressLogger; +import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; +import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.Directory; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.TaskInputs; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.testing.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +public class DistroTestPlugin implements Plugin { + + private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691"; + + // all distributions used by distro tests. this is temporary until tests are per distribution + private static final String PACKAGING_DISTRIBUTION = "packaging"; + private static final String COPY_PACKAGING_TASK = "copyPackagingArchives"; + private static final String IN_VM_SYSPROP = "tests.inVM"; + + private static Version upgradeVersion; + private Provider archivesDir; + private TaskProvider copyPackagingArchives; + private Jdk gradleJdk; + + @Override + public void apply(Project project) { + project.getPluginManager().apply(JdkDownloadPlugin.class); + project.getPluginManager().apply(DistributionDownloadPlugin.class); + project.getPluginManager().apply(VagrantBasePlugin.class); + project.getPluginManager().apply(JavaPlugin.class); + + configureVM(project); + + if (upgradeVersion == null) { + // just read this once, since it is the same for all projects. this is safe because gradle configuration is single threaded + upgradeVersion = getUpgradeVersion(project); + } + + // setup task to run inside VM + configureDistributions(project); + configureCopyPackagingTask(project); + configureDistroTest(project); + configureBatsTest(project, "oss"); + configureBatsTest(project, "default"); + } + + private static Jdk createJdk(NamedDomainObjectContainer jdksContainer, String name, String version, String platform) { + Jdk jdk = jdksContainer.create(name); + jdk.setVersion(version); + jdk.setPlatform(platform); + return jdk; + } + + private static Version getUpgradeVersion(Project project) { + String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion"); + if (upgradeFromVersionRaw != null) { + return Version.fromString(upgradeFromVersionRaw); + } + + // was not passed in, so randomly choose one from bwc versions + ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class); + + if ((boolean) extraProperties.get("bwc_tests_enabled") == false) { + // Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests + return Version.fromString(project.getVersion().toString()); + } + + ExtraPropertiesExtension rootExtraProperties = project.getRootProject().getExtensions().getByType(ExtraPropertiesExtension.class); + String firstPartOfSeed = rootExtraProperties.get("testSeed").toString().split(":")[0]; + final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16); + BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions"); + final List indexCompatVersions = bwcVersions.getIndexCompatible(); + return indexCompatVersions.get(new Random(seed).nextInt(indexCompatVersions.size())); + } + + private void configureVM(Project project) { + String box = project.getName(); + + // setup jdks used by the distro tests, and by gradle executing + + NamedDomainObjectContainer jdksContainer = JdkDownloadPlugin.getContainer(project); + String platform = box.contains("windows") ? "windows" : "linux"; + this.gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VERSION, platform); + + // setup VM used by these tests + VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); + vagrant.setBox(box); + vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdk, "/bin:$PATH", "\\bin;$Env:PATH")); + vagrant.setIsWindowsVM(box.contains("windows")); + } + + private static Object convertPath(Project project, VagrantExtension vagrant, Jdk jdk, + String additionaLinux, String additionalWindows) { + return new Object() { + @Override + public String toString() { + if (vagrant.isWindowsVM()) { + return convertWindowsPath(project, jdk.getPath()) + additionalWindows; + } + return convertLinuxPath(project, jdk.getPath()) + additionaLinux; + } + }; + } + + private void configureCopyPackagingTask(Project project) { + this.archivesDir = project.getParent().getLayout().getBuildDirectory().dir("packaging/archives"); + // temporary, until we have tasks per distribution + this.copyPackagingArchives = Boilerplate.maybeRegister(project.getParent().getTasks(), COPY_PACKAGING_TASK, Copy.class, + t -> { + t.into(archivesDir); + t.from(project.getConfigurations().getByName(PACKAGING_DISTRIBUTION)); + + Path archivesPath = archivesDir.get().getAsFile().toPath(); + + // write bwc version, and append -SNAPSHOT if it is an unreleased version + ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class); + BwcVersions bwcVersions = (BwcVersions) extraProperties.get("bwcVersions"); + final String upgradeFromVersion; + if (bwcVersions.unreleasedInfo(upgradeVersion) != null) { + upgradeFromVersion = upgradeVersion.toString() + "-SNAPSHOT"; + } else { + upgradeFromVersion = upgradeVersion.toString(); + } + TaskInputs inputs = t.getInputs(); + inputs.property("version", VersionProperties.getElasticsearch()); + inputs.property("upgrade_from_version", upgradeFromVersion); + // TODO: this is serializable, need to think how to represent this as an input + //inputs.property("bwc_versions", bwcVersions); + t.doLast(action -> { + try { + Files.writeString(archivesPath.resolve("version"), VersionProperties.getElasticsearch()); + Files.writeString(archivesPath.resolve("upgrade_from_version"), upgradeFromVersion); + // this is always true, but bats tests rely on it. It is just temporary until bats is removed. + Files.writeString(archivesPath.resolve("upgrade_is_oss"), ""); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + }); + } + + private void configureDistroTest(Project project) { + BuildPlugin.configureCompile(project); + BuildPlugin.configureRepositories(project); + BuildPlugin.configureTestTasks(project); + BuildPlugin.configureInputNormalization(project); + + TaskProvider destructiveTest = project.getTasks().register("destructiveDistroTest", Test.class, + t -> { + t.setMaxParallelForks(1); + t.setWorkingDir(archivesDir.get()); + if (System.getProperty(IN_VM_SYSPROP) == null) { + t.dependsOn(copyPackagingArchives, gradleJdk); + } + }); + + // setup outer task to run + project.getTasks().register("distroTest", GradleDistroTestTask.class, + t -> { + t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + t.setDescription("Runs distribution tests within vagrant"); + t.setTaskName(project.getPath() + ":" + destructiveTest.getName()); + t.extraArg("-D'" + IN_VM_SYSPROP + "'"); + t.dependsOn(copyPackagingArchives, gradleJdk); + }); + } + + private void configureBatsTest(Project project, String type) { + + // destructive task to run inside + TaskProvider destructiveTest = project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class, + t -> { + // this is hacky for shared source, but bats are a temporary thing we are removing, so it is not worth + // the overhead of a real project dependency + Directory batsDir = project.getParent().getLayout().getProjectDirectory().dir("bats"); + t.setTestsDir(batsDir.dir(type)); + t.setUtilsDir(batsDir.dir("utils")); + t.setArchivesDir(archivesDir.get()); + t.setPackageName("elasticsearch" + (type.equals("oss") ? "-oss" : "")); + if (System.getProperty(IN_VM_SYSPROP) == null) { + t.dependsOn(copyPackagingArchives, gradleJdk); + } + }); + + VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); + // setup outer task to run + project.getTasks().register("batsTest." + type, GradleDistroTestTask.class, + t -> { + t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + t.setDescription("Runs bats tests within vagrant"); + t.setTaskName(project.getPath() + ":" + destructiveTest.getName()); + t.setProgressHandler(new BatsProgressLogger(project.getLogger())); + t.extraArg("-D'" + IN_VM_SYSPROP + "'"); + t.dependsOn(copyPackagingArchives, gradleJdk); + t.onlyIf(spec -> vagrant.isWindowsVM() == false); // bats doesn't run on windows + }); + } + + private void configureDistributions(Project project) { + NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); + + for (Type type : Arrays.asList(Type.DEB, Type.RPM)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch()); + } + } + // upgrade version is always bundled jdk + // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced + // witha dedicated upgrade test from every bwc version like other bwc tests + addDistro(distributions, type, null, Flavor.DEFAULT, true, upgradeVersion.toString()); + if (upgradeVersion.onOrAfter("6.3.0")) { + addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString()); + } + } + for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + addDistro(distributions, Type.ARCHIVE, platform, flavor, bundledJdk, VersionProperties.getElasticsearch()); + } + } + } + + // temporary until distro tests have one test per distro + Configuration packagingConfig = project.getConfigurations().create(PACKAGING_DISTRIBUTION); + List distroConfigs = distributions.stream().map(ElasticsearchDistribution::getConfiguration) + .collect(Collectors.toList()); + packagingConfig.setExtendsFrom(distroConfigs); + } + + private static void addDistro(NamedDomainObjectContainer distributions, + Type type, Platform platform, Flavor flavor, boolean bundledJdk, String version) { + + String name = flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") + "-" + version; + if (distributions.findByName(name) != null) { + return; + } + distributions.create(name, d -> { + d.setFlavor(flavor); + d.setType(type); + if (type == Type.ARCHIVE) { + d.setPlatform(platform); + } + d.setBundledJdk(bundledJdk); + d.setVersion(version); + }); + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy deleted file mode 100644 index fa08a8f9c66..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/VagrantFixture.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.test - -import org.elasticsearch.gradle.vagrant.VagrantCommandTask -import org.gradle.api.Task - -/** - * A fixture for integration tests which runs in a virtual machine launched by Vagrant. - */ -class VagrantFixture extends VagrantCommandTask implements Fixture { - - private VagrantCommandTask stopTask - - public VagrantFixture() { - this.stopTask = project.tasks.create(name: "${name}#stop", type: VagrantCommandTask) { - command 'halt' - } - finalizedBy this.stopTask - } - - @Override - void setBoxName(String boxName) { - super.setBoxName(boxName) - this.stopTask.setBoxName(boxName) - } - - @Override - void setEnvironmentVars(Map environmentVars) { - super.setEnvironmentVars(environmentVars) - this.stopTask.setEnvironmentVars(environmentVars) - } - - @Override - public Task getStopTask() { - return this.stopTask - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy deleted file mode 100644 index bcc612c7afa..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.apache.commons.io.output.TeeOutputStream -import org.elasticsearch.gradle.LoggedExec -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.Optional -import org.gradle.internal.logging.progress.ProgressLoggerFactory - -import javax.inject.Inject - -/** - * Runs a vagrant command. Pretty much like Exec task but with a nicer output - * formatter and defaults to `vagrant` as first part of commandLine. - */ -public class VagrantCommandTask extends LoggedExec { - - @Input - String command - - @Input @Optional - String subcommand - - @Input - String boxName - - @Input - Map environmentVars - - public VagrantCommandTask() { - executable = 'vagrant' - - // We're using afterEvaluate here to slot in some logic that captures configurations and - // modifies the command line right before the main execution happens. The reason that we - // call doFirst instead of just doing the work in the afterEvaluate is that the latter - // restricts how subclasses can extend functionality. Calling afterEvaluate is like having - // all the logic of a task happening at construction time, instead of at execution time - // where a subclass can override or extend the logic. - project.afterEvaluate { - doFirst { - if (environmentVars != null) { - environment environmentVars - } - - // Build our command line for vagrant - def vagrantCommand = [executable, command] - if (subcommand != null) { - vagrantCommand = vagrantCommand + subcommand - } - commandLine([*vagrantCommand, boxName, *args]) - - // It'd be nice if --machine-readable were, well, nice - standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream()) - } - } - } - - @Inject - ProgressLoggerFactory getProgressLoggerFactory() { - throw new UnsupportedOperationException() - } - - protected OutputStream createLoggerOutputStream() { - return new VagrantLoggerOutputStream(getProgressLoggerFactory().newOperation(boxName + " " + command).setDescription(boxName), - /* Vagrant tends to output a lot of stuff, but most of the important - stuff starts with ==> $box */ - "==> $boxName: ") - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy deleted file mode 100644 index e9b664a5a31..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.elasticsearch.gradle.Version -import org.gradle.api.tasks.Input - -class VagrantPropertiesExtension { - - @Input - List boxes - - @Input - Version upgradeFromVersion - - @Input - List upgradeFromVersions - - @Input - String batsDir - - @Input - Boolean inheritTests - - @Input - Boolean inheritTestUtils - - @Input - String testClass - - VagrantPropertiesExtension(List availableBoxes) { - this.boxes = availableBoxes - this.batsDir = 'src/test/resources/packaging' - } - - void boxes(String... boxes) { - this.boxes = Arrays.asList(boxes) - } - - void setBatsDir(String batsDir) { - this.batsDir = batsDir - } - - void setInheritTests(Boolean inheritTests) { - this.inheritTests = inheritTests - } - - void setInheritTestUtils(Boolean inheritTestUtils) { - this.inheritTestUtils = inheritTestUtils - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy deleted file mode 100644 index 9dfe487e830..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy +++ /dev/null @@ -1,127 +0,0 @@ -package org.elasticsearch.gradle.vagrant - -import org.gradle.api.GradleException -import org.gradle.api.InvalidUserDataException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.process.ExecResult -import org.gradle.process.internal.ExecException - -/** - * Global configuration for if Vagrant tasks are supported in this - * build environment. - */ -class VagrantSupportPlugin implements Plugin { - - @Override - void apply(Project project) { - if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) { - Map vagrantInstallation = getVagrantInstallation(project) - Map virtualBoxInstallation = getVirtualBoxInstallation(project) - - project.rootProject.ext.vagrantInstallation = vagrantInstallation - project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation - project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported - project.rootProject.ext.vagrantEnvChecksDone = true - - // Finding that HOME needs to be set when performing vagrant updates - String homeLocation = System.getenv("HOME") - if (project.rootProject.ext.vagrantSupported && homeLocation == null) { - throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " + - "and requires \$HOME to be set to function properly.") - } - } - - addVerifyInstallationTasks(project) - } - - private Map getVagrantInstallation(Project project) { - try { - ByteArrayOutputStream pipe = new ByteArrayOutputStream() - ExecResult runResult = project.exec { - commandLine 'vagrant', '--version' - standardOutput pipe - ignoreExitValue true - } - String version = pipe.toString().trim() - if (runResult.exitValue == 0) { - if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) { - return [ 'supported' : true ] - } else { - return [ 'supported' : false, - 'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ] - } - } else { - return [ 'supported' : false, - 'info' : "Could not read installed vagrant version:\n" + version ] - } - } catch (ExecException e) { - // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set. - // Swallow error. Vagrant isn't installed. Don't halt the build here. - return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ] - } - } - - private Map getVirtualBoxInstallation(Project project) { - try { - ByteArrayOutputStream pipe = new ByteArrayOutputStream() - ExecResult runResult = project.exec { - commandLine 'vboxmanage', '--version' - standardOutput = pipe - ignoreExitValue true - } - String version = pipe.toString().trim() - if (runResult.exitValue == 0) { - try { - String[] versions = version.split('\\.') - int major = Integer.parseInt(versions[0]) - int minor = Integer.parseInt(versions[1]) - if ((major < 5) || (major == 5 && minor < 1)) { - return [ 'supported' : false, - 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ] - } else { - return [ 'supported' : true ] - } - } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { - return [ 'supported' : false, - 'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ] - } - } else { - return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ] - } - } catch (ExecException e) { - // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set. - // Swallow error. VirtualBox isn't installed. Don't halt the build here. - return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ] - } - } - - private void addVerifyInstallationTasks(Project project) { - createCheckVagrantVersionTask(project) - createCheckVirtualBoxVersionTask(project) - } - - private void createCheckVagrantVersionTask(Project project) { - project.tasks.create('vagrantCheckVersion') { - description 'Check the Vagrant version' - group 'Verification' - doLast { - if (project.rootProject.vagrantInstallation.supported == false) { - throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info) - } - } - } - } - - private void createCheckVirtualBoxVersionTask(Project project) { - project.tasks.create('virtualboxCheckVersion') { - description 'Check the Virtualbox version' - group 'Verification' - doLast { - if (project.rootProject.virtualBoxInstallation.supported == false) { - throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info) - } - } - } - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy deleted file mode 100644 index 3868e0417f4..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ /dev/null @@ -1,658 +0,0 @@ -package org.elasticsearch.gradle.vagrant - -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.BwcVersions -import org.elasticsearch.gradle.FileContentsTask -import org.elasticsearch.gradle.Jdk -import org.elasticsearch.gradle.JdkDownloadPlugin -import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.Version -import org.gradle.api.GradleException -import org.gradle.api.InvalidUserDataException -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.artifacts.dsl.RepositoryHandler -import org.gradle.api.execution.TaskExecutionAdapter -import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Delete -import org.gradle.api.tasks.Exec -import org.gradle.api.tasks.StopExecutionException -import org.gradle.api.tasks.TaskState - -import java.nio.file.Paths - -import static java.util.Collections.unmodifiableList - -class VagrantTestPlugin implements Plugin { - - /** All Linux boxes that we test. These are all always supplied **/ - static final List LINUX_BOXES = unmodifiableList([ - 'centos-6', - 'centos-7', - 'debian-8', - 'debian-9', - 'fedora-28', - 'fedora-29', - 'oel-6', - 'oel-7', - 'opensuse-42', - /* TODO: need a real RHEL license now that it is out of beta 'rhel-8',*/ - 'sles-12', - 'ubuntu-1604', - 'ubuntu-1804' - ]) - - /** All Windows boxes that we test, which may or may not be supplied **/ - static final List WINDOWS_BOXES = unmodifiableList([ - 'windows-2012r2', - 'windows-2016' - ]) - - /** All boxes that we test, some of which may not be supplied **/ - static final List ALL_BOXES = unmodifiableList(LINUX_BOXES + WINDOWS_BOXES) - - /** Boxes used when sampling the tests **/ - static final List SAMPLE = unmodifiableList([ - 'centos-7', - 'ubuntu-1604' - ]) - - /** All distributions to bring into test VM, whether or not they are used **/ - static final List DISTRIBUTIONS = unmodifiableList([ - 'archives:linux-tar', - 'archives:oss-linux-tar', - 'archives:windows-zip', - 'archives:oss-windows-zip', - 'packages:rpm', - 'packages:oss-rpm', - 'packages:deb', - 'packages:oss-deb', - 'archives:no-jdk-linux-tar', - 'archives:oss-no-jdk-linux-tar', - 'archives:no-jdk-windows-zip', - 'archives:oss-no-jdk-windows-zip', - 'packages:no-jdk-rpm', - 'packages:oss-no-jdk-rpm', - 'packages:no-jdk-deb', - 'packages:oss-no-jdk-deb' - ]) - - /** Packages onboarded for upgrade tests **/ - static final List UPGRADE_FROM_ARCHIVES = unmodifiableList(['rpm', 'deb']) - - private static final PACKAGING_CONFIGURATION = 'packaging' - private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest' - private static final BATS = 'bats' - private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS" - - /** Boxes that have been supplied and are available for testing **/ - List availableBoxes = [] - - /** extra env vars to pass to vagrant for box configuration **/ - Map vagrantBoxEnvVars = [:] - - private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691" - private Jdk linuxGradleJdk; - private Jdk windowsGradleJdk; - - @Override - void apply(Project project) { - project.pluginManager.apply(JdkDownloadPlugin.class) - NamedDomainObjectContainer jdksContainer = (NamedDomainObjectContainer) project.getExtensions().getByName("jdks"); - linuxGradleJdk = jdksContainer.create("linux_gradle") { - version = GRADLE_JDK_VERSION - platform = "linux" - } - windowsGradleJdk = jdksContainer.create("windows_gradle") { - version = GRADLE_JDK_VERSION - platform = "windows" - } - - collectAvailableBoxes(project) - - // Creates the Vagrant extension for the project - project.extensions.create('esvagrant', VagrantPropertiesExtension, listSelectedBoxes(project)) - - // Add required repositories for packaging tests - configurePackagingArchiveRepositories(project) - - // Creates custom configurations for Bats testing files (and associated scripts and archives) - createPackagingConfiguration(project) - project.configurations.create(PACKAGING_TEST_CONFIGURATION) - - // Creates all the main Vagrant tasks - createVagrantTasks(project) - - if (project.extensions.esvagrant.boxes == null || project.extensions.esvagrant.boxes.size() == 0) { - throw new InvalidUserDataException('Must specify at least one vagrant box') - } - - for (String box : project.extensions.esvagrant.boxes) { - if (ALL_BOXES.contains(box) == false) { - throw new InvalidUserDataException("Vagrant box [${box}] is unknown to this plugin. Valid boxes are ${ALL_BOXES}") - } - - if (availableBoxes.contains(box) == false) { - throw new InvalidUserDataException("Vagrant box [${box}] is not available because an image is not supplied for it. " + - "Available boxes with supplied images are ${availableBoxes}") - } - } - - // Creates all tasks related to the Vagrant boxes - createVagrantBoxesTasks(project) - } - - /** - * Enumerate all the boxes that we know about and could possibly choose to test - */ - private void collectAvailableBoxes(Project project) { - // these images are hardcoded in the Vagrantfile and are always available - availableBoxes.addAll(LINUX_BOXES) - - // these images need to be provided at runtime - String windows_2012r2_box = project.getProperties().get('vagrant.windows-2012r2.id') - if (windows_2012r2_box != null && windows_2012r2_box.isEmpty() == false) { - availableBoxes.add('windows-2012r2') - vagrantBoxEnvVars['VAGRANT_WINDOWS_2012R2_BOX'] = windows_2012r2_box - } - - String windows_2016_box = project.getProperties().get('vagrant.windows-2016.id') - if (windows_2016_box != null && windows_2016_box.isEmpty() == false) { - availableBoxes.add('windows-2016') - vagrantBoxEnvVars['VAGRANT_WINDOWS_2016_BOX'] = windows_2016_box - } - } - - /** - * Enumerate all the boxes that we have chosen to test - */ - private static List listSelectedBoxes(Project project) { - String vagrantBoxes = project.getProperties().get('vagrant.boxes', 'sample') - switch (vagrantBoxes) { - case 'sample': - return SAMPLE - case 'linux-all': - return LINUX_BOXES - case 'windows-all': - return WINDOWS_BOXES - case 'all': - return ALL_BOXES - case '': - return [] - default: - return vagrantBoxes.split(',') - } - } - - private static void configurePackagingArchiveRepositories(Project project) { - RepositoryHandler repos = project.repositories - - repos.jcenter() // will have releases before 5.0.0 - - /* Setup a repository that tries to download from - https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext] - which should work for 5.0.0+. This isn't a real ivy repository but gradle - is fine with that */ - repos.ivy { - name "elasticsearch" - artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]" - } - } - - private static void createPackagingConfiguration(Project project) { - project.configurations.create(PACKAGING_CONFIGURATION) - - String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion"); - Version upgradeFromVersion - if (upgradeFromVersionRaw == null) { - String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0) - final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16) - final def indexCompatVersions = project.bwcVersions.indexCompatible - upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())] - } else { - upgradeFromVersion = Version.fromString(upgradeFromVersionRaw) - } - - List dependencies = new ArrayList<>() - DISTRIBUTIONS.each { - // Adds a dependency for the current version - dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) - } - - if (project.ext.bwc_tests_enabled) { - // The version of elasticsearch that we upgrade *from* - // we only add them as dependencies if the bwc tests are enabled, so we don't trigger builds otherwise - BwcVersions.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion) - if (unreleasedInfo != null) { - // handle snapshots pointing to bwc build - UPGRADE_FROM_ARCHIVES.each { - dependencies.add(project.dependencies.project( - path: "${unreleasedInfo.gradleProjectPath}", configuration: it)) - if (upgradeFromVersion.onOrAfter('6.3.0')) { - dependencies.add(project.dependencies.project( - path: "${unreleasedInfo.gradleProjectPath}", configuration: "oss-${it}")) - } - } - } else { - UPGRADE_FROM_ARCHIVES.each { - // The version of elasticsearch that we upgrade *from* - if (upgradeFromVersion.onOrAfter('7.0.0')) { - String arch = it == "rpm" ? "x86_64" : "amd64" - dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}-${arch}@${it}") - dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}-${arch}@${it}") - } else { - dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") - if (upgradeFromVersion.onOrAfter('6.3.0')) { - dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") - } - } - } - } - } else { - // Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests. - upgradeFromVersion = Version.fromString(project.version) - } - - for (Object dependency : dependencies) { - project.dependencies.add(PACKAGING_CONFIGURATION, dependency) - } - - project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion - } - - private static void createCleanTask(Project project) { - if (project.tasks.findByName('clean') == null) { - project.tasks.create('clean', Delete.class) { - description 'Clean the project build directory' - group 'Build' - delete project.buildDir - } - } - } - - private static void createStopTask(Project project) { - project.tasks.create('stop') { - description 'Stop any tasks from tests that still may be running' - group 'Verification' - } - } - - private static void createSmokeTestTask(Project project) { - project.tasks.create('vagrantSmokeTest') { - description 'Smoke test the specified vagrant boxes' - group 'Verification' - } - } - - private void createPrepareVagrantTestEnvTask(Project project) { - File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION) - - File archivesDir = new File(packagingDir, 'archives') - Copy copyPackagingArchives = project.tasks.create('copyPackagingArchives', Copy) { - into archivesDir - from project.configurations[PACKAGING_CONFIGURATION] - } - - File testsDir = new File(packagingDir, 'tests') - Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) { - into testsDir - from project.configurations[PACKAGING_TEST_CONFIGURATION] - } - - Task createLinuxRunnerScript = project.tasks.create('createLinuxRunnerScript', FileContentsTask) { - dependsOn copyPackagingTests, linuxGradleJdk - file "${testsDir}/run-tests.sh" - contents """\ - if [ "\$#" -eq 0 ]; then - test_args=( "${-> project.extensions.esvagrant.testClass}" ) - else - test_args=( "\$@" ) - fi - - "${-> convertLinuxPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}" - """ - } - Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) { - dependsOn copyPackagingTests, windowsGradleJdk - file "${testsDir}/run-tests.ps1" - // the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely - // a little trappy for those unfamiliar with powershell - contents """\ - try { - if (\$args.Count -eq 0) { - \$testArgs = @("${-> project.extensions.esvagrant.testClass}") - } else { - \$testArgs = \$args - } - & "${-> convertWindowsPath(project, windowsGradleJdk.toString()) }/bin/java" -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs - exit \$LASTEXITCODE - } catch { - # catch if we have a failure to even run the script at all above, equivalent to set -e, sort of - echo "\$_.Exception.Message" - exit 1 - } - """ - } - - Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) { - dependsOn copyPackagingArchives - file "${archivesDir}/version" - contents project.version - } - - Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) { - String version = project.extensions.esvagrant.upgradeFromVersion - if (project.bwcVersions.unreleased.contains(project.extensions.esvagrant.upgradeFromVersion)) { - version += "-SNAPSHOT" - } - dependsOn copyPackagingArchives - file "${archivesDir}/upgrade_from_version" - contents version - } - - Task createUpgradeIsOssFile = project.tasks.create('createUpgradeIsOssFile', FileContentsTask) { - dependsOn copyPackagingArchives - doFirst { - project.delete("${archivesDir}/upgrade_is_oss") - if (project.extensions.esvagrant.upgradeFromVersion.before('6.3.0')) { - throw new StopExecutionException("upgrade version is before 6.3.0") - } - } - file "${archivesDir}/upgrade_is_oss" - contents '' - } - - File batsDir = new File(packagingDir, BATS) - Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) { - into "${batsDir}/tests" - from { - "${project.extensions.esvagrant.batsDir}/tests" - } - } - - Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) { - into "${batsDir}/utils" - from { - "${project.extensions.esvagrant.batsDir}/utils" - } - } - - // Now we iterate over dependencies of the bats configuration. When a project dependency is found, - // we bring back its test files or test utils. - project.afterEvaluate { - project.configurations[PACKAGING_CONFIGURATION].dependencies - .findAll {it.targetConfiguration == PACKAGING_CONFIGURATION } - .each { d -> - if (d instanceof DefaultProjectDependency) { - DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d - Project externalBatsProject = externalBatsDependency.dependencyProject - String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir - - if (project.extensions.esvagrant.inheritTests) { - copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests")) - } - if (project.extensions.esvagrant.inheritTestUtils) { - copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils")) - } - } - } - } - - Task vagrantSetUpTask = project.tasks.create('setupPackagingTest') - vagrantSetUpTask.dependsOn( - 'vagrantCheckVersion', - copyPackagingArchives, - copyPackagingTests, - createLinuxRunnerScript, - createWindowsRunnerScript, - createVersionFile, - createUpgradeFromFile, - createUpgradeIsOssFile, - copyBatsTests, - copyBatsUtils - ) - } - - private static void createPackagingTestTask(Project project) { - project.tasks.create('packagingTest') { - group 'Verification' - description "Tests distribution installation on different platforms using vagrant. See TESTING.asciidoc for details." - dependsOn 'vagrantCheckVersion' - } - } - - private void createBoxListTasks(Project project) { - project.tasks.create('listAllBoxes') { - group 'Verification' - description 'List all vagrant boxes which can be tested by this plugin' - doLast { - println("All vagrant boxes supported by ${project.path}") - for (String box : ALL_BOXES) { - println(box) - } - } - dependsOn 'vagrantCheckVersion' - } - - project.tasks.create('listAvailableBoxes') { - group 'Verification' - description 'List all vagrant boxes which are available for testing' - doLast { - println("All vagrant boxes available to ${project.path}") - for (String box : availableBoxes) { - println(box) - } - } - dependsOn 'vagrantCheckVersion' - } - } - - private void createVagrantTasks(Project project) { - createCleanTask(project) - createStopTask(project) - createSmokeTestTask(project) - createPrepareVagrantTestEnvTask(project) - createPackagingTestTask(project) - createBoxListTasks(project) - } - - private void createVagrantBoxesTasks(Project project) { - assert project.extensions.esvagrant.boxes != null - - assert project.tasks.stop != null - Task stop = project.tasks.stop - - assert project.tasks.vagrantSmokeTest != null - Task vagrantSmokeTest = project.tasks.vagrantSmokeTest - - assert project.tasks.vagrantCheckVersion != null - Task vagrantCheckVersion = project.tasks.vagrantCheckVersion - - assert project.tasks.virtualboxCheckVersion != null - Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion - - assert project.tasks.setupPackagingTest != null - Task setupPackagingTest = project.tasks.setupPackagingTest - - assert project.tasks.packagingTest != null - Task packagingTest = project.tasks.packagingTest - - /* - * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD) - * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin. - */ - def vagrantEnvVars = [ - 'VAGRANT_CWD' : "${project.rootDir.absolutePath}", - 'VAGRANT_VAGRANTFILE' : 'Vagrantfile', - 'VAGRANT_PROJECT_DIR' : "${project.projectDir.absolutePath}" - ] - vagrantEnvVars.putAll(vagrantBoxEnvVars) - - // Each box gets it own set of tasks - for (String box : availableBoxes) { - String boxTask = box.capitalize().replace('-', '') - - // always add a halt task for all boxes, so clean makes sure they are all shutdown - Task halt = project.tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) { - command 'halt' - boxName box - environmentVars vagrantEnvVars - } - stop.dependsOn(halt) - - Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) { - command 'box' - subcommand 'update' - boxName box - environmentVars vagrantEnvVars - dependsOn vagrantCheckVersion, virtualboxCheckVersion - } - update.mustRunAfter(setupPackagingTest) - - /* - * Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag - * vagrant.destroy that defaults to true that can be used to control whether or not to destroy any test boxes before test - * execution. - */ - final String vagrantDestroyProperty = project.getProperties().get('vagrant.destroy', 'true') - boolean vagrantDestroy - if ("true".equals(vagrantDestroyProperty)) { - vagrantDestroy = true - } else if ("false".equals(vagrantDestroyProperty)) { - vagrantDestroy = false - } else { - throw new GradleException("[vagrant.destroy] must be [true] or [false] but was [" + vagrantDestroyProperty + "]") - } - /* - * Some versions of Vagrant will fail destroy if the box does not exist. Therefore we check if the box exists before attempting - * to destroy the box. - */ - final Task destroy = project.tasks.create("vagrant${boxTask}#destroy", LoggedExec) { - commandLine "bash", "-c", "vagrant status ${box} | grep -q \"${box}\\s\\+not created\" || vagrant destroy ${box} --force" - workingDir project.rootProject.rootDir - environment vagrantEnvVars - } - destroy.onlyIf { vagrantDestroy } - update.mustRunAfter(destroy) - - Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) { - command 'up' - boxName box - environmentVars vagrantEnvVars - /* We lock the provider to virtualbox because the Vagrantfile specifies - lots of boxes that only work properly in virtualbox. Virtualbox is - vagrant's default but its possible to change that default and folks do. - But the boxes that we use are unlikely to work properly with other - virtualization providers. Thus the lock. */ - args '--provision', '--provider', 'virtualbox' - /* It'd be possible to check if the box is already up here and output - SKIPPED but that would require running vagrant status which is slow! */ - dependsOn destroy, update - } - - Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) { - environment vagrantEnvVars - dependsOn up - finalizedBy halt - } - vagrantSmokeTest.dependsOn(smoke) - if (LINUX_BOXES.contains(box)) { - smoke.commandLine = ['vagrant', 'ssh', box, '--command', - "set -o pipefail && echo 'Hello from ${project.path}' | sed -ue 's/^/ ${box}: /'"] - } else { - smoke.commandLine = ['vagrant', 'winrm', box, '--command', - "Write-Host ' ${box}: Hello from ${project.path}'"] - } - - if (LINUX_BOXES.contains(box)) { - Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) { - remoteCommand BATS_TEST_COMMAND - boxName box - environmentVars vagrantEnvVars - dependsOn up, setupPackagingTest - finalizedBy halt - } - - TaskExecutionAdapter batsPackagingReproListener = createReproListener(project, batsPackagingTest.path) - batsPackagingTest.doFirst { - project.gradle.addListener(batsPackagingReproListener) - } - batsPackagingTest.doLast { - project.gradle.removeListener(batsPackagingReproListener) - } - if (project.extensions.esvagrant.boxes.contains(box)) { - // these tests are temporarily disabled for suse boxes while we debug an issue - // https://github.com/elastic/elasticsearch/issues/30295 - if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { - packagingTest.dependsOn(batsPackagingTest) - } - } - } - - Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) { - boxName box - environmentVars vagrantEnvVars - dependsOn up, setupPackagingTest - finalizedBy halt - } - - // todo remove this onlyIf after all packaging tests are consolidated - javaPackagingTest.onlyIf { - project.extensions.esvagrant.testClass != null - } - - if (LINUX_BOXES.contains(box)) { - javaPackagingTest.command = 'ssh' - javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"'] - } else { - // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however - // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user - // without administrator elevation), which we need to do for these tests. passing --elevated runs the session - // as a scheduled job locally on the vm as a true administrator to get around this limitation - // - // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225 - // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html - javaPackagingTest.command = 'winrm' - javaPackagingTest.args = ['--elevated', '--command', '& "$Env:PACKAGING_TESTS/run-tests.ps1"; exit $LASTEXITCODE'] - } - - TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path) - javaPackagingTest.doFirst { - project.gradle.addListener(javaPackagingReproListener) - } - javaPackagingTest.doLast { - project.gradle.removeListener(javaPackagingReproListener) - } - if (project.extensions.esvagrant.boxes.contains(box)) { - // these tests are temporarily disabled for suse boxes while we debug an issue - // https://github.com/elastic/elasticsearch/issues/30295 - if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { - packagingTest.dependsOn(javaPackagingTest) - } - } - } - } - - private static TaskExecutionAdapter createReproListener(Project project, String reproTaskPath) { - return new TaskExecutionAdapter() { - @Override - void afterExecute(Task task, TaskState state) { - final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew" - if (state.failure != null) { - println "REPRODUCE WITH: ${gradlew} \"${reproTaskPath}\" -Dtests.seed=${project.testSeed} " - } - } - } - } - - // convert the given path from an elasticsearch repo path to a VM path - private String convertLinuxPath(Project project, String path) { - return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path)); - } - private String convertWindowsPath(Project project, String path) { - return "C:\\elasticsearch\\" + project.rootDir.toPath().relativize(Paths.get(path)).toString().replace('/', '\\'); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 5a3a4a277dd..d8c693b77d2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -203,12 +203,16 @@ public class DistributionDownloadPlugin implements Plugin { String extension = distribution.getType().toString(); String classifier = "x86_64"; - if (distribution.getType() == Type.ARCHIVE) { + if (distribution.getVersion().before("7.0.0")) { + classifier = null; // no platform specific distros before 7.0 + } else if (distribution.getType() == Type.ARCHIVE) { extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; classifier = distribution.getPlatform() + "-" + classifier; + } else if (distribution.getType() == Type.DEB) { + classifier = "amd64"; } return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":") - + distribution.getVersion() + ":" + classifier + "@" + extension; + + distribution.getVersion() + (classifier == null ? "" : ":" + classifier) + "@" + extension; } private static Dependency projectDependency(Project project, String projectPath, String projectConfig) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 53089f9b3d7..815da77a154 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -20,9 +20,7 @@ package org.elasticsearch.gradle; import org.gradle.api.Buildable; -import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.file.FileTree; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskDependency; @@ -30,9 +28,8 @@ import org.gradle.api.tasks.TaskDependency; import java.io.File; import java.util.Iterator; import java.util.Locale; -import java.util.concurrent.Callable; -public class ElasticsearchDistribution implements Buildable { +public class ElasticsearchDistribution implements Buildable, Iterable { public enum Platform { LINUX, @@ -93,10 +90,6 @@ public class ElasticsearchDistribution implements Buildable { return configuration.getBuildDependencies(); } - public FileTree getFileTree(Project project) { - return project.fileTree((Callable) configuration::getSingleFile); - } - @Override public String toString() { return configuration.getSingleFile().toString(); @@ -190,6 +183,16 @@ public class ElasticsearchDistribution implements Buildable { return configuration.getBuildDependencies(); } + @Override + public Iterator iterator() { + return configuration.iterator(); + } + + // TODO: remove this when distro tests are per distribution + public Configuration getConfiguration() { + return configuration; + } + // internal, make this distribution's configuration unmodifiable void finalizeValues() { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java index aa26f398e8b..91516e26af9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java @@ -83,9 +83,13 @@ public class Jdk implements Buildable, Iterable { return configuration; } + public String getPath() { + return configuration.getSingleFile().toString(); + } + @Override public String toString() { - return configuration.getSingleFile().toString(); + return getPath(); } @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java index d4f0d9941da..7c57af701fe 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java @@ -48,13 +48,14 @@ import java.util.regex.Matcher; public class JdkDownloadPlugin implements Plugin { private static final String REPO_NAME_PREFIX = "jdk_repo_"; + private static final String CONTAINER_NAME = "jdks"; @Override public void apply(Project project) { NamedDomainObjectContainer jdksContainer = project.container(Jdk.class, name -> new Jdk(name, project) ); - project.getExtensions().add("jdks", jdksContainer); + project.getExtensions().add(CONTAINER_NAME, jdksContainer); project.afterEvaluate(p -> { for (Jdk jdk : jdksContainer) { @@ -82,6 +83,11 @@ public class JdkDownloadPlugin implements Plugin { }); } + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(CONTAINER_NAME); + } + private static void setupRootJdkDownload(Project rootProject, String platform, String version) { String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version; // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java similarity index 52% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy rename to buildSrc/src/main/java/org/elasticsearch/gradle/Util.java index 1d85d8584bb..ffc5ce353d2 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java @@ -16,30 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.gradle.vagrant -import org.gradle.api.tasks.Input +package org.elasticsearch.gradle; -/** - * Runs bats over vagrant. Pretty much like running it using Exec but with a - * nicer output formatter. - */ -public class BatsOverVagrantTask extends VagrantCommandTask { +import org.gradle.api.GradleException; - @Input - Object remoteCommand +public class Util { - BatsOverVagrantTask() { - command = 'ssh' - } - - void setRemoteCommand(Object remoteCommand) { - this.remoteCommand = Objects.requireNonNull(remoteCommand) - setArgs((Iterable) ['--command', remoteCommand]) - } - - @Override - protected OutputStream createLoggerOutputStream() { - return new TapLoggerOutputStream(logger, getProgressLoggerFactory().newOperation(boxName).setDescription(boxName)); + public static boolean getBooleanProperty(String property, boolean defaultValue) { + String propertyValue = System.getProperty(property); + if (propertyValue == null) { + return defaultValue; + } + if ("true".equals(propertyValue)) { + return true; + } else if ("false".equals(propertyValue)) { + return false; + } else { + throw new GradleException("Sysprop [" + property + "] must be [true] or [false] but was [" + propertyValue + "]"); + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java new file mode 100644 index 00000000000..c3d79f44ae3 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.Directory; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class BatsTestTask extends DefaultTask { + + private Directory testsDir; + private Directory utilsDir; + private Directory archivesDir; + private String packageName; + + @InputDirectory + public Directory getTestsDir() { + return testsDir; + } + + public void setTestsDir(Directory testsDir) { + this.testsDir = testsDir; + } + + @InputDirectory + public Directory getUtilsDir() { + return utilsDir; + } + + public void setUtilsDir(Directory utilsDir) { + this.utilsDir = utilsDir; + } + + @InputDirectory + public Directory getArchivesDir() { + return archivesDir; + } + + public void setArchivesDir(Directory archivesDir) { + this.archivesDir = archivesDir; + } + + @Input + public String getPackageName() { + return packageName; + } + + public void setPackageName(String packageName) { + this.packageName = packageName; + } + + @TaskAction + public void runBats() { + List command = new ArrayList<>(); + command.add("bats"); + command.add("--tap"); + command.addAll(testsDir.getAsFileTree().getFiles().stream() + .filter(f -> f.getName().endsWith(".bats")) + .sorted().collect(Collectors.toList())); + getProject().exec(spec -> { + spec.setWorkingDir(archivesDir.getAsFile()); + spec.environment(System.getenv()); + spec.environment("BATS_TESTS", testsDir.getAsFile().toString()); + spec.environment("BATS_UTILS", utilsDir.getAsFile().toString()); + spec.environment("PACKAGE_NAME", packageName); + spec.setCommandLine(command); + }); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java new file mode 100644 index 00000000000..cfb960e4e36 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.vagrant.VagrantShellTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.options.Option; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +/** + * Run a gradle task of the current build, within the configured vagrant VM. + */ +public class GradleDistroTestTask extends VagrantShellTask { + + private String taskName; + private String testClass; + private List extraArgs = new ArrayList<>(); + + public void setTaskName(String taskName) { + this.taskName = taskName; + } + + @Input + public String getTaskName() { + return taskName; + } + + @Option(option = "tests", description = "Sets test class or method name to be included, '*' is supported.") + public void setTestClass(String testClass) { + this.testClass = testClass; + } + + @Input + public List getExtraArgs() { + return extraArgs; + } + + public void extraArg(String arg) { + this.extraArgs.add(arg); + } + + @Override + protected List getWindowsScript() { + return getScript(true); + } + + @Override + protected List getLinuxScript() { + return getScript(false); + } + + private List getScript(boolean isWindows) { + String cacheDir = getProject().getBuildDir() + "/gradle-cache"; + StringBuilder line = new StringBuilder(); + line.append(isWindows ? "& .\\gradlew " : "./gradlew "); + line.append(taskName); + line.append(" --project-cache-dir "); + line.append(isWindows ? convertWindowsPath(getProject(), cacheDir) : convertLinuxPath(getProject(), cacheDir)); + line.append(" -S"); + line.append(" -D'org.gradle.logging.level'=" + getProject().getGradle().getStartParameter().getLogLevel()); + if (testClass != null) { + line.append(" --tests="); + line.append(testClass); + } + extraArgs.stream().map(s -> " " + s).forEach(line::append); + return Collections.singletonList(line.toString()); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java index 70926c8982d..760e5f60f1c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java @@ -52,6 +52,14 @@ public abstract class Boilerplate { } + public static TaskProvider maybeRegister(TaskContainer tasks, String name, Class clazz, Action action) { + try { + return tasks.named(name, clazz); + } catch (UnknownTaskException e) { + return tasks.register(name, clazz, action); + } + } + public static void maybeConfigure(TaskContainer tasks, String name, Action config) { TaskProvider task; try { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java similarity index 69% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java index 353b2687ad1..8db4e704fb4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java @@ -19,12 +19,10 @@ package org.elasticsearch.gradle.vagrant; -import org.elasticsearch.gradle.LoggingOutputStream; -import org.gradle.api.GradleScriptException; import org.gradle.api.logging.Logger; -import org.gradle.internal.logging.progress.ProgressLogger; import java.util.Formatter; +import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,49 +39,43 @@ import java.util.regex.Pattern; * There is a Tap4j project but we can't use it because it wants to parse the * entire TAP stream at once and won't parse it stream-wise. */ -public class TapLoggerOutputStream extends LoggingOutputStream { +public class BatsProgressLogger implements UnaryOperator { private static final Pattern lineRegex = Pattern.compile("(?ok|not ok) \\d+(? # skip (?\\(.+\\))?)? \\[(?.+)\\] (?.+)"); + private static final Pattern startRegex = Pattern.compile("1..(\\d+)"); private final Logger logger; - private final ProgressLogger progressLogger; - private boolean isStarted = false; private int testsCompleted = 0; private int testsFailed = 0; private int testsSkipped = 0; private Integer testCount; private String countsFormat; - TapLoggerOutputStream(Logger logger, ProgressLogger progressLogger) { + public BatsProgressLogger(Logger logger) { this.logger = logger; - this.progressLogger = progressLogger; } @Override - public void logLine(String line) { - if (isStarted == false) { - progressLogger.started("started"); - isStarted = true; - } + public String apply(String line) { if (testCount == null) { - try { - int lastDot = line.lastIndexOf('.'); - testCount = Integer.parseInt(line.substring(lastDot + 1)); - int length = String.valueOf(testCount).length(); - String count = "%0" + length + "d"; - countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; - return; - } catch (Exception e) { - throw new GradleScriptException("Error parsing first line of TAP stream!!", e); + Matcher m = startRegex.matcher(line); + if (m.matches() == false) { + // haven't reached start of bats test yet, pass through whatever we see + return line; } + testCount = Integer.parseInt(m.group(1)); + int length = String.valueOf(testCount).length(); + String count = "%0" + length + "d"; + countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; + return null; } Matcher m = lineRegex.matcher(line); if (m.matches() == false) { /* These might be failure report lines or comments or whatever. Its hard to tell and it doesn't matter. */ logger.warn(line); - return; + return null; } boolean skipped = m.group("skip") != null; boolean success = skipped == false && m.group("status").equals("ok"); @@ -104,15 +96,9 @@ public class TapLoggerOutputStream extends LoggingOutputStream { } String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString(); - progressLogger.progress("BATS " + counts + ", " + status + " [" + suiteName + "] " + testName); if (success == false) { logger.warn(line); } - } - - @Override - public void close() { - flush(); - progressLogger.completed(); + return "BATS " + counts + ", " + status + " [" + suiteName + "] " + testName; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java new file mode 100644 index 00000000000..f77fe982f74 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.elasticsearch.gradle.ReaperPlugin; +import org.elasticsearch.gradle.ReaperService; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.execution.TaskActionListener; +import org.gradle.api.execution.TaskExecutionListener; +import org.gradle.api.tasks.TaskState; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class VagrantBasePlugin implements Plugin { + + @Override + public void apply(Project project) { + project.getRootProject().getPluginManager().apply(VagrantSetupCheckerPlugin.class); + project.getRootProject().getPluginManager().apply(VagrantManagerPlugin.class); + project.getRootProject().getPluginManager().apply(ReaperPlugin.class); + + ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class); + VagrantExtension extension = project.getExtensions().create("vagrant", VagrantExtension.class, project); + VagrantMachine service = project.getExtensions().create("vagrantService", VagrantMachine.class, project, extension, reaper); + + project.getGradle().getTaskGraph().whenReady(graph -> + service.refs = graph.getAllTasks().stream() + .filter(t -> t instanceof VagrantShellTask) + .filter(t -> t.getProject() == project) + .count()); + } + + + /** + * Check vagrant and virtualbox versions, if any vagrant test tasks will be run. + */ + static class VagrantSetupCheckerPlugin implements Plugin { + + private static final Pattern VAGRANT_VERSION = Pattern.compile("Vagrant (\\d+\\.\\d+\\.\\d+)"); + private static final Pattern VIRTUAL_BOX_VERSION = Pattern.compile("(\\d+\\.\\d+)"); + + @Override + public void apply(Project project) { + if (project != project.getRootProject()) { + throw new IllegalArgumentException("VagrantSetupCheckerPlugin can only be applied to the root project of a build"); + } + + project.getGradle().getTaskGraph().whenReady(graph -> { + boolean needsVagrant = graph.getAllTasks().stream().anyMatch(t -> t instanceof VagrantShellTask); + if (needsVagrant) { + checkVersion(project, "vagrant", VAGRANT_VERSION, 1, 8, 6); + checkVersion(project, "vboxmanage", VIRTUAL_BOX_VERSION, 5, 1); + } + }); + } + + void checkVersion(Project project, String tool, Pattern versionRegex, int... minVersion) { + ByteArrayOutputStream pipe = new ByteArrayOutputStream(); + project.exec(spec -> { + spec.setCommandLine(tool, "--version"); + spec.setStandardOutput(pipe); + }); + String output = pipe.toString(StandardCharsets.UTF_8).trim(); + Matcher matcher = versionRegex.matcher(output); + if (matcher.find() == false) { + throw new IllegalStateException(tool + + " version output [" + output + "] did not match regex [" + versionRegex.pattern() + "]"); + } + + String version = matcher.group(1); + List versionParts = Stream.of(version.split("\\.")).map(Integer::parseInt).collect(Collectors.toList()); + for (int i = 0; i < minVersion.length; ++i) { + int found = versionParts.get(i); + if (found > minVersion[i]) { + break; // most significant version is good + } else if (found < minVersion[i]) { + throw new IllegalStateException("Unsupported version of " + tool + ". Found [" + version + "], expected [" + + Stream.of(minVersion).map(String::valueOf).collect(Collectors.joining(".")) + "+"); + } // else equal, so check next element + } + } + } + + /** + * Adds global hooks to manage destroying, starting and updating VMs. + */ + static class VagrantManagerPlugin implements Plugin, TaskActionListener, TaskExecutionListener { + + @Override + public void apply(Project project) { + if (project != project.getRootProject()) { + throw new IllegalArgumentException("VagrantManagerPlugin can only be applied to the root project of a build"); + } + project.getGradle().addListener(this); + } + + private void callIfVagrantTask(Task task, Consumer method) { + if (task instanceof VagrantShellTask) { + VagrantMachine service = task.getProject().getExtensions().getByType(VagrantMachine.class); + method.accept(service); + } + } + + @Override + public void beforeExecute(Task task) { /* nothing to do */} + + @Override + public void afterActions(Task task) { /* nothing to do */ } + + @Override + public void beforeActions(Task task) { + callIfVagrantTask(task, VagrantMachine::maybeStartVM); + } + + @Override + public void afterExecute(Task task, TaskState state) { + callIfVagrantTask(task, service -> service.maybeStopVM(state.getFailure() != null)); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java new file mode 100644 index 00000000000..10ec03f7f10 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.gradle.api.Project; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; + +import java.io.File; +import java.util.Map; + +public class VagrantExtension { + + private final Property box; + private final MapProperty hostEnv; + private final MapProperty vmEnv; + private final RegularFileProperty vagrantfile; + private boolean isWindowsVM; + + public VagrantExtension(Project project) { + this.box = project.getObjects().property(String.class); + this.hostEnv = project.getObjects().mapProperty(String.class, Object.class); + this.vmEnv = project.getObjects().mapProperty(String.class, Object.class); + this.vagrantfile = project.getObjects().fileProperty(); + this.vagrantfile.convention(project.getRootProject().getLayout().getProjectDirectory().file("Vagrantfile")); + this.isWindowsVM = false; + } + + @Input + public String getBox() { + return box.get(); + } + + public void setBox(String box) { + // TODO: should verify this against the Vagrantfile, but would need to do so in afterEvaluate once vagrantfile is unmodifiable + this.box.set(box); + } + + @Input + public Map getHostEnv() { + return hostEnv.get(); + } + + public void hostEnv(String name, Object value) { + hostEnv.put(name, value); + } + + @Input + public Map getVmEnv() { + return vmEnv.get(); + } + + public void vmEnv(String name, Object value) { + vmEnv.put(name, value); + } + + @Input + public boolean isWindowsVM() { + return isWindowsVM; + } + + public void setIsWindowsVM(boolean isWindowsVM) { + this.isWindowsVM = isWindowsVM; + } + + @Input + public File getVagrantfile() { + return this.vagrantfile.get().getAsFile(); + } + + public void setVagrantfile(File file) { + vagrantfile.set(file); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java new file mode 100644 index 00000000000..aa89658d9a9 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java @@ -0,0 +1,210 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.apache.commons.io.output.TeeOutputStream; +import org.elasticsearch.gradle.LoggedExec; +import org.elasticsearch.gradle.LoggingOutputStream; +import org.elasticsearch.gradle.ReaperService; +import org.elasticsearch.gradle.Util; +import org.gradle.api.Action; +import org.gradle.api.Project; +import org.gradle.internal.logging.progress.ProgressLogger; +import org.gradle.internal.logging.progress.ProgressLoggerFactory; + +import javax.inject.Inject; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Objects; +import java.util.function.UnaryOperator; + +/** + * An helper to manage a vagrant box. + * + * This is created alongside a {@link VagrantExtension} for a project to manage starting and + * stopping a single vagrant box. + */ +public class VagrantMachine { + + private final Project project; + private final VagrantExtension extension; + private final ReaperService reaper; + // pkg private so plugin can set this after construction + long refs; + private boolean isVMStarted = false; + + public VagrantMachine(Project project, VagrantExtension extension, ReaperService reaper) { + this.project = project; + this.extension = extension; + this.reaper = reaper; + } + + @Inject + protected ProgressLoggerFactory getProgressLoggerFactory() { + throw new UnsupportedOperationException(); + } + + public void execute(Action action) { + VagrantExecSpec vagrantSpec = new VagrantExecSpec(); + action.execute(vagrantSpec); + + Objects.requireNonNull(vagrantSpec.command); + + LoggedExec.exec(project, execSpec -> { + execSpec.setExecutable("vagrant"); + File vagrantfile = extension.getVagrantfile(); + execSpec.setEnvironment(System.getenv()); // pass through env + execSpec.environment("VAGRANT_CWD", vagrantfile.getParentFile().toString()); + execSpec.environment("VAGRANT_VAGRANTFILE", vagrantfile.getName()); + execSpec.environment("VAGRANT_LOG", "debug"); + extension.getHostEnv().forEach(execSpec::environment); + + execSpec.args(vagrantSpec.command); + if (vagrantSpec.subcommand != null) { + execSpec.args(vagrantSpec.subcommand); + } + execSpec.args(extension.getBox()); + if (vagrantSpec.args != null) { + execSpec.args(Arrays.asList(vagrantSpec.args)); + } + + UnaryOperator progressHandler = vagrantSpec.progressHandler; + if (progressHandler == null) { + progressHandler = new VagrantProgressLogger("==> " + extension.getBox() + ": "); + } + OutputStream output = execSpec.getStandardOutput(); + // output from vagrant needs to be manually curated because --machine-readable isn't actually "readable" + OutputStream progressStream = new ProgressOutputStream(vagrantSpec.command, progressHandler); + execSpec.setStandardOutput(new TeeOutputStream(output, progressStream)); + }); + } + + // start the configuration VM if it hasn't been started yet + void maybeStartVM() { + if (isVMStarted) { + return; + } + + execute(spec -> { + spec.setCommand("box"); + spec.setSubcommand("update"); + }); + + // Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag that defaults + // to true that can be used to control whether or not to destroy any test boxes before test execution. + boolean destroyVM = Util.getBooleanProperty("vagrant.destroy", true); + if (destroyVM) { + execute(spec -> { + spec.setCommand("destroy"); + spec.setArgs("--force"); + }); + } + + // register box to be shutdown if gradle dies + reaper.registerCommand(extension.getBox(), "vagrant", "halt", "-f", extension.getBox()); + + // We lock the provider to virtualbox because the Vagrantfile specifies lots of boxes that only work + // properly in virtualbox. Virtualbox is vagrant's default but its possible to change that default and folks do. + execute(spec -> { + spec.setCommand("up"); + spec.setArgs("--provision", "--provider", "virtualbox"); + }); + isVMStarted = true; + } + + // stops the VM if refs are down to 0, or force was called + void maybeStopVM(boolean force) { + assert refs >= 1; + this.refs--; + if ((refs == 0 || force) && isVMStarted) { + execute(spec -> spec.setCommand("halt")); + reaper.unregister(extension.getBox()); + } + } + + // convert the given path from an elasticsearch repo path to a VM path + public static String convertLinuxPath(Project project, String path) { + return "/elasticsearch/" + project.getRootDir().toPath().relativize(Paths.get(path)); + } + + public static String convertWindowsPath(Project project, String path) { + return "C:\\elasticsearch\\" + project.getRootDir().toPath().relativize(Paths.get(path)).toString().replace('/', '\\'); + } + + public static class VagrantExecSpec { + private String command; + private String subcommand; + private String[] args; + private UnaryOperator progressHandler; + + private VagrantExecSpec() {} + + public void setCommand(String command) { + this.command = command; + } + + public void setSubcommand(String subcommand) { + this.subcommand = subcommand; + } + + public void setArgs(String... args) { + this.args = args; + } + + /** + * A function to translate output from the vagrant command execution to the progress line. + * + * The function takes the current line of output from vagrant, and returns a new + * progress line, or {@code null} if there is no update. + */ + public void setProgressHandler(UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + } + } + + private class ProgressOutputStream extends LoggingOutputStream { + + private ProgressLogger progressLogger; + private UnaryOperator progressHandler; + + ProgressOutputStream(String command, UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + this.progressLogger = getProgressLoggerFactory().newOperation("vagrant"); + progressLogger.start(extension.getBox() + "> " + command, "hello"); + } + + @Override + protected void logLine(String line) { + String progress = progressHandler.apply(line); + if (progress != null) { + progressLogger.progress(progress); + } + System.out.println(line); + } + + @Override + public void close() { + progressLogger.completed(); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java similarity index 73% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java index 2e4a6123556..d041ebbda92 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java @@ -19,30 +19,23 @@ package org.elasticsearch.gradle.vagrant; -import org.elasticsearch.gradle.LoggingOutputStream; -import org.gradle.internal.logging.progress.ProgressLogger; +import java.util.function.UnaryOperator; + +public class VagrantProgressLogger implements UnaryOperator { -public class VagrantLoggerOutputStream extends LoggingOutputStream { private static final String HEADING_PREFIX = "==> "; - private final ProgressLogger progressLogger; private final String squashedPrefix; - private boolean isStarted = false; private String lastLine = ""; - private boolean inProgressReport = false; private String heading = ""; + private boolean inProgressReport = false; - VagrantLoggerOutputStream(ProgressLogger progressLogger, String squashedPrefix) { - this.progressLogger = progressLogger; + public VagrantProgressLogger(String squashedPrefix) { this.squashedPrefix = squashedPrefix; } @Override - protected void logLine(String line) { - if (isStarted == false) { - progressLogger.started("started"); - isStarted = true; - } + public String apply(String line) { if (line.startsWith("\r\u001b")) { /* We don't want to try to be a full terminal emulator but we want to keep the escape sequences from leaking and catch _some_ of the @@ -51,7 +44,7 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream { if ("[K".equals(line)) { inProgressReport = true; } - return; + return null; } if (line.startsWith(squashedPrefix)) { line = line.substring(squashedPrefix.length()); @@ -67,14 +60,8 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream { inProgressReport = false; line = lastLine + line; } else { - return; + return null; } - progressLogger.progress(line); - } - - @Override - public void close() { - flush(); - progressLogger.completed(); + return line; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java new file mode 100644 index 00000000000..12561712ccc --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.TaskAction; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; + +/** + * A shell script to run within a vagrant VM. + * + * The script is run as root within the VM. + */ +public abstract class VagrantShellTask extends DefaultTask { + + private final VagrantExtension extension; + private final VagrantMachine service; + private UnaryOperator progressHandler = UnaryOperator.identity(); + + public VagrantShellTask() { + extension = getProject().getExtensions().findByType(VagrantExtension.class); + if (extension == null) { + throw new IllegalStateException("elasticsearch.vagrant-base must be applied to create " + getClass().getName()); + } + service = getProject().getExtensions().getByType(VagrantMachine.class); + } + + @Input + protected abstract List getWindowsScript(); + + @Input + protected abstract List getLinuxScript(); + + @Input + public UnaryOperator getProgressHandler() { + return progressHandler; + } + + public void setProgressHandler(UnaryOperator progressHandler) { + this.progressHandler = progressHandler; + } + + @TaskAction + public void runScript() { + String rootDir = getProject().getRootDir().toString(); + if (extension.isWindowsVM()) { + service.execute(spec -> { + spec.setCommand("winrm"); + + List script = new ArrayList<>(); + script.add("try {"); + script.add("cd " + convertWindowsPath(getProject(), rootDir)); + extension.getVmEnv().forEach((k, v) -> script.add("$Env:" + k + " = \"" + v + "\"")); + script.addAll(getWindowsScript().stream().map(s -> " " + s).collect(Collectors.toList())); + script.addAll(Arrays.asList( + " exit $LASTEXITCODE", + "} catch {", + // catch if we have a failure to even run the script at all above, equivalent to set -e, sort of + " echo $_.Exception.Message", + " exit 1", + "}")); + spec.setArgs("--elevated", "--command", String.join("\n", script)); + spec.setProgressHandler(progressHandler); + }); + } else { + service.execute(spec -> { + spec.setCommand("ssh"); + + List script = new ArrayList<>(); + script.add("sudo bash -c '"); // start inline bash script + script.add("pwd"); + script.add("cd " + convertLinuxPath(getProject(), rootDir)); + extension.getVmEnv().forEach((k, v) -> script.add("export " + k + "=" + v)); + script.addAll(getLinuxScript()); + script.add("'"); // end inline bash script + spec.setArgs("--command", String.join("\n", script)); + spec.setProgressHandler(progressHandler); + }); + } + } + + +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties new file mode 100644 index 00000000000..90b2914e452 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.DistroTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties deleted file mode 100644 index 844310fa9d7..00000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.vagrant.VagrantTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties deleted file mode 100644 index 73a3f412349..00000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin \ No newline at end of file diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java index 873cdc7d7bf..343ead0bdf6 100644 --- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java @@ -16,10 +16,13 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Pattern; /** * A wrapper around gradle's Exec task to capture output and log on error. @@ -98,6 +101,8 @@ public class LoggedExec extends Exec { return genericExec(project, project::javaexec, action); } + private static final Pattern NEWLINE = Pattern.compile(System.lineSeparator()); + private static ExecResult genericExec( Project project, Function,ExecResult> function, @@ -107,19 +112,20 @@ public class LoggedExec extends Exec { return function.apply(action); } ByteArrayOutputStream output = new ByteArrayOutputStream(); - ByteArrayOutputStream error = new ByteArrayOutputStream(); try { return function.apply(spec -> { spec.setStandardOutput(output); - spec.setErrorOutput(error); + spec.setErrorOutput(output); action.execute(spec); + try { + output.write(("Output for " + spec.getExecutable() + ":").getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } }); } catch (Exception e) { try { - project.getLogger().error("Standard output:"); - project.getLogger().error(output.toString("UTF-8")); - project.getLogger().error("Standard error:"); - project.getLogger().error(error.toString("UTF-8")); + NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> project.getLogger().error("| " + s)); } catch (UnsupportedEncodingException ue) { throw new GradleException("Failed to read exec output", ue); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java index d83de5f2173..5f728392313 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java @@ -63,12 +63,12 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase { Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) { filebytes = stream.readAllBytes(); } - String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip"; + String urlPath = "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip"; wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200))); wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes))); wireMock.start(); - assertExtractedDistro("1.0.0", "archive", "windows", null, null, + assertExtractedDistro("7.0.0", "archive", "windows", null, null, "tests.download_service", wireMock.baseUrl()); } catch (Exception e) { // for debugging diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/10_basic.bats b/qa/vagrant/bats/default/10_basic.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/10_basic.bats rename to qa/vagrant/bats/default/10_basic.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats b/qa/vagrant/bats/default/20_tar_bootstrap_password.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats rename to qa/vagrant/bats/default/20_tar_bootstrap_password.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats b/qa/vagrant/bats/default/25_package_bootstrap_password.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats rename to qa/vagrant/bats/default/25_package_bootstrap_password.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats b/qa/vagrant/bats/default/30_tar_setup_passwords.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats rename to qa/vagrant/bats/default/30_tar_setup_passwords.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats b/qa/vagrant/bats/default/35_package_setup_passwords.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats rename to qa/vagrant/bats/default/35_package_setup_passwords.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats b/qa/vagrant/bats/default/40_tar_certgen.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats rename to qa/vagrant/bats/default/40_tar_certgen.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats b/qa/vagrant/bats/default/45_package_certgen.bats similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats rename to qa/vagrant/bats/default/45_package_certgen.bats diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash b/qa/vagrant/bats/default/bootstrap_password.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash rename to qa/vagrant/bats/default/bootstrap_password.bash diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/qa/vagrant/bats/default/certgen.bash similarity index 98% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash rename to qa/vagrant/bats/default/certgen.bash index 13aed28b4c1..b52e286742c 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash +++ b/qa/vagrant/bats/default/certgen.bash @@ -15,7 +15,6 @@ instances="/tmp/instances.yml" certificates="/tmp/certificates.zip" setup() { - export PACKAGE_NAME="elasticsearch" if [ $BATS_TEST_NUMBER == 1 ]; then clean_before_test fi @@ -176,6 +175,7 @@ NEW_PASS } @test "[$GROUP] create instances file" { + rm -f /tmp/instances.yml run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE" cat > /tmp/instances.yml <<- EOF instances: @@ -426,3 +426,8 @@ DATA_SETTINGS false } } + +@test "[$GROUP] remove Elasticsearch" { + # NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed + clean_before_test +} diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash b/qa/vagrant/bats/default/setup_passwords.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash rename to qa/vagrant/bats/default/setup_passwords.bash diff --git a/qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats b/qa/vagrant/bats/oss/25_tar_plugins.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/25_tar_plugins.bats rename to qa/vagrant/bats/oss/25_tar_plugins.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats b/qa/vagrant/bats/oss/50_modules_and_plugins.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/50_modules_and_plugins.bats rename to qa/vagrant/bats/oss/50_modules_and_plugins.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats b/qa/vagrant/bats/oss/70_sysv_initd.bats similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats rename to qa/vagrant/bats/oss/70_sysv_initd.bats diff --git a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats b/qa/vagrant/bats/oss/80_upgrade.bats similarity index 98% rename from qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats rename to qa/vagrant/bats/oss/80_upgrade.bats index 697e6456d1f..0c80751a58f 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats +++ b/qa/vagrant/bats/oss/80_upgrade.bats @@ -122,6 +122,7 @@ setup() { curl -s localhost:9200/library2/book/1?pretty | grep Darkness } -@test "[UPGRADE] stop version under test" { +@test "[UPGRADE] cleanup version under test" { stop_elasticsearch_service + clean_before_test } diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/bats/oss/module_and_plugin_test_cases.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash rename to qa/vagrant/bats/oss/module_and_plugin_test_cases.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/modules.bash b/qa/vagrant/bats/utils/modules.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/modules.bash rename to qa/vagrant/bats/utils/modules.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/bats/utils/packages.bash similarity index 99% rename from qa/vagrant/src/test/resources/packaging/utils/packages.bash rename to qa/vagrant/bats/utils/packages.bash index 5df432c35b3..2da3a02c543 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/bats/utils/packages.bash @@ -48,7 +48,7 @@ export_elasticsearch_paths() { export ESDATA="/var/lib/elasticsearch" export ESLOG="/var/log/elasticsearch" export ESENVFILE=$(env_file) - export PACKAGE_NAME=${PACKAGE_NAME:-"elasticsearch-oss"} + export PACKAGE_NAME } diff --git a/qa/vagrant/src/test/resources/packaging/utils/plugins.bash b/qa/vagrant/bats/utils/plugins.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/plugins.bash rename to qa/vagrant/bats/utils/plugins.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/bats/utils/tar.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/tar.bash rename to qa/vagrant/bats/utils/tar.bash diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/bats/utils/utils.bash similarity index 100% rename from qa/vagrant/src/test/resources/packaging/utils/utils.bash rename to qa/vagrant/bats/utils/utils.bash diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash b/qa/vagrant/bats/utils/xpack.bash similarity index 100% rename from x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash rename to qa/vagrant/bats/utils/xpack.bash diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index f5cfcdda03c..7cbc475e933 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -18,10 +18,7 @@ */ plugins { - id 'java' id 'elasticsearch.build' - id 'elasticsearch.vagrantsupport' - id 'elasticsearch.vagrant' } dependencies { @@ -36,34 +33,14 @@ dependencies { compile "commons-logging:commons-logging:${versions.commonslogging}" compile project(':libs:elasticsearch-core') - - // pulls in the jar built by this project and its dependencies - packagingTest project(path: project.path, configuration: 'runtime') } -List plugins = [] -for (Project subproj : project.rootProject.subprojects) { - if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) { - // add plugin as a dep - dependencies { - packaging project(path: "${subproj.path}", configuration: 'zip') - } - plugins.add(subproj.name) - } -} -plugins = plugins.toSorted() +configurations.create('testClasses') -setupPackagingTest { - doFirst { - File expectedPlugins = file('build/plugins/expected') - expectedPlugins.parentFile.mkdirs() - expectedPlugins.setText(plugins.join('\n'), 'UTF-8') - } -} - -esvagrant { - testClass 'org.elasticsearch.packaging.PackagingTests' -} +String classesDir = project.file(project.sourceSets.main.output.classesDirs.singleFile).toString() +artifacts.add('testClasses', project.layout.projectDirectory.dir(classesDir)) { + builtBy tasks.named('testClasses') +} forbiddenApisMain { replaceSignatureFiles 'jdk-signatures' @@ -91,3 +68,61 @@ tasks.thirdPartyAudit.ignoreMissingClasses ( 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener' ) + +boolean sample = project.properties.get('vagrant.boxes') != 'all' + +subprojects { Project platformProject -> + apply plugin: 'elasticsearch.distro-test' + apply plugin: 'java' + + configurations.create('testClasses') + dependencies { + testClasses project(path: ':qa:vagrant', configuration: 'testClasses') + testRuntime project(path: ':qa:vagrant', configuration: 'runtime') + } + + tasks.named('destructiveDistroTest') { + testClassesDirs += project.files(configurations.testClasses.singleFile) + } + + // TODO: remove this property lookup once CI is switched to use an explicit task for the sample tests + boolean allBoxes = project.properties.get('vagrant.boxes', '') == 'all' + if (allBoxes || ['centos-7', 'ubuntu-1604'].contains(platformProject.name)) { + tasks.register('packagingTest') { + dependsOn 'distroTest', 'batsTest.oss', 'batsTest.default' + } + } + + vagrant { + hostEnv 'VAGRANT_PROJECT_DIR', platformProject.projectDir.absolutePath + } + +} + +configurations { + allPlugins +} + +List plugins = [] +for (Project subproj : project.rootProject.subprojects) { + if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) { + // add plugin as a dep + dependencies { + allPlugins project(path: "${subproj.path}", configuration: 'zip') + } + plugins.add(subproj.name) + } +} +plugins = plugins.toSorted() + +copyPackagingArchives { + from configurations.allPlugins + doLast { + // TODO: this was copied from the old way bats tests get the plugins list. we should pass + // this in differently when converting to java tests + File expectedPlugins = file('build/plugins/expected') + expectedPlugins.parentFile.mkdirs() + expectedPlugins.setText(plugins.join('\n'), 'UTF-8') + } +} + diff --git a/qa/vagrant/centos-6/build.gradle b/qa/vagrant/centos-6/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/centos-7/build.gradle b/qa/vagrant/centos-7/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/debian-8/build.gradle b/qa/vagrant/debian-8/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/debian-9/build.gradle b/qa/vagrant/debian-9/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/fedora-28/build.gradle b/qa/vagrant/fedora-28/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/fedora-29/build.gradle b/qa/vagrant/fedora-29/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/oel-6/build.gradle b/qa/vagrant/oel-6/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/oel-7/build.gradle b/qa/vagrant/oel-7/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/opensuse-42/build.gradle b/qa/vagrant/opensuse-42/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/sles-12/build.gradle b/qa/vagrant/sles-12/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java deleted file mode 100644 index 06c978b823a..00000000000 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.packaging; - -import org.elasticsearch.packaging.test.DefaultDebBasicTests; -import org.elasticsearch.packaging.test.DefaultDebPreservationTests; -import org.elasticsearch.packaging.test.DefaultLinuxTarTests; -import org.elasticsearch.packaging.test.DefaultNoJdkDebBasicTests; -import org.elasticsearch.packaging.test.DefaultNoJdkLinuxTarTests; -import org.elasticsearch.packaging.test.DefaultNoJdkRpmBasicTests; -import org.elasticsearch.packaging.test.DefaultNoJdkWindowsZipTests; -import org.elasticsearch.packaging.test.DefaultRpmBasicTests; -import org.elasticsearch.packaging.test.DefaultRpmPreservationTests; -import org.elasticsearch.packaging.test.DefaultWindowsServiceTests; -import org.elasticsearch.packaging.test.DefaultWindowsZipTests; -import org.elasticsearch.packaging.test.OssDebBasicTests; -import org.elasticsearch.packaging.test.OssDebPreservationTests; -import org.elasticsearch.packaging.test.OssLinuxTarTests; -import org.elasticsearch.packaging.test.OssNoJdkDebBasicTests; -import org.elasticsearch.packaging.test.OssNoJdkLinuxTarTests; -import org.elasticsearch.packaging.test.OssNoJdkRpmBasicTests; -import org.elasticsearch.packaging.test.OssNoJdkWindowsZipTests; -import org.elasticsearch.packaging.test.OssRpmBasicTests; -import org.elasticsearch.packaging.test.OssRpmPreservationTests; -import org.elasticsearch.packaging.test.OssWindowsServiceTests; -import org.elasticsearch.packaging.test.OssWindowsZipTests; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.junit.runners.Suite.SuiteClasses; - -@RunWith(Suite.class) -@SuiteClasses({ - DefaultLinuxTarTests.class, - OssLinuxTarTests.class, - DefaultWindowsZipTests.class, - OssWindowsZipTests.class, - DefaultRpmBasicTests.class, - OssRpmBasicTests.class, - DefaultDebBasicTests.class, - OssDebBasicTests.class, - DefaultDebPreservationTests.class, - OssDebPreservationTests.class, - DefaultRpmPreservationTests.class, - OssRpmPreservationTests.class, - DefaultWindowsServiceTests.class, - OssWindowsServiceTests.class, - DefaultNoJdkLinuxTarTests.class, - OssNoJdkLinuxTarTests.class, - DefaultNoJdkWindowsZipTests.class, - OssNoJdkWindowsZipTests.class, - DefaultNoJdkRpmBasicTests.class, - OssNoJdkRpmBasicTests.class, - DefaultNoJdkDebBasicTests.class, - OssNoJdkDebBasicTests.class -}) -public class PackagingTests {} diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index e5cbbcc60cc..d6ba57ba075 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -20,7 +20,6 @@ package org.elasticsearch.packaging.test; import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.http.client.fluent.Request; import org.elasticsearch.packaging.util.Archives; import org.elasticsearch.packaging.util.Distribution; @@ -36,7 +35,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.Stream; -import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; @@ -225,17 +223,16 @@ public abstract class ArchiveTestCase extends PackagingTestCase { final Shell sh = new Shell(); // Create temporary directory with a space and link to java binary. // Use it as java_home - String nameWithSpace = RandomStrings.randomAsciiAlphanumOfLength(getRandom(), 10) + "java home"; - String test_java_home = FileUtils.mkdir(Paths.get("/home",ARCHIVE_OWNER, nameWithSpace)).toAbsolutePath().toString(); + String testJavaHome = FileUtils.mkdir(Paths.get("/home", ARCHIVE_OWNER, "java home")).toAbsolutePath().toString(); try { final String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); final String java = systemJavaHome + "/bin/java"; - sh.run("mkdir -p \"" + test_java_home + "/bin\""); - sh.run("ln -s \"" + java + "\" \"" + test_java_home + "/bin/java\""); - sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + test_java_home + "\""); + sh.run("mkdir -p \"" + testJavaHome + "/bin\""); + sh.run("ln -s \"" + java + "\" \"" + testJavaHome + "/bin/java\""); + sh.run("chown -R " + ARCHIVE_OWNER + ":" + ARCHIVE_OWNER + " \"" + testJavaHome + "\""); - sh.getEnv().put("JAVA_HOME", test_java_home); + sh.getEnv().put("JAVA_HOME", testJavaHome); //verify ES can start, stop and run plugin list Archives.runElasticsearch(installation, sh); @@ -246,7 +243,7 @@ public abstract class ArchiveTestCase extends PackagingTestCase { Result result = sh.run(pluginListCommand); assertThat(result.exitCode, equalTo(0)); } finally { - FileUtils.rm(Paths.get("\"" + test_java_home + "\"")); + FileUtils.rm(Paths.get(testJavaHome)); } }); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java index ca6c3e48d41..857fad55eea 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java @@ -45,10 +45,8 @@ import java.util.StringJoiner; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.core.IsNot.not; -import static org.hamcrest.text.IsEmptyString.isEmptyOrNullString; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -228,9 +226,7 @@ public class FileUtils { } public static Path getPackagingArchivesDir() { - String fromEnv = System.getenv("PACKAGING_ARCHIVES"); - assertThat(fromEnv, not(isEmptyOrNullString())); - return Paths.get(fromEnv); + return Paths.get(""); // tests are started in the packaging archives dir, ie the empty relative path } public static Path getDistributionFile(Distribution distribution) { diff --git a/qa/vagrant/ubuntu-1604/build.gradle b/qa/vagrant/ubuntu-1604/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/ubuntu-1804/build.gradle b/qa/vagrant/ubuntu-1804/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/vagrant/windows-2012r2/build.gradle b/qa/vagrant/windows-2012r2/build.gradle new file mode 100644 index 00000000000..f49de70eae7 --- /dev/null +++ b/qa/vagrant/windows-2012r2/build.gradle @@ -0,0 +1,11 @@ + +String boxId = project.properties.get('vagrant.windows-2012r2.id') +if (boxId != null) { + vagrant { + hostEnv 'VAGRANT_WINDOWS_2012R2_BOX', boxId + } +} else { + tasks.named('distroTest').configure { + onlyIf { false } + } +} diff --git a/qa/vagrant/windows-2016/build.gradle b/qa/vagrant/windows-2016/build.gradle new file mode 100644 index 00000000000..e0cfa1c6875 --- /dev/null +++ b/qa/vagrant/windows-2016/build.gradle @@ -0,0 +1,11 @@ + +String boxId = project.properties.get('vagrant.windows-2016.id') +if (boxId != null) { + vagrant { + hostEnv 'VAGRANT_WINDOWS_2016_BOX', boxId + } +} else { + tasks.named('distroTest').configure { + onlyIf { true } + } +} diff --git a/x-pack/qa/vagrant/build.gradle b/x-pack/qa/vagrant/build.gradle deleted file mode 100644 index 411b8d90c6d..00000000000 --- a/x-pack/qa/vagrant/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -apply plugin: 'elasticsearch.vagrantsupport' -apply plugin: 'elasticsearch.vagrant' - -esvagrant { - inheritTestUtils true -} - -dependencies { - // Inherit Bats test utils from :qa:vagrant project - packaging project(path: ':qa:vagrant', configuration: 'packaging') -}