honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-04-03 22:08:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert packaging upgrade tests to java () ()

Browse Source 
This commit removes the last of the bats tests, converting the rpm/deb
upgrade tests to java. It adds a new pattern of tasks, similar in nature
but separate from the existing distro tests, named `distroUpgradeTest`.
For each index compatible version, a `distroUpgradeTest.VERSION` task
exxists. Each distribution then has a task, named
`distroUpgradeTest.VERSION.DISTRO`.

One thing to note is these new tests do not cover no-jdk versions of
the rpm/deb packages, since the distribution/bwc project does not
currently build those.

closes 
closes
This commit is contained in:
Ryan Ernst 2020-08-05 14:05:37 -07:00 committed by GitHub
parent 3a9bf33993
commit 7514526b6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 356 additions and 1618 deletions

5
buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@ -244,11 +244,6 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
return configuration.iterator(); return configuration.iterator();
} }
// TODO: remove this when distro tests are per distribution
public Configuration getConfiguration() {
return configuration;
}
// internal, make this distribution's configuration unmodifiable // internal, make this distribution's configuration unmodifiable
void finalizeValues() { void finalizeValues() {

131
buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java

@ -1,131 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.Directory;
import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.TaskAction;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class BatsTestTask extends DefaultTask {
private final DirectoryProperty testsDir;
private final DirectoryProperty utilsDir;
private final DirectoryProperty distributionsDir;
private final DirectoryProperty pluginsDir;
private final DirectoryProperty upgradeDir;
private String packageName;
public BatsTestTask() {
this.testsDir = getProject().getObjects().directoryProperty();
this.utilsDir = getProject().getObjects().directoryProperty();
this.distributionsDir = getProject().getObjects().directoryProperty();
this.pluginsDir = getProject().getObjects().directoryProperty();
this.upgradeDir = getProject().getObjects().directoryProperty();
}
@InputDirectory
public Provider<Directory> getTestsDir() {
return testsDir;
}
public void setTestsDir(Directory testsDir) {
this.testsDir.set(testsDir);
}
@InputDirectory
public Provider<Directory> getUtilsDir() {
return utilsDir;
}
public void setUtilsDir(Directory utilsDir) {
this.utilsDir.set(utilsDir);
}
@InputDirectory
public Provider<Directory> getDistributionsDir() {
return distributionsDir;
}
public void setDistributionsDir(Provider<Directory> distributionsDir) {
this.distributionsDir.set(distributionsDir);
}
@InputDirectory
@Optional
public Provider<Directory> getPluginsDir() {
return pluginsDir;
}
public void setPluginsDir(Provider<Directory> pluginsDir) {
this.pluginsDir.set(pluginsDir);
}
@InputDirectory
@Optional
public Provider<Directory> getUpgradeDir() {
return upgradeDir;
}
public void setUpgradeDir(Provider<Directory> upgradeDir) {
this.upgradeDir.set(upgradeDir);
}
@Input
public String getPackageName() {
return packageName;
}
public void setPackageName(String packageName) {
this.packageName = packageName;
}
@TaskAction
public void runBats() {
List<Object> command = new ArrayList<>();
command.add("bats");
command.add("--tap");
command.addAll(
testsDir.getAsFileTree().getFiles().stream().filter(f -> f.getName().endsWith(".bats")).sorted().collect(Collectors.toList())
);
getProject().exec(spec -> {
spec.setWorkingDir(distributionsDir.getAsFile());
spec.environment(System.getenv());
spec.environment("BATS_TESTS", testsDir.getAsFile().get().toString());
spec.environment("BATS_UTILS", utilsDir.getAsFile().get().toString());
if (pluginsDir.isPresent()) {
spec.environment("BATS_PLUGINS", pluginsDir.getAsFile().get().toString());
}
if (upgradeDir.isPresent()) {
spec.environment("BATS_UPGRADE", upgradeDir.getAsFile().get().toString());
}
spec.environment("PACKAGE_NAME", packageName);
spec.setCommandLine(command);
});
}
}

413
buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@ -20,7 +20,6 @@
package org.elasticsearch.gradle.test; package org.elasticsearch.gradle.test;
import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.Architecture;
import org.elasticsearch.gradle.BwcVersions;
import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.DistributionDownloadPlugin;
import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.ElasticsearchDistribution;
import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
@ -28,6 +27,7 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
import org.elasticsearch.gradle.ElasticsearchDistribution.Type; import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
import org.elasticsearch.gradle.Jdk; import org.elasticsearch.gradle.Jdk;
import org.elasticsearch.gradle.JdkDownloadPlugin; import org.elasticsearch.gradle.JdkDownloadPlugin;
import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider;
import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.Version;
import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.VersionProperties;
import org.elasticsearch.gradle.docker.DockerSupportPlugin; import org.elasticsearch.gradle.docker.DockerSupportPlugin;
@ -35,37 +35,28 @@ import org.elasticsearch.gradle.docker.DockerSupportService;
import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.info.BuildParams;
import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
import org.elasticsearch.gradle.util.GradleUtils; import org.elasticsearch.gradle.util.GradleUtils;
import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
import org.elasticsearch.gradle.vagrant.VagrantExtension; import org.elasticsearch.gradle.vagrant.VagrantExtension;
import org.gradle.api.Action;
import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin; import org.gradle.api.Plugin;
import org.gradle.api.Project; import org.gradle.api.Project;
import org.gradle.api.Task; import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.dsl.DependencyHandler;
import org.gradle.api.file.Directory;
import org.gradle.api.plugins.ExtraPropertiesExtension;
import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaBasePlugin;
import org.gradle.api.provider.Provider; import org.gradle.api.provider.Provider;
import org.gradle.api.specs.Specs; import org.gradle.api.specs.Specs;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.TaskInputs;
import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.TaskProvider;
import org.gradle.api.tasks.testing.Test; import org.gradle.api.tasks.testing.Test;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath;
@ -78,13 +69,10 @@ public class DistroTestPlugin implements Plugin<Project> {
private static final String GRADLE_JDK_VENDOR = "openjdk"; private static final String GRADLE_JDK_VENDOR = "openjdk";
// all distributions used by distro tests. this is temporary until tests are per distribution // all distributions used by distro tests. this is temporary until tests are per distribution
private static final String DISTRIBUTIONS_CONFIGURATION = "distributions";
private static final String UPGRADE_CONFIGURATION = "upgradeDistributions";
private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin"; private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin";
private static final String COPY_DISTRIBUTIONS_TASK = "copyDistributions";
private static final String COPY_UPGRADE_TASK = "copyUpgradePackages";
private static final String IN_VM_SYSPROP = "tests.inVM"; private static final String IN_VM_SYSPROP = "tests.inVM";
private static final String DISTRIBUTION_SYSPROP = "tests.distribution"; private static final String DISTRIBUTION_SYSPROP = "tests.distribution";
private static final String BWC_DISTRIBUTION_SYSPROP = "tests.bwc-distribution";
private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin"; private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin";
@Override @Override
@ -100,34 +88,74 @@ public class DistroTestPlugin implements Plugin<Project> {
// TODO: it would be useful to also have the SYSTEM_JAVA_HOME setup in the root project, so that running from GCP only needs // TODO: it would be useful to also have the SYSTEM_JAVA_HOME setup in the root project, so that running from GCP only needs
// a java for gradle to run, and the tests are self sufficient and consistent with the java they use // a java for gradle to run, and the tests are self sufficient and consistent with the java they use
NamedDomainObjectContainer<ElasticsearchDistribution> allDistributions = DistributionDownloadPlugin.getContainer(project);
List<ElasticsearchDistribution> testDistributions = configureDistributions(project);
Version upgradeVersion = getUpgradeVersion(project); Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest");
Provider<Directory> distributionsDir = project.getLayout().getBuildDirectory().dir("packaging/distributions"); Map<String, TaskProvider<?>> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest");
Provider<Directory> upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade");
List<ElasticsearchDistribution> distributions = configureDistributions(project, upgradeVersion);
TaskProvider<Copy> copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir);
TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = lifecyleTasks(project, "destructiveDistroTest");
TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
Configuration examplePlugin = configureExamplePlugin(project); Configuration examplePlugin = configureExamplePlugin(project);
for (ElasticsearchDistribution distribution : distributions) { List<TaskProvider<Test>> windowsTestTasks = new ArrayList<>();
TaskProvider<?> destructiveTask = configureDistroTest(project, distribution, dockerSupport, examplePlugin); Map<Type, List<TaskProvider<Test>>> linuxTestTasks = new HashMap<>();
destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); Map<String, List<TaskProvider<Test>>> upgradeTestTasks = new HashMap<>();
lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); Map<String, TaskProvider<?>> depsTasks = new HashMap<>();
} for (ElasticsearchDistribution distribution : testDistributions) {
String taskname = destructiveDistroTestTaskName(distribution);
TaskProvider<?> depsTask = project.getTasks().register(taskname + "#deps");
depsTask.configure(t -> t.dependsOn(distribution, examplePlugin));
depsTasks.put(taskname, depsTask);
TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable);
addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString);
addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
t.exclude("**/PackageUpgradeTests.class");
}, depsTask);
TaskProvider<BatsTestTask> batsUpgradeTest = configureBatsTest( if (distribution.getPlatform() == Platform.WINDOWS) {
project, windowsTestTasks.add(destructiveTask);
"upgrade", } else {
distributionsDir, linuxTestTasks.computeIfAbsent(distribution.getType(), k -> new ArrayList<>()).add(destructiveTask);
copyDistributionsTask, }
copyUpgradeTask destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
); lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
batsUpgradeTest.configure(t -> t.setUpgradeDir(upgradeDir));
if ((distribution.getType() == Type.DEB || distribution.getType() == Type.RPM) && distribution.getBundledJdk()) {
for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) {
if (distribution.getFlavor() == Flavor.OSS && version.before("6.3.0")) {
continue; // before opening xpack
}
final ElasticsearchDistribution bwcDistro;
if (version.equals(Version.fromString(distribution.getVersion()))) {
// this is the same as the distribution we are testing
bwcDistro = distribution;
} else {
bwcDistro = createDistro(
allDistributions,
distribution.getArchitecture(),
distribution.getType(),
distribution.getPlatform(),
distribution.getFlavor(),
distribution.getBundledJdk(),
version.toString()
);
}
String upgradeTaskname = destructiveDistroUpgradeTestTaskName(distribution, version.toString());
TaskProvider<?> upgradeDepsTask = project.getTasks().register(upgradeTaskname + "#deps");
upgradeDepsTask.configure(t -> t.dependsOn(distribution, bwcDistro));
depsTasks.put(upgradeTaskname, upgradeDepsTask);
TaskProvider<Test> upgradeTest = configureTestTask(project, upgradeTaskname, distribution, t -> {
addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString);
addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::toString);
t.include("**/PackageUpgradeTests.class");
}, upgradeDepsTask);
versionTasks.get(version.toString()).configure(t -> t.dependsOn(upgradeTest));
upgradeTestTasks.computeIfAbsent(version.toString(), k -> new ArrayList<>()).add(upgradeTest);
}
}
}
project.subprojects(vmProject -> { project.subprojects(vmProject -> {
vmProject.getPluginManager().apply(VagrantBasePlugin.class); vmProject.getPluginManager().apply(VagrantBasePlugin.class);
@ -135,23 +163,26 @@ public class DistroTestPlugin implements Plugin<Project> {
List<Object> vmDependencies = new ArrayList<>(configureVM(vmProject)); List<Object> vmDependencies = new ArrayList<>(configureVM(vmProject));
vmDependencies.add(project.getConfigurations().getByName("testRuntimeClasspath")); vmDependencies.add(project.getConfigurations().getByName("testRuntimeClasspath"));
Map<ElasticsearchDistribution.Type, TaskProvider<?>> vmLifecyleTasks = lifecyleTasks(vmProject, "distroTest"); Map<ElasticsearchDistribution.Type, TaskProvider<?>> vmLifecyleTasks = lifecycleTasks(vmProject, "distroTest");
Map<String, TaskProvider<?>> vmVersionTasks = versionTasks(vmProject, "distroUpgradeTest");
TaskProvider<Task> distroTest = vmProject.getTasks().register("distroTest"); TaskProvider<Task> distroTest = vmProject.getTasks().register("distroTest");
for (ElasticsearchDistribution distribution : distributions) {
String destructiveTaskName = destructiveDistroTestTaskName(distribution);
Platform platform = distribution.getPlatform();
// this condition ensures windows boxes get windows distributions, and linux boxes get linux distributions
if (isWindows(vmProject) == (platform == Platform.WINDOWS)) {
TaskProvider<GradleDistroTestTask> vmTask = configureVMWrapperTask(
vmProject,
distribution.getName() + " distribution",
destructiveTaskName,
vmDependencies
);
vmTask.configure(t -> t.dependsOn(distribution, examplePlugin));
vmLifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(vmTask)); // windows boxes get windows distributions, and linux boxes get linux distributions
distroTest.configure(t -> { if (isWindows(vmProject)) {
configureVMWrapperTasks(
vmProject,
windowsTestTasks,
depsTasks,
wrapperTask -> { vmLifecyleTasks.get(Type.ARCHIVE).configure(t -> t.dependsOn(wrapperTask)); },
vmDependencies
);
} else {
for (var entry : linuxTestTasks.entrySet()) {
Type type = entry.getKey();
TaskProvider<?> vmLifecycleTask = vmLifecyleTasks.get(type);
configureVMWrapperTasks(vmProject, entry.getValue(), depsTasks, wrapperTask -> {
vmLifecycleTask.configure(t -> t.dependsOn(wrapperTask));
// Only VM sub-projects that are specifically opted-in to testing Docker should // Only VM sub-projects that are specifically opted-in to testing Docker should
// have the Docker task added as a dependency. Although we control whether Docker // have the Docker task added as a dependency. Although we control whether Docker
// is installed in the VM via `Vagrantfile` and we could auto-detect its presence // is installed in the VM via `Vagrantfile` and we could auto-detect its presence
@ -160,26 +191,30 @@ public class DistroTestPlugin implements Plugin<Project> {
// auto-detection doesn't work. // auto-detection doesn't work.
// //
// The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals() // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals()
boolean shouldExecute = distribution.getType() != Type.DOCKER boolean shouldExecute = type != Type.DOCKER || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker"));
|| Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker"));
if (shouldExecute) { if (shouldExecute) {
t.dependsOn(vmTask); distroTest.configure(t -> t.dependsOn(wrapperTask));
} }
}); }, vmDependencies);
}
for (var entry : upgradeTestTasks.entrySet()) {
String version = entry.getKey();
TaskProvider<?> vmVersionTask = vmVersionTasks.get(version);
configureVMWrapperTasks(
vmProject,
entry.getValue(),
depsTasks,
wrapperTask -> { vmVersionTask.configure(t -> t.dependsOn(wrapperTask)); },
vmDependencies
);
} }
} }
configureVMWrapperTask(vmProject, "bats upgrade", batsUpgradeTest.getName(), vmDependencies).configure(t -> {
t.setProgressHandler(new BatsProgressLogger(project.getLogger()));
t.onlyIf(spec -> isWindows(vmProject) == false); // bats doesn't run on windows
t.dependsOn(copyDistributionsTask, copyUpgradeTask);
});
}); });
} }
private static Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks(Project project, String taskPrefix) { private static Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecycleTasks(Project project, String taskPrefix) {
Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = new HashMap<>(); Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = new HashMap<>();
lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker")); lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker"));
@ -190,6 +225,16 @@ public class DistroTestPlugin implements Plugin<Project> {
return lifecyleTasks; return lifecyleTasks;
} }
private static Map<String, TaskProvider<?>> versionTasks(Project project, String taskPrefix) {
Map<String, TaskProvider<?>> versionTasks = new HashMap<>();
for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) {
versionTasks.put(version.toString(), project.getTasks().register(taskPrefix + ".v" + version));
}
return versionTasks;
}
private static Jdk createJdk( private static Jdk createJdk(
NamedDomainObjectContainer<Jdk> jdksContainer, NamedDomainObjectContainer<Jdk> jdksContainer,
String name, String name,
@ -206,27 +251,6 @@ public class DistroTestPlugin implements Plugin<Project> {
return jdk; return jdk;
} }
private static Version getUpgradeVersion(Project project) {
String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion");
if (upgradeFromVersionRaw != null) {
return Version.fromString(upgradeFromVersionRaw);
}
// was not passed in, so randomly choose one from bwc versions
ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class);
if ((boolean) extraProperties.get("bwc_tests_enabled") == false) {
// Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests
return Version.fromString(project.getVersion().toString());
}
String firstPartOfSeed = BuildParams.getTestSeed().split(":")[0];
final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16);
BwcVersions bwcVersions = BuildParams.getBwcVersions();
final List<Version> indexCompatVersions = bwcVersions.getIndexCompatible();
return indexCompatVersions.get(new Random(seed).nextInt(indexCompatVersions.size()));
}
private static List<Object> configureVM(Project project) { private static List<Object> configureVM(Project project) {
String box = project.getName(); String box = project.getName();
@ -263,63 +287,6 @@ public class DistroTestPlugin implements Plugin<Project> {
}; };
} }
private static TaskProvider<Copy> configureCopyDistributionsTask(Project project, Provider<Directory> distributionsDir) {
// temporary, until we have tasks per distribution
return project.getTasks().register(COPY_DISTRIBUTIONS_TASK, Copy.class, t -> {
t.into(distributionsDir);
t.from(project.getConfigurations().getByName(DISTRIBUTIONS_CONFIGURATION));
Path distributionsPath = distributionsDir.get().getAsFile().toPath();
TaskInputs inputs = t.getInputs();
inputs.property("version", VersionProperties.getElasticsearch());
t.doLast(action -> {
try {
Files.write(distributionsPath.resolve("version"), VersionProperties.getElasticsearch().getBytes());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
});
}
private static TaskProvider<Copy> configureCopyUpgradeTask(Project project, Version upgradeVersion, Provider<Directory> upgradeDir) {
// temporary, until we have tasks per distribution
return project.getTasks().register(COPY_UPGRADE_TASK, Copy.class, t -> {
t.into(upgradeDir);
t.from(project.getConfigurations().getByName(UPGRADE_CONFIGURATION));
Path upgradePath = upgradeDir.get().getAsFile().toPath();
// write bwc version, and append -SNAPSHOT if it is an unreleased version
BwcVersions bwcVersions = BuildParams.getBwcVersions();
final String upgradeFromVersion;
if (bwcVersions.unreleasedInfo(upgradeVersion) != null) {
upgradeFromVersion = upgradeVersion.toString() + "-SNAPSHOT";
} else {
upgradeFromVersion = upgradeVersion.toString();
}
TaskInputs inputs = t.getInputs();
inputs.property("upgrade_from_version", upgradeFromVersion);
// TODO: this is serializable, need to think how to represent this as an input
// inputs.property("bwc_versions", bwcVersions);
t.doLast(action -> {
try {
Files.write(upgradePath.resolve("version"), VersionProperties.getElasticsearch().getBytes());
Files.write(upgradePath.resolve("upgrade_from_version"), upgradeFromVersion.getBytes());
Path upgradeMarkerPath = upgradePath.resolve("upgrade_is_oss");
project.delete(upgradeMarkerPath);
// this is always true, but bats tests rely on it. It is just temporary until bats is removed.
if (upgradeVersion.onOrAfter("6.3.0")) {
Files.write(upgradeMarkerPath, new byte[0]);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
});
}
private static Configuration configureExamplePlugin(Project project) { private static Configuration configureExamplePlugin(Project project) {
Configuration examplePlugin = project.getConfigurations().create(EXAMPLE_PLUGIN_CONFIGURATION); Configuration examplePlugin = project.getConfigurations().create(EXAMPLE_PLUGIN_CONFIGURATION);
DependencyHandler deps = project.getDependencies(); DependencyHandler deps = project.getDependencies();
@ -328,69 +295,52 @@ public class DistroTestPlugin implements Plugin<Project> {
return examplePlugin; return examplePlugin;
} }
private static TaskProvider<GradleDistroTestTask> configureVMWrapperTask( private static void configureVMWrapperTasks(
Project project, Project project,
String type, List<TaskProvider<Test>> destructiveTasks,
String destructiveTaskPath, Map<String, TaskProvider<?>> depsTasks,
List<Object> dependsOn Action<TaskProvider<GradleDistroTestTask>> configure,
Object... additionalDeps
) { ) {
int taskNameStart = destructiveTaskPath.lastIndexOf(':') + "destructive".length() + 1; for (TaskProvider<? extends Task> destructiveTask : destructiveTasks) {
String taskname = destructiveTaskPath.substring(taskNameStart); String destructiveTaskName = destructiveTask.getName();
taskname = taskname.substring(0, 1).toLowerCase(Locale.ROOT) + taskname.substring(1); String taskname = destructiveTaskName.substring("destructive".length());
return project.getTasks().register(taskname, GradleDistroTestTask.class, t -> { taskname = taskname.substring(0, 1).toLowerCase(Locale.ROOT) + taskname.substring(1);
t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); TaskProvider<GradleDistroTestTask> vmTask = project.getTasks().register(taskname, GradleDistroTestTask.class, t -> {
t.setDescription("Runs " + type + " tests within vagrant"); t.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
t.setTaskName(destructiveTaskPath); t.setDescription("Runs " + destructiveTaskName.split("\\.", 2)[1] + " tests within vagrant");
t.extraArg("-D'" + IN_VM_SYSPROP + "'"); t.setTaskName(destructiveTaskName);
t.dependsOn(dependsOn); t.extraArg("-D'" + IN_VM_SYSPROP + "'");
}); t.dependsOn(depsTasks.get(destructiveTaskName));
t.dependsOn(additionalDeps);
});
configure.execute(vmTask);
}
} }
private static TaskProvider<?> configureDistroTest( private static TaskProvider<Test> configureTestTask(
Project project, Project project,
String taskname,
ElasticsearchDistribution distribution, ElasticsearchDistribution distribution,
Provider<DockerSupportService> dockerSupport, Action<? super Test> configure,
Configuration examplePlugin Object... deps
) { ) {
return project.getTasks().register(destructiveDistroTestTaskName(distribution), Test.class, t -> { return project.getTasks().register(taskname, Test.class, t -> {
// Disable Docker distribution tests unless a Docker installation is available
t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable);
// Only run tests for the current architecture // Only run tests for the current architecture
t.onlyIf(t3 -> distribution.getArchitecture() == Architecture.current()); t.onlyIf(t3 -> distribution.getArchitecture() == Architecture.current());
t.getOutputs().doNotCacheIf("Build cache is disabled for packaging tests", Specs.satisfyAll()); t.getOutputs().doNotCacheIf("Build cache is disabled for packaging tests", Specs.satisfyAll());
t.setMaxParallelForks(1); t.setMaxParallelForks(1);
t.setWorkingDir(project.getProjectDir()); t.setWorkingDir(project.getProjectDir());
t.systemProperty(DISTRIBUTION_SYSPROP, distribution.toString());
t.systemProperty(EXAMPLE_PLUGIN_SYSPROP, examplePlugin.getSingleFile().toString());
if (System.getProperty(IN_VM_SYSPROP) == null) {
t.dependsOn(distribution);
t.dependsOn(examplePlugin);
}
});
}
private static TaskProvider<BatsTestTask> configureBatsTest(
Project project,
String type,
Provider<Directory> distributionsDir,
Object... deps
) {
return project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class, t -> {
Directory batsDir = project.getLayout().getProjectDirectory().dir("bats");
t.setTestsDir(batsDir.dir(type));
t.setUtilsDir(batsDir.dir("utils"));
t.setDistributionsDir(distributionsDir);
t.setPackageName("elasticsearch" + (type.equals("oss") ? "-oss" : ""));
if (System.getProperty(IN_VM_SYSPROP) == null) { if (System.getProperty(IN_VM_SYSPROP) == null) {
t.dependsOn(deps); t.dependsOn(deps);
} }
configure.execute(t);
}); });
} }
private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion) { private List<ElasticsearchDistribution> configureDistributions(Project project) {
NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project); NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
List<ElasticsearchDistribution> currentDistros = new ArrayList<>(); List<ElasticsearchDistribution> currentDistros = new ArrayList<>();
List<ElasticsearchDistribution> upgradeDistros = new ArrayList<>();
for (Architecture architecture : Architecture.values()) { for (Architecture architecture : Architecture.values()) {
for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) { for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) {
@ -401,35 +351,20 @@ public class DistroTestPlugin implements Plugin<Project> {
boolean skip = bundledJdk == false && (type == Type.DOCKER || architecture == Architecture.AARCH64); boolean skip = bundledJdk == false && (type == Type.DOCKER || architecture == Architecture.AARCH64);
if (skip == false) { if (skip == false) {
addDistro( currentDistros.add(
distributions, createDistro(
architecture, distributions,
type, architecture,
null, type,
flavor, null,
bundledJdk, flavor,
VersionProperties.getElasticsearch(), bundledJdk,
currentDistros VersionProperties.getElasticsearch()
)
); );
} }
} }
} }
// We don't configure distributions for prior versions for Docker. This is because doing
// so prompts Gradle to try and resolve the Docker dependencies, which doesn't work as
// they can't be downloaded via Ivy (configured in DistributionDownloadPlugin). Since we
// need these for the BATS upgrade tests, and those tests only cover .rpm and .deb, it's
// OK to omit creating such distributions in the first place. We may need to revisit
// this in the future, so allow upgrade testing using Docker containers.
if (type != Type.DOCKER) {
// upgrade version is always bundled jdk
// NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced
// witha dedicated upgrade test from every bwc version like other bwc tests
addDistro(distributions, architecture, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros);
if (upgradeVersion.onOrAfter("6.3.0")) {
addDistro(distributions, architecture, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros);
}
}
} }
} }
@ -443,52 +378,35 @@ public class DistroTestPlugin implements Plugin<Project> {
continue; continue;
} }
addDistro( currentDistros.add(
distributions, createDistro(
architecture, distributions,
Type.ARCHIVE, architecture,
platform, Type.ARCHIVE,
flavor, platform,
bundledJdk, flavor,
VersionProperties.getElasticsearch(), bundledJdk,
currentDistros VersionProperties.getElasticsearch()
)
); );
} }
} }
} }
} }
// temporary until distro tests have one test per distro
Configuration packagingConfig = project.getConfigurations().create(DISTRIBUTIONS_CONFIGURATION);
List<Configuration> distroConfigs = currentDistros.stream()
.filter(d -> d.getType() != Type.DOCKER)
.map(ElasticsearchDistribution::getConfiguration)
.collect(Collectors.toList());
packagingConfig.setExtendsFrom(distroConfigs);
Configuration packagingUpgradeConfig = project.getConfigurations().create(UPGRADE_CONFIGURATION);
List<Configuration> distroUpgradeConfigs = upgradeDistros.stream()
.map(ElasticsearchDistribution::getConfiguration)
.collect(Collectors.toList());
packagingUpgradeConfig.setExtendsFrom(distroUpgradeConfigs);
return currentDistros; return currentDistros;
} }
private static void addDistro( private static ElasticsearchDistribution createDistro(
NamedDomainObjectContainer<ElasticsearchDistribution> distributions, NamedDomainObjectContainer<ElasticsearchDistribution> distributions,
Architecture architecture, Architecture architecture,
Type type, Type type,
Platform platform, Platform platform,
Flavor flavor, Flavor flavor,
boolean bundledJdk, boolean bundledJdk,
String version, String version
List<ElasticsearchDistribution> container
) { ) {
String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version; String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version;
if (distributions.findByName(name) != null) {
return;
}
ElasticsearchDistribution distro = distributions.create(name, d -> { ElasticsearchDistribution distro = distributions.create(name, d -> {
d.setArchitecture(architecture); d.setArchitecture(architecture);
d.setFlavor(flavor); d.setFlavor(flavor);
@ -508,7 +426,7 @@ public class DistroTestPlugin implements Plugin<Project> {
distro.setFailIfUnavailable(false); distro.setFailIfUnavailable(false);
} }
container.add(distro); return distro;
} }
// return true if the project is for a windows VM, false otherwise // return true if the project is for a windows VM, false otherwise
@ -530,4 +448,17 @@ public class DistroTestPlugin implements Plugin<Project> {
return "destructiveDistroTest." return "destructiveDistroTest."
+ distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture());
} }
private static String destructiveDistroUpgradeTestTaskName(ElasticsearchDistribution distro, String bwcVersion) {
Type type = distro.getType();
return "destructiveDistroUpgradeTest.v"
+ bwcVersion
+ "."
+ distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture());
}
private static void addDistributionSysprop(Test task, String sysprop, Supplier<String> valueSupplier) {
SystemPropertyCommandLineArgumentProvider props = task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class);
props.systemProperty(sysprop, valueSupplier);
}
} }

105
buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java

@ -1,105 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.vagrant;
import org.gradle.api.logging.Logger;
import java.util.Formatter;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Adapts an OutputStream containing TAP output from bats into a ProgressLogger and a Logger.
*
* TAP (Test Anything Protocol, https://testanything.org) is used by BATS for its output format.
*
* Every test output goes to the ProgressLogger and all failures
* and non-test output goes to the Logger. That means you can always glance
* at the result of the last test and the cumulative pass/fail/skip stats and
* the failures are all logged.
*
* There is a Tap4j project but we can't use it because it wants to parse the
* entire TAP stream at once and won't parse it stream-wise.
*/
public class BatsProgressLogger implements UnaryOperator<String> {
private static final Pattern lineRegex = Pattern.compile(
"(?<status>ok|not ok) \\d+(?<skip> # skip (?<skipReason>\\(.+\\))?)? \\[(?<suite>.+)\\] (?<test>.+)"
);
private static final Pattern startRegex = Pattern.compile("1..(\\d+)");
private final Logger logger;
private int testsCompleted = 0;
private int testsFailed = 0;
private int testsSkipped = 0;
private Integer testCount;
private String countsFormat;
public BatsProgressLogger(Logger logger) {
this.logger = logger;
}
@Override
public String apply(String line) {
if (testCount == null) {
Matcher m = startRegex.matcher(line);
if (m.matches() == false) {
// haven't reached start of bats test yet, pass through whatever we see
return line;
}
testCount = Integer.parseInt(m.group(1));
int length = String.valueOf(testCount).length();
String count = "%0" + length + "d";
countsFormat = "[" + count + "|" + count + "|" + count + "/" + count + "]";
return null;
}
Matcher m = lineRegex.matcher(line);
if (m.matches() == false) {
/* These might be failure report lines or comments or whatever. Its hard
to tell and it doesn't matter. */
logger.warn(line);
return null;
}
boolean skipped = m.group("skip") != null;
boolean success = skipped == false && m.group("status").equals("ok");
String skipReason = m.group("skipReason");
String suiteName = m.group("suite");
String testName = m.group("test");
final String status;
if (skipped) {
status = "SKIPPED";
testsSkipped++;
} else if (success) {
status = " OK";
testsCompleted++;
} else {
status = " FAILED";
testsFailed++;
}
String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString();
if (success == false) {
logger.warn(line);
}
return "BATS " + counts + ", " + status + " [" + suiteName + "] " + testName;
}
}

130
qa/os/bats/upgrade/80_upgrade.bats

@ -1,130 +0,0 @@
#!/usr/bin/env bats
# Tests upgrading elasticsearch from a previous version with the deb or rpm
# packages. Just uses a single node cluster on the current machine rather than
# fancy rolling restarts.
# WARNING: This testing file must be executed as root and can
# dramatically change your system. It should only be executed
# in a throw-away VM like those made by the Vagrantfile at
# the root of the Elasticsearch source code. This should
# cause the script to fail if it is executed any other way:
[ -f /etc/is_vagrant_vm ] || {
>&2 echo "must be run on a vagrant VM"
exit 1
}
# The test case can be executed with the Bash Automated
# Testing System tool available at https://github.com/sstephenson/bats
# Thanks to Sam Stephenson!
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Load test utilities
load $BATS_UTILS/utils.bash
load $BATS_UTILS/packages.bash
# Cleans everything for the 1st execution
setup() {
skip_not_dpkg_or_rpm
sameVersion="false"
if [ "$(cat $BATS_UPGRADE/upgrade_from_version)" == "$(cat version)" ]; then
sameVersion="true"
else
echo "BWC test version: $(cat $BATS_UPGRADE/upgrade_from_version)"
fi
# TODO: this needs to conditionally change based on version > 6.3.0
if [ -f $BATS_UPGRADE/upgrade_is_oss ]; then
export PACKAGE_NAME="elasticsearch-oss"
else
skip "upgrade cannot happen from pre 6.3.0 to elasticsearch-oss"
fi
}
@test "[UPGRADE] install old version" {
clean_before_test
install_package -v $(cat $BATS_UPGRADE/upgrade_from_version) -d $BATS_UPGRADE
}
@test "[UPGRADE] modify keystore" {
# deliberately modify the keystore to force it to be preserved during package upgrade
export_elasticsearch_paths
sudo -E "$ESHOME/bin/elasticsearch-keystore" remove keystore.seed
sudo -E echo keystore_seed | "$ESHOME/bin/elasticsearch-keystore" add -x keystore.seed
}
@test "[UPGRADE] start old version" {
export JAVA_HOME=$SYSTEM_JAVA_HOME
start_elasticsearch_service
unset JAVA_HOME
}
@test "[UPGRADE] check elasticsearch version is old version" {
check_elasticsearch_version "$(cat $BATS_UPGRADE/upgrade_from_version)"
}
@test "[UPGRADE] index some documents into a few indexes" {
curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library/book/1?pretty -d '{
"title": "Elasticsearch - The Definitive Guide"
}'
curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library/book/2?pretty -d '{
"title": "Brave New World"
}'
curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library2/book/1?pretty -d '{
"title": "The Left Hand of Darkness"
}'
}
@test "[UPGRADE] verify that the documents are there" {
curl -s localhost:9200/library/book/1?pretty | grep Elasticsearch
curl -s localhost:9200/library/book/2?pretty | grep World
curl -s localhost:9200/library2/book/1?pretty | grep Darkness
}
@test "[UPGRADE] stop old version" {
stop_elasticsearch_service
}
@test "[UPGRADE] install version under test" {
if [ "$sameVersion" == "true" ]; then
install_package -f
else
install_package -u
fi
}
@test "[UPGRADE] start version under test" {
start_elasticsearch_service yellow library
wait_for_elasticsearch_status yellow library2
}
@test "[UPGRADE] check elasticsearch version is version under test" {
check_elasticsearch_version "$(cat version)"
}
@test "[UPGRADE] verify that the documents are there after restart" {
curl -s localhost:9200/library/book/1?pretty | grep Elasticsearch
curl -s localhost:9200/library/book/2?pretty | grep World
curl -s localhost:9200/library2/book/1?pretty | grep Darkness
}
@test "[UPGRADE] cleanup version under test" {
stop_elasticsearch_service
clean_before_test
}

179
qa/os/bats/utils/packages.bash

@ -1,179 +0,0 @@
#!/bin/bash
# This file contains some utilities to test the elasticsearch scripts with
# the .deb/.rpm packages.
# WARNING: This testing file must be executed as root and can
# dramatically change your system. It should only be executed
# in a throw-away VM like those made by the Vagrantfile at
# the root of the Elasticsearch source code. This should
# cause the script to fail if it is executed any other way:
[ -f /etc/is_vagrant_vm ] || {
>&2 echo "must be run on a vagrant VM"
exit 1
}
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
env_file() {
if is_dpkg; then
echo "/etc/default/elasticsearch"
fi
if is_rpm; then
echo "/etc/sysconfig/elasticsearch"
fi
}
# Export some useful paths.
export_elasticsearch_paths() {
export ESHOME="/usr/share/elasticsearch"
export ESPLUGINS="$ESHOME/plugins"
export ESMODULES="$ESHOME/modules"
export ESCONFIG="/etc/elasticsearch"
export ESDATA="/var/lib/elasticsearch"
export ESLOG="/var/log/elasticsearch"
export ESENVFILE=$(env_file)
export PACKAGE_NAME
}
# Install the rpm or deb package.
# -u upgrade rather than install. This only matters for rpm.
# -v the version to upgrade to. Defaults to the version under test.
install_package() {
local version=$(cat version)
local rpmCommand='-i'
local dir='./'
while getopts ":ufd:v:" opt; do
case $opt in
u)
rpmCommand='-U'
dpkgCommand='--force-confnew'
;;
f)
rpmCommand='-U --force'
dpkgCommand='--force-conflicts'
;;
d)
dir=$OPTARG
;;
v)
version=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
;;
esac
done
local rpm_classifier="-x86_64"
local deb_classifier="-amd64"
if [[ $version == 6* ]]; then
rpm_classifier=""
deb_classifier=""
fi
if is_rpm; then
rpm $rpmCommand $dir/$PACKAGE_NAME-$version$rpm_classifier.rpm
elif is_dpkg; then
run dpkg $dpkgCommand -i $dir/$PACKAGE_NAME-$version$deb_classifier.deb
[[ "$status" -eq 0 ]] || {
echo "dpkg failed:"
echo "$output"
run lsof /var/lib/dpkg/lock
echo "lsof /var/lib/dpkg/lock:"
echo "$output"
false
}
else
skip "Only rpm or deb supported"
fi
# pass through java home to package
echo "JAVA_HOME=\"$SYSTEM_JAVA_HOME\"" >> $(env_file)
}
# Checks that all directories & files are correctly installed after a deb or
# rpm install.
verify_package_installation() {
id elasticsearch
getent group elasticsearch
# homedir is set in /etc/passwd but to a non existent directory
assert_file_not_exist $(getent passwd elasticsearch | cut -d: -f6)
assert_file "$ESHOME" d root root 755
assert_file "$ESHOME/bin" d root root 755
assert_file "$ESHOME/bin/elasticsearch" f root root 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
assert_file "$ESHOME/bin/elasticsearch-node" f root root 755
assert_file "$ESHOME/lib" d root root 755
assert_file "$ESCONFIG" d root elasticsearch 2750
assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660
sudo -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed"
assert_file "$ESCONFIG/.elasticsearch.keystore.initial_md5sum" f root elasticsearch 644
assert_file "$ESCONFIG/elasticsearch.yml" f root elasticsearch 660
assert_file "$ESCONFIG/jvm.options" f root elasticsearch 660
assert_file "$ESCONFIG/log4j2.properties" f root elasticsearch 660
assert_file "$ESDATA" d elasticsearch elasticsearch 2750
assert_file "$ESLOG" d elasticsearch elasticsearch 2750
assert_file "$ESPLUGINS" d root root 755
assert_file "$ESMODULES" d root root 755
assert_file "$ESHOME/NOTICE.txt" f root root 644
assert_file "$ESHOME/README.asciidoc" f root root 644
if is_dpkg; then
# Env file
assert_file "/etc/default/elasticsearch" f root elasticsearch 660
# Machine-readable debian/copyright file
local copyrightDir=$(readlink -f /usr/share/doc/$PACKAGE_NAME)
assert_file $copyrightDir d root root 755
assert_file "$copyrightDir/copyright" f root root 644
fi
if is_rpm; then
# Env file
assert_file "/etc/sysconfig/elasticsearch" f root elasticsearch 660
# License file
assert_file "/usr/share/elasticsearch/LICENSE.txt" f root root 644
fi
if is_systemd; then
assert_file "/usr/lib/systemd/system/elasticsearch.service" f root root 644
assert_file "/usr/lib/tmpfiles.d/elasticsearch.conf" f root root 644
assert_file "/usr/lib/sysctl.d/elasticsearch.conf" f root root 644
if is_rpm; then
[[ $(/usr/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]]
else
[[ $(/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]]
fi
fi
if is_sysvinit; then
assert_file "/etc/init.d/elasticsearch" f root root 750
fi
run sudo -E -u vagrant LANG="en_US.UTF-8" cat "$ESCONFIG/elasticsearch.yml"
[ $status = 1 ]
[[ "$output" == *"Permission denied"* ]] || {
echo "Expected permission denied but found $output:"
false
}
}

110
qa/os/bats/utils/tar.bash

@ -1,110 +0,0 @@
#!/bin/bash
# This file contains some utilities to test the elasticsearch
# tar distribution.
# WARNING: This testing file must be executed as root and can
# dramatically change your system. It should only be executed
# in a throw-away VM like those made by the Vagrantfile at
# the root of the Elasticsearch source code. This should
# cause the script to fail if it is executed any other way:
[ -f /etc/is_vagrant_vm ] || {
>&2 echo "must be run on a vagrant VM"
exit 1
}
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Install the tar.gz archive
install_archive() {
export ESHOME=${1:-/tmp/elasticsearch}
local version=$(cat version)
echo "Unpacking tarball to $ESHOME"
rm -rf /tmp/untar
mkdir -p /tmp/untar
tar -xzpf "${PACKAGE_NAME}-${version}-linux-x86_64.tar.gz" -C /tmp/untar
find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null
# ES cannot run as root so create elasticsearch user & group if needed
if ! getent group "elasticsearch" > /dev/null 2>&1 ; then
if is_dpkg; then
addgroup --system "elasticsearch"
else
groupadd -r "elasticsearch"
fi
fi
if ! id "elasticsearch" > /dev/null 2>&1 ; then
if is_dpkg; then
adduser --quiet --system --no-create-home --ingroup "elasticsearch" --disabled-password --shell /bin/false "elasticsearch"
else
useradd --system -M --gid "elasticsearch" --shell /sbin/nologin --comment "elasticsearch user" "elasticsearch"
fi
fi
chown -R elasticsearch:elasticsearch "$ESHOME"
export_elasticsearch_paths
}
# Move the unzipped tarball to another location.
move_elasticsearch() {
local oldhome="$ESHOME"
export ESHOME="$1"
rm -rf "$ESHOME"
mv "$oldhome" "$ESHOME"
export_elasticsearch_paths
}
# Export some useful paths.
export_elasticsearch_paths() {
export ESMODULES="$ESHOME/modules"
export ESPLUGINS="$ESHOME/plugins"
export ESCONFIG="$ESHOME/config"
export ESSCRIPTS="$ESCONFIG/scripts"
export ESDATA="$ESHOME/data"
export ESLOG="$ESHOME/logs"
export PACKAGE_NAME=${PACKAGE_NAME:-"elasticsearch-oss"}
}
# Checks that all directories & files are correctly installed
# after a archive (tar.gz/zip) install
verify_archive_installation() {
assert_file "$ESHOME" d elasticsearch elasticsearch 755
assert_file "$ESHOME/bin" d elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-node" f elasticsearch elasticsearch 755
assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660
assert_file "$ESCONFIG/log4j2.properties" f elasticsearch elasticsearch 660
assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755
assert_file "$ESHOME/lib" d elasticsearch elasticsearch 755
assert_file "$ESHOME/logs" d elasticsearch elasticsearch 755
assert_file "$ESHOME/NOTICE.txt" f elasticsearch elasticsearch 644
assert_file "$ESHOME/LICENSE.txt" f elasticsearch elasticsearch 644
assert_file "$ESHOME/README.asciidoc" f elasticsearch elasticsearch 644
assert_file_not_exist "$ESCONFIG/elasticsearch.keystore"
}

596
qa/os/bats/utils/utils.bash

@ -1,596 +0,0 @@
#!/bin/bash
# This file contains some utilities to test the .deb/.rpm
# packages and the SysV/Systemd scripts.
# WARNING: This testing file must be executed as root and can
# dramatically change your system. It should only be executed
# in a throw-away VM like those made by the Vagrantfile at
# the root of the Elasticsearch source code. This should
# cause the script to fail if it is executed any other way:
[ -f /etc/is_vagrant_vm ] || {
>&2 echo "must be run on a vagrant VM"
exit 1
}
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Checks if necessary commands are available to run the tests
if [ ! -x /usr/bin/which ]; then
echo "'which' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which wget 2>/dev/null`" ]; then
echo "'wget' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which curl 2>/dev/null`" ]; then
echo "'curl' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which pgrep 2>/dev/null`" ]; then
echo "'pgrep' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which unzip 2>/dev/null`" ]; then
echo "'unzip' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which tar 2>/dev/null`" ]; then
echo "'tar' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which unzip 2>/dev/null`" ]; then
echo "'unzip' command is mandatory to run the tests"
exit 1
fi
if [ ! -x "`which java 2>/dev/null`" ]; then
# there are some tests that move java temporarily
if [ ! -x "`command -v java.bak 2>/dev/null`" ]; then
echo "'java' command is mandatory to run the tests"
exit 1
fi
fi
# Returns 0 if the 'dpkg' command is available
is_dpkg() {
[ -x "`which dpkg 2>/dev/null`" ]
}
# Returns 0 if the 'rpm' command is available
is_rpm() {
[ -x "`which rpm 2>/dev/null`" ]
}
# Skip test if the 'dpkg' command is not supported
skip_not_dpkg() {
is_dpkg || skip "dpkg is not supported"
}
# Skip test if the 'rpm' command is not supported
skip_not_rpm() {
is_rpm || skip "rpm is not supported"
}
skip_not_dpkg_or_rpm() {
is_dpkg || is_rpm || skip "only dpkg or rpm systems are supported"
}
# Returns 0 if the system supports Systemd
is_systemd() {
[ -x /bin/systemctl ]
}
# Skip test if Systemd is not supported
skip_not_systemd() {
if [ ! -x /bin/systemctl ]; then
skip "systemd is not supported"
fi
}
# Returns 0 if the system supports SysV
is_sysvinit() {
[ -x "`which service 2>/dev/null`" ]
}
# Skip test if SysV is not supported
skip_not_sysvinit() {
if [ -x "`which service 2>/dev/null`" ] && is_systemd; then
skip "sysvinit is supported, but systemd too"
fi
if [ ! -x "`which service 2>/dev/null`" ]; then
skip "sysvinit is not supported"
fi
}
# Skip if tar is not supported
skip_not_tar_gz() {
if [ ! -x "`which tar 2>/dev/null`" ]; then
skip "tar is not supported"
fi
}
# Skip if unzip is not supported
skip_not_zip() {
if [ ! -x "`which unzip 2>/dev/null`" ]; then
skip "unzip is not supported"
fi
}
assert_file_exist() {
local file="$1"
local count=$(echo "$file" | wc -l)
[[ "$count" == "1" ]] || {
echo "assert_file_exist must be run on a single file at a time but was called on [$count] files: $file"
false
}
if [ ! -e "$file" ]; then
echo "Should exist: ${file} but does not"
fi
local file=$(readlink -m "${file}")
[ -e "$file" ]
}
assert_file_not_exist() {
local file="$1"
if [ -e "$file" ]; then
echo "Should not exist: ${file} but does"
fi
local file=$(readlink -m "${file}")
[ ! -e "$file" ]
}
assert_file() {
local file="$1"
local type=$2
local user=$3
local group=$4
local privileges=$5
assert_file_exist "$file"
if [ "$type" = "d" ]; then
if [ ! -d "$file" ]; then
echo "[$file] should be a directory but is not"
fi
[ -d "$file" ]
else
if [ ! -f "$file" ]; then
echo "[$file] should be a regular file but is not"
fi
[ -f "$file" ]
fi
if [ "x$user" != "x" ]; then
realuser=$(find "$file" -maxdepth 0 -printf "%u")
if [ "$realuser" != "$user" ]; then
echo "Expected user: $user, found $realuser [$file]"
fi
[ "$realuser" = "$user" ]
fi
if [ "x$group" != "x" ]; then
realgroup=$(find "$file" -maxdepth 0 -printf "%g")
if [ "$realgroup" != "$group" ]; then
echo "Expected group: $group, found $realgroup [$file]"
fi
[ "$realgroup" = "$group" ]
fi
if [ "x$privileges" != "x" ]; then
realprivileges=$(find "$file" -maxdepth 0 -printf "%m")
if [ "$realprivileges" != "$privileges" ]; then
echo "Expected privileges: $privileges, found $realprivileges [$file]"
fi
[ "$realprivileges" = "$privileges" ]
fi
}
assert_module_or_plugin_directory() {
local directory=$1
shift
#owner group and permissions vary depending on how es was installed
#just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u")
config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g")
assert_file $directory d $config_user $config_owner 755
}
assert_module_or_plugin_file() {
local file=$1
shift
assert_file_exist "$(readlink -m $file)"
assert_file $file f $config_user $config_owner 644
}
assert_output() {
echo "$output" | grep -E "$1"
}
# Deletes everything before running a test file
clean_before_test() {
# List of files to be deleted
ELASTICSEARCH_TEST_FILES=("/usr/share/elasticsearch" \
"/etc/elasticsearch" \
"/var/lib/elasticsearch" \
"/var/log/elasticsearch" \
"/tmp/elasticsearch" \
"/etc/default/elasticsearch" \
"/etc/sysconfig/elasticsearch" \
"/var/run/elasticsearch" \
"/usr/share/doc/elasticsearch" \
"/usr/share/doc/elasticsearch-oss" \
"/tmp/elasticsearch" \
"/usr/lib/systemd/system/elasticsearch.conf" \
"/usr/lib/tmpfiles.d/elasticsearch.conf" \
"/usr/lib/sysctl.d/elasticsearch.conf")
# Kills all processes of user elasticsearch
if id elasticsearch > /dev/null 2>&1; then
pkill -u elasticsearch 2>/dev/null || true
fi
# Kills all running Elasticsearch processes
ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true
purge_elasticsearch
# Removes user & group
userdel elasticsearch > /dev/null 2>&1 || true
groupdel elasticsearch > /dev/null 2>&1 || true
# Removes all files
for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do
if [ -e "$d" ]; then
rm -rf "$d"
fi
done
if is_systemd; then
systemctl unmask systemd-sysctl.service
fi
}
purge_elasticsearch() {
# Removes RPM package
if is_rpm; then
rpm --quiet -e $PACKAGE_NAME > /dev/null 2>&1 || true
fi
if [ -x "`which yum 2>/dev/null`" ]; then
yum remove -y $PACKAGE_NAME > /dev/null 2>&1 || true
fi
# Removes DEB package
if is_dpkg; then
dpkg --purge $PACKAGE_NAME > /dev/null 2>&1 || true
fi
if [ -x "`which apt-get 2>/dev/null`" ]; then
apt-get --quiet --yes purge $PACKAGE_NAME > /dev/null 2>&1 || true
fi
}
# Start elasticsearch and wait for it to come up with a status.
# $1 - expected status - defaults to green
start_elasticsearch_service() {
local desiredStatus=${1:-green}
local index=$2
local commandLineArgs=$3
run_elasticsearch_service 0 $commandLineArgs
wait_for_elasticsearch_status $desiredStatus $index
if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
[ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
echo "Looking for elasticsearch pid...."
ps $pid
elif is_systemd; then
run systemctl is-active elasticsearch.service
[ "$status" -eq 0 ]
run systemctl status elasticsearch.service
[ "$status" -eq 0 ]
elif is_sysvinit; then
run service elasticsearch status
[ "$status" -eq 0 ]
fi
}
# Start elasticsearch
# $1 expected status code
# $2 additional command line args
run_elasticsearch_service() {
local expectedStatus=$1
local commandLineArgs=$2
# Set the ES_PATH_CONF setting in case we start as a service
if [ ! -z "$ES_PATH_CONF" ] ; then
if is_dpkg; then
echo "ES_PATH_CONF=$ES_PATH_CONF" >> /etc/default/elasticsearch;
elif is_rpm; then
echo "ES_PATH_CONF=$ES_PATH_CONF" >> /etc/sysconfig/elasticsearch;
fi
fi
if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
# we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
local background=""
local timeoutCommand=""
if [ "$expectedStatus" = 0 ]; then
background="-d"
else
timeoutCommand="timeout 180s "
fi
# su and the Elasticsearch init script work together to break bats.
# sudo isolates bats enough from the init script so everything continues
# to tick along
run sudo -u elasticsearch bash <<BASH
# If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try.
# If it doesn't ignore it then Elasticsearch will fail to start because of security errors.
# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
export ES_PATH_CONF=$ES_PATH_CONF
export ES_JAVA_OPTS=$ES_JAVA_OPTS
$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $commandLineArgs
BASH
[ "$status" -eq "$expectedStatus" ]
elif is_systemd; then
run systemctl daemon-reload
[ "$status" -eq 0 ]
run systemctl enable elasticsearch.service
[ "$status" -eq 0 ]
run systemctl is-enabled elasticsearch.service
[ "$status" -eq 0 ]
run systemctl start elasticsearch.service
[ "$status" -eq "$expectedStatus" ]
elif is_sysvinit; then
run service elasticsearch start
[ "$status" -eq "$expectedStatus" ]
fi
}
stop_elasticsearch_service() {
if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
[ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
kill -SIGTERM $pid
elif is_systemd; then
run systemctl stop elasticsearch.service
[ "$status" -eq 0 ]
run systemctl is-active elasticsearch.service
[ "$status" -eq 3 ]
echo "$output" | grep -E 'inactive|failed'
elif is_sysvinit; then
run service elasticsearch stop
[ "$status" -eq 0 ]
run service elasticsearch status
[ "$status" -ne 0 ]
fi
}
# the default netcat packages in the distributions we test are not all compatible
# so we use /dev/tcp - a feature of bash which makes tcp connections
# http://tldp.org/LDP/abs/html/devref1.html#DEVTCP
test_port() {
local host="$1"
local port="$2"
cat < /dev/null > "/dev/tcp/$host/$port"
}
describe_port() {
local host="$1"
local port="$2"
if test_port "$host" "$port"; then
echo "port $port on host $host is open"
else
echo "port $port on host $host is not open"
fi
}
debug_collect_logs() {
local es_logfile="/var/log/elasticsearch/elasticsearch.log"
local system_logfile='/var/log/messages'
if [ -e "$es_logfile" ]; then
echo "Here's the elasticsearch log:"
cat "$es_logfile"
else
echo "The elasticsearch log doesn't exist at $es_logfile"
fi
if [ -e "$system_logfile" ]; then
echo "Here's the tail of the log at $system_logfile:"
tail -n20 "$system_logfile"
else
echo "The logfile at $system_logfile doesn't exist"
fi
echo "Current java processes:"
ps aux | grep java || true
echo "Testing if ES ports are open:"
describe_port 127.0.0.1 9200
describe_port 127.0.0.1 9201
}
set_debug_logging() {
if [ "$ESCONFIG" ] && [ -d "$ESCONFIG" ] && [ -f /etc/os-release ] && (grep -qi suse /etc/os-release); then
echo 'logger.org.elasticsearch.indices: TRACE' >> "$ESCONFIG/elasticsearch.yml"
echo 'logger.org.elasticsearch.gateway: TRACE' >> "$ESCONFIG/elasticsearch.yml"
echo 'logger.org.elasticsearch.cluster: DEBUG' >> "$ESCONFIG/elasticsearch.yml"
fi
}
# Waits for Elasticsearch to reach some status.
# $1 - expected status - defaults to green
wait_for_elasticsearch_status() {
local desiredStatus=${1:-green}
local index=$2
echo "Making sure elasticsearch is up..."
wget -O - --retry-connrefused --waitretry=1 --timeout=120 --tries=120 http://localhost:9200/_cluster/health || {
echo "Looks like elasticsearch never started"
debug_collect_logs
false
}
if [ -z "index" ]; then
echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..."
curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=180s&pretty"
else
echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index"
curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=180s&pretty"
fi
if [ $? -eq 0 ]; then
echo "Connected"
else
echo "Unable to connect to Elasticsearch"
false
fi
echo "Checking that the cluster health matches the waited for status..."
run curl -sS -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
if [ "$status" -ne 0 ]; then
echo "error when checking cluster health. code=$status output="
echo $output
false
fi
echo $output | grep $desiredStatus || {
echo "unexpected status: '$output' wanted '$desiredStatus'"
debug_collect_logs
false
}
}
# Checks the current elasticsearch version using the Info REST endpoint
# $1 - expected version
check_elasticsearch_version() {
local version=$1
local versionToCheck
local major=$(echo ${version} | cut -d. -f1 )
if [ $major -ge 7 ] ; then
versionToCheck=$version
else
versionToCheck=$(echo ${version} | sed -e 's/-SNAPSHOT//')
fi
run curl -s localhost:9200
[ "$status" -eq 0 ]
echo $output | grep \"number\"\ :\ \"$versionToCheck\" || {
echo "Expected $versionToCheck but installed an unexpected version:"
curl -s localhost:9200
false
}
}
# Executes some basic Elasticsearch tests
run_elasticsearch_tests() {
# TODO this assertion is the same the one made when waiting for
# elasticsearch to start
run curl -XGET 'http://localhost:9200/_cat/health?h=status&v=false'
[ "$status" -eq 0 ]
echo "$output" | grep -w "green"
curl -s -H "Content-Type: application/json" -XPOST 'http://localhost:9200/library/book/1?refresh=true&pretty' -d '{
"title": "Book #1",
"pages": 123
}'
curl -s -H "Content-Type: application/json" -XPOST 'http://localhost:9200/library/book/2?refresh=true&pretty' -d '{
"title": "Book #2",
"pages": 456
}'
curl -s -XGET 'http://localhost:9200/_count?pretty' |
grep \"count\"\ :\ 2
curl -s -XDELETE 'http://localhost:9200/_all'
}
# Move the config directory to another directory and properly chown it.
move_config() {
local oldConfig="$ESCONFIG"
# The custom config directory is not under /tmp or /var/tmp because
# systemd's private temp directory functionally means different
# processes can have different views of what's in these directories
export ESCONFIG="${1:-$(mktemp -p /etc -d -t 'config.XXXX')}"
echo "Moving configuration directory from $oldConfig to $ESCONFIG"
# Move configuration files to the new configuration directory
mv "$oldConfig"/* "$ESCONFIG"
chown -R elasticsearch:elasticsearch "$ESCONFIG"
assert_file_exist "$ESCONFIG/elasticsearch.yml"
assert_file_exist "$ESCONFIG/jvm.options"
assert_file_exist "$ESCONFIG/log4j2.properties"
}
# permissions from the user umask with the executable bit set
executable_privileges_for_user_from_umask() {
local user=$1
shift
echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111))
}
# permissions from the user umask without the executable bit set
file_privileges_for_user_from_umask() {
local user=$1
shift
echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111))
}
# move java to simulate it not being in the path
move_java() {
which_java=`command -v java`
assert_file_exist $which_java
mv $which_java ${which_java}.bak
}
# move java back to its original location
unmove_java() {
which_java=`command -v java.bak`
assert_file_exist $which_java
mv $which_java `dirname $which_java`/java
}

100
qa/os/bats/utils/xpack.bash

@ -1,100 +0,0 @@
#!/bin/bash
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.
# Checks that X-Pack files are correctly installed
verify_xpack_installation() {
local name="x-pack"
local user="$ESPLUGIN_COMMAND_USER"
local group="$ESPLUGIN_COMMAND_USER"
# Verify binary files
# nocommit: already verified by "main" package verification
#assert_file "$ESHOME/bin" d $user $group 755
local binaryFiles=(
'elasticsearch-certgen'
'elasticsearch-certutil'
'elasticsearch-croneval'
'elasticsearch-migrate'
'elasticsearch-saml-metadata'
'elasticsearch-setup-passwords'
'elasticsearch-sql-cli'
"elasticsearch-sql-cli-$(cat version).jar" # This jar is executable so we pitch it in bin so folks will find it
'elasticsearch-syskeygen'
'elasticsearch-users'
'x-pack-env'
'x-pack-security-env'
'x-pack-watcher-env'
)
local binaryFilesCount=5 # start with oss distro number
for binaryFile in ${binaryFiles[@]}; do
echo "checking for bin file ${binaryFile}"
assert_file "$ESHOME/bin/${binaryFile}" f $user $group 755
binaryFilesCount=$(( binaryFilesCount + 1 ))
done
ls "$ESHOME/bin/"
# nocommit: decide whether to check the files added by the distribution, not part of xpack...
#assert_number_of_files "$ESHOME/bin/" $binaryFilesCount
# Verify config files
# nocommit: already verified by "main" package verification
#assert_file "$ESCONFIG" d $user elasticsearch 755
local configFiles=(
'users'
'users_roles'
'roles.yml'
'role_mapping.yml'
'log4j2.properties'
)
local configFilesCount=2 # start with ES files, excluding log4j2
for configFile in ${configFiles[@]}; do
assert_file "$ESCONFIG/${configFile}" f $user elasticsearch 660
configFilesCount=$(( configFilesCount + 1 ))
done
# nocommit: decide whether to check the files added by the distribution, not part of xpack...
#assert_number_of_files "$ESCONFIG/" $configFilesCount
}
assert_number_of_files() {
local directory=$1
local expected=$2
local count=$(ls "$directory" | wc -l)
[ "$count" -eq "$expected" ] || {
echo "Expected $expected files in $directory but found: $count"
false
}
}
generate_trial_license() {
sudo -E -u $ESPLUGIN_COMMAND_USER sh <<"NODE_SETTINGS"
cat >> $ESCONFIG/elasticsearch.yml <<- EOF
xpack.license.self_generated.type: trial
xpack.security.enabled: true
EOF
NODE_SETTINGS
}
wait_for_xpack() {
local host=${1:-localhost}
local port=${2:-9200}
local listening=1
for i in {1..60}; do
if test_port "$host" "$port"; then
listening=0
break
else
sleep 1
fi
done
[ "$listening" -eq 0 ] || {
echo "Looks like elasticsearch with x-pack never started."
debug_collect_logs
false
}
}

110
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java Normal file

@ -0,0 +1,110 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.packaging.test;
import org.apache.http.client.fluent.Request;
import org.apache.http.entity.ContentType;
import org.elasticsearch.packaging.util.Distribution;
import org.elasticsearch.packaging.util.Packages;
import java.nio.file.Paths;
import static org.elasticsearch.packaging.util.Packages.assertInstalled;
import static org.elasticsearch.packaging.util.Packages.installPackage;
import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
import static org.hamcrest.Matchers.containsString;
public class PackageUpgradeTests extends PackagingTestCase {
// the distribution being upgraded
protected static final Distribution bwcDistribution;
static {
bwcDistribution = new Distribution(Paths.get(System.getProperty("tests.bwc-distribution")));
}
public void test10InstallBwcVersion() throws Exception {
installation = installPackage(sh, bwcDistribution);
assertInstalled(bwcDistribution);
verifyPackageInstallation(installation, bwcDistribution, sh);
}
public void test11ModifyKeystore() throws Exception {
// deliberately modify the keystore to force it to be preserved during package upgrade
installation.executables().keystoreTool.run("remove keystore.seed");
installation.executables().keystoreTool.run("add -x keystore.seed", "keystore_seed");
}
public void test12SetupBwcVersion() throws Exception {
startElasticsearch();
// create indexes explicitly with 0 replicas so when restarting we can reach green state
makeRequest(
Request.Put("http://localhost:9200/library")
.bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON)
);
makeRequest(
Request.Put("http://localhost:9200/library2")
.bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON)
);
// add some docs
makeRequest(
Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
.bodyString("{ \"title\": \"Elasticsearch - The Definitive Guide\"}", ContentType.APPLICATION_JSON)
);
makeRequest(
Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty")
.bodyString("{ \"title\": \"Brave New World\"}", ContentType.APPLICATION_JSON)
);
makeRequest(
Request.Post("http://localhost:9200/library2/_doc/1?refresh=true&pretty")
.bodyString("{ \"title\": \"The Left Hand of Darkness\"}", ContentType.APPLICATION_JSON)
);
assertDocsExist();
stopElasticsearch();
}
public void test20InstallUpgradedVersion() throws Exception {
if (bwcDistribution.path.equals(distribution.path)) {
// the old and new distributions are the same, so we are testing force upgrading
Packages.forceUpgradePackage(sh, distribution);
} else {
Packages.upgradePackage(sh, distribution);
}
assertInstalled(distribution);
verifyPackageInstallation(installation, distribution, sh);
}
public void test21CheckUpgradedVersion() throws Exception {
assertWhileRunning(() -> { assertDocsExist(); });
}
private void assertDocsExist() throws Exception {
String response1 = makeRequest(Request.Get("http://localhost:9200/library/_doc/1?pretty"));
assertThat(response1, containsString("Elasticsearch"));
String response2 = makeRequest(Request.Get("http://localhost:9200/library/_doc/2?pretty"));
assertThat(response2, containsString("World"));
String response3 = makeRequest(Request.Get("http://localhost:9200/library2/_doc/1?pretty"));
assertThat(response3, containsString("Darkness"));
}
}

6
qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java

@ -29,6 +29,7 @@ public class Distribution {
public final Platform platform; public final Platform platform;
public final Flavor flavor; public final Flavor flavor;
public final boolean hasJdk; public final boolean hasJdk;
public final String version;
public Distribution(Path path) { public Distribution(Path path) {
this.path = path; this.path = path;
@ -46,6 +47,11 @@ public class Distribution {
this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX; this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX;
this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT; this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT;
this.hasJdk = filename.contains("no-jdk") == false; this.hasJdk = filename.contains("no-jdk") == false;
String version = filename.split("-", 3)[1];
if (filename.contains("-SNAPSHOT")) {
version += "-SNAPSHOT";
}
this.version = version;
} }
public boolean isDefault() { public boolean isDefault() {

89
qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@ -28,6 +28,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -40,7 +41,6 @@ import static org.elasticsearch.packaging.util.FileMatcher.p644;
import static org.elasticsearch.packaging.util.FileMatcher.p660; import static org.elasticsearch.packaging.util.FileMatcher.p660;
import static org.elasticsearch.packaging.util.FileMatcher.p750; import static org.elasticsearch.packaging.util.FileMatcher.p750;
import static org.elasticsearch.packaging.util.FileMatcher.p755; import static org.elasticsearch.packaging.util.FileMatcher.p755;
import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion;
import static org.elasticsearch.packaging.util.Platforms.isSysVInit; import static org.elasticsearch.packaging.util.Platforms.isSysVInit;
import static org.elasticsearch.packaging.util.Platforms.isSystemd; import static org.elasticsearch.packaging.util.Platforms.isSystemd;
import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch; import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch;
@ -83,17 +83,8 @@ public class Packages {
} }
public static Result packageStatus(Distribution distribution) { public static Result packageStatus(Distribution distribution) {
final Shell sh = new Shell();
final Result result;
logger.info("Package type: " + distribution.packaging); logger.info("Package type: " + distribution.packaging);
if (distribution.packaging == Distribution.Packaging.RPM) { return runPackageManager(distribution, new Shell(), PackageManagerCommand.QUERY);
result = sh.runIgnoreExitCode("rpm -qe " + distribution.flavor.name);
} else {
result = sh.runIgnoreExitCode("dpkg -s " + distribution.flavor.name);
}
return result;
} }
public static Installation installPackage(Shell sh, Distribution distribution) throws IOException { public static Installation installPackage(Shell sh, Distribution distribution) throws IOException {
@ -101,7 +92,7 @@ public class Packages {
if (distribution.hasJdk == false) { if (distribution.hasJdk == false) {
sh.getEnv().put("JAVA_HOME", systemJavaHome); sh.getEnv().put("JAVA_HOME", systemJavaHome);
} }
final Result result = runInstallCommand(distribution, sh); final Result result = runPackageManager(distribution, sh, PackageManagerCommand.INSTALL);
if (result.exitCode != 0) { if (result.exitCode != 0) {
throw new RuntimeException("Installing distribution " + distribution + " failed: " + result); throw new RuntimeException("Installing distribution " + distribution + " failed: " + result);
} }
@ -114,13 +105,35 @@ public class Packages {
return installation; return installation;
} }
private static Result runInstallCommand(Distribution distribution, Shell sh) { public static Installation upgradePackage(Shell sh, Distribution distribution) throws IOException {
final Path distributionFile = distribution.path; final Result result = runPackageManager(distribution, sh, PackageManagerCommand.UPGRADE);
if (result.exitCode != 0) {
throw new RuntimeException("Upgrading distribution " + distribution + " failed: " + result);
}
return Installation.ofPackage(sh, distribution);
}
public static Installation forceUpgradePackage(Shell sh, Distribution distribution) throws IOException {
final Result result = runPackageManager(distribution, sh, PackageManagerCommand.FORCE_UPGRADE);
if (result.exitCode != 0) {
throw new RuntimeException("Force upgrading distribution " + distribution + " failed: " + result);
}
return Installation.ofPackage(sh, distribution);
}
private static Result runPackageManager(Distribution distribution, Shell sh, PackageManagerCommand command) {
final String distributionArg = command == PackageManagerCommand.QUERY || command == PackageManagerCommand.REMOVE
? distribution.flavor.name
: distribution.path.toString();
if (Platforms.isRPM()) { if (Platforms.isRPM()) {
return sh.runIgnoreExitCode("rpm -i " + distributionFile); String rpmOptions = RPM_OPTIONS.get(command);
return sh.runIgnoreExitCode("rpm " + rpmOptions + " " + distributionArg);
} else { } else {
Result r = sh.runIgnoreExitCode("dpkg -i " + distributionFile); String debOptions = DEB_OPTIONS.get(command);
Result r = sh.runIgnoreExitCode("dpkg " + debOptions + " " + distributionArg);
if (r.exitCode != 0) { if (r.exitCode != 0) {
Result lockOF = sh.runIgnoreExitCode("lsof /var/lib/dpkg/lock"); Result lockOF = sh.runIgnoreExitCode("lsof /var/lib/dpkg/lock");
if (lockOF.exitCode == 0) { if (lockOF.exitCode == 0) {
@ -133,15 +146,15 @@ public class Packages {
public static void remove(Distribution distribution) throws Exception { public static void remove(Distribution distribution) throws Exception {
final Shell sh = new Shell(); final Shell sh = new Shell();
Result result = runPackageManager(distribution, sh, PackageManagerCommand.REMOVE);
assertThat(result.toString(), result.isSuccess(), is(true));
Platforms.onRPM(() -> { Platforms.onRPM(() -> {
sh.run("rpm -e " + distribution.flavor.name);
final Result status = packageStatus(distribution); final Result status = packageStatus(distribution);
assertThat(status.exitCode, is(1)); assertThat(status.exitCode, is(1));
}); });
Platforms.onDPKG(() -> { Platforms.onDPKG(() -> {
sh.run("dpkg -r " + distribution.flavor.name);
final Result status = packageStatus(distribution); final Result status = packageStatus(distribution);
assertThat(status.exitCode, is(0)); assertThat(status.exitCode, is(0));
assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find()); assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find());
@ -151,7 +164,7 @@ public class Packages {
public static void verifyPackageInstallation(Installation installation, Distribution distribution, Shell sh) { public static void verifyPackageInstallation(Installation installation, Distribution distribution, Shell sh) {
verifyOssInstallation(installation, distribution, sh); verifyOssInstallation(installation, distribution, sh);
if (distribution.flavor == Distribution.Flavor.DEFAULT) { if (distribution.flavor == Distribution.Flavor.DEFAULT) {
verifyDefaultInstallation(installation); verifyDefaultInstallation(installation, distribution);
} }
} }
@ -215,7 +228,7 @@ public class Packages {
} }
} }
private static void verifyDefaultInstallation(Installation es) { private static void verifyDefaultInstallation(Installation es, Distribution distribution) {
Stream.of( Stream.of(
"elasticsearch-certgen", "elasticsearch-certgen",
@ -234,7 +247,7 @@ public class Packages {
// at this time we only install the current version of archive distributions, but if that changes we'll need to pass // at this time we only install the current version of archive distributions, but if that changes we'll need to pass
// the version through here // the version through here
assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, "root", "root", p755)); assertThat(es.bin("elasticsearch-sql-cli-" + distribution.version + ".jar"), file(File, "root", "root", p755));
Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties") Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties")
.forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660))); .forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
@ -317,4 +330,38 @@ public class Packages {
return sh.run("journalctl -u elasticsearch.service --after-cursor='" + this.cursor + "'"); return sh.run("journalctl -u elasticsearch.service --after-cursor='" + this.cursor + "'");
} }
} }
private enum PackageManagerCommand {
QUERY,
INSTALL,
UPGRADE,
FORCE_UPGRADE,
REMOVE
}
private static Map<PackageManagerCommand, String> RPM_OPTIONS = org.elasticsearch.common.collect.Map.of(
PackageManagerCommand.QUERY,
"-qe",
PackageManagerCommand.INSTALL,
"-i",
PackageManagerCommand.UPGRADE,
"-U",
PackageManagerCommand.FORCE_UPGRADE,
"-U --force",
PackageManagerCommand.REMOVE,
"-e"
);
private static Map<PackageManagerCommand, String> DEB_OPTIONS = org.elasticsearch.common.collect.Map.of(
PackageManagerCommand.QUERY,
"-s",
PackageManagerCommand.INSTALL,
"-i",
PackageManagerCommand.UPGRADE,
"-i --force-confnew",
PackageManagerCommand.FORCE_UPGRADE,
"-i --force-conflicts",
PackageManagerCommand.REMOVE,
"-r"
);
} }