honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Create utility for custom config setup in packaging tests (#58352) 

This commit creates a shared withCustomConfig method that may be used by
any packaging test. The method will copy the config directory and
override the conf path appropriately depending on the distribution type.
This commit is contained in:
Ryan Ernst 2020-06-23 14:49:20 -07:00 committed by Ryan Ernst
parent 2ca09cddaf
commit 89c03e593c
No known key found for this signature in database
GPG Key ID: 5F7EA39E15F54DCE
5 changed files with 76 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
server/src/main/java/org/elasticsearch/common/CheckedConsumer.java → libs/core/src/main/java/org/elasticsearch/common/CheckedConsumer.java
View File

46
qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
View File

@ -30,9 +30,7 @@ import org.junit.BeforeClass;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.nio.file.StandardOpenOption.CREATE;
@ -41,8 +39,6 @@ import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallatio
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
import static org.elasticsearch.packaging.util.FileUtils.append;
import static org.elasticsearch.packaging.util.FileUtils.cp;
import static org.elasticsearch.packaging.util.FileUtils.mkdir;
import static org.elasticsearch.packaging.util.FileUtils.mv;
import static org.elasticsearch.packaging.util.FileUtils.rm;
import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
@ -251,25 +247,10 @@ public class ArchiveTests extends PackagingTestCase {
public void test70CustomPathConfAndJvmOptions() throws Exception {
final Path tempConf = createTempDir("esconf-alternate");
try {
mkdir(tempConf);
cp(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml"));
cp(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties"));
// we have to disable Log4j from using JMX lest it will hit a security
// manager exception before we have configured logging; this will fail
// startup since we detect usages of logging before it is configured
final List<String> jvmOptions = new ArrayList<>();
jvmOptions.add("-Xms512m");
jvmOptions.add("-Xmx512m");
jvmOptions.add("-Dlog4j2.disable.jmx=true");
withCustomConfig(tempConf -> {
final List<String> jvmOptions = org.elasticsearch.common.collect.List.of("-Xms512m", "-Xmx512m", "-Dlog4j2.disable.jmx=true");
Files.write(tempConf.resolve("jvm.options"), jvmOptions, CREATE, APPEND);
sh.chown(tempConf);
sh.getEnv().put("ES_PATH_CONF", tempConf.toString());
sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
startElasticsearch();
@ -279,10 +260,7 @@ public class ArchiveTests extends PackagingTestCase {
assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
stopElasticsearch();
} finally {
rm(tempConf);
}
});
}
public void test71CustomJvmOptionsDirectoryFile() throws Exception {
@ -343,30 +321,16 @@ public class ArchiveTests extends PackagingTestCase {
public void test80RelativePathConf() throws Exception {
final Path temp = createTempDir("esconf-alternate");
final Path tempConf = temp.resolve("config");
try {
mkdir(tempConf);
Stream.of("elasticsearch.yml", "log4j2.properties", "jvm.options")
.forEach(file -> cp(installation.config(file), tempConf.resolve(file)));
withCustomConfig(tempConf -> {
append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");
sh.chown(temp);
sh.setWorkingDirectory(temp);
sh.getEnv().put("ES_PATH_CONF", "config");
startElasticsearch();
final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
assertThat(nodesResponse, containsString("\"name\":\"relative\""));
stopElasticsearch();
} finally {
rm(tempConf);
}
});
}
public void test90SecurityCliPackaging() throws Exception {

62
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.packaging.test;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.http.client.fluent.Request;
import org.elasticsearch.packaging.util.FileUtils;
import org.elasticsearch.packaging.util.Packages;
@ -32,7 +31,6 @@ import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.util.Collections.singletonList;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
@ -40,9 +38,7 @@ import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
import static org.elasticsearch.packaging.util.FileUtils.append;
import static org.elasticsearch.packaging.util.FileUtils.assertPathsDoNotExist;
import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
import static org.elasticsearch.packaging.util.FileUtils.cp;
import static org.elasticsearch.packaging.util.FileUtils.fileWithGlobExist;
import static org.elasticsearch.packaging.util.FileUtils.mkdir;
import static org.elasticsearch.packaging.util.FileUtils.mv;
import static org.elasticsearch.packaging.util.FileUtils.rm;
import static org.elasticsearch.packaging.util.FileUtils.slurp;
@ -301,17 +297,24 @@ public class PackageTests extends PackagingTestCase {
}
public void test81CustomPathConfAndJvmOptions() throws Exception {
assumeTrue(isSystemd());
assertPathsExist(installation.envFile);
stopElasticsearch();
withCustomConfig(tempConf -> {
append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops");
startElasticsearch();
final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824"));
assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
stopElasticsearch();
});
cleanup();
}
public void test83SystemdMask() throws Exception {
@ -354,6 +357,11 @@ public class PackageTests extends PackagingTestCase {
}
public void test90DoNotCloseStderrWhenQuiet() throws Exception {
assumeTrue(isSystemd());
assertPathsExist(installation.envFile);
stopElasticsearch();
withCustomConfig(tempConf -> {
// Create a startup problem by adding an invalid YAML line to the config
append(tempConf.resolve("elasticsearch.yml"), "discovery.zen.ping.unicast.hosts:15172.30.5.3416172.30.5.35, 172.30.5.17]\n");
@ -366,48 +374,4 @@ public class PackageTests extends PackagingTestCase {
assertThat(logs.stdout, containsString("Failed to load settings from [elasticsearch.yml]"));
});
}
@FunctionalInterface
private interface CustomConfigConsumer {
void accept(Path path) throws Exception;
}
private void withCustomConfig(CustomConfigConsumer pathConsumer) throws Exception {
assumeTrue(isSystemd());
assertPathsExist(installation.envFile);
stopElasticsearch();
// The custom config directory is not under /tmp or /var/tmp because
// systemd's private temp directory functionally means different
// processes can have different views of what's in these directories
String randomName = RandomStrings.randomAsciiAlphanumOfLength(getRandom(), 10);
sh.run("mkdir /etc/" + randomName);
final Path tempConf = Paths.get("/etc/" + randomName);
try {
mkdir(tempConf);
cp(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml"));
cp(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties"));
// we have to disable Log4j from using JMX lest it will hit a security
// manager exception before we have configured logging; this will fail
// startup since we detect usages of logging before it is configured
final String jvmOptions = "-Xms512m\n-Xmx512m\n-Dlog4j2.disable.jmx=true\n";
append(tempConf.resolve("jvm.options"), jvmOptions);
sh.runIgnoreExitCode("chown -R elasticsearch:elasticsearch " + tempConf);
cp(installation.envFile, tempConf.resolve("elasticsearch.bk"));// backup
append(installation.envFile, "ES_PATH_CONF=" + tempConf + "\n");
pathConsumer.accept(tempConf);
} finally {
rm(installation.envFile);
cp(tempConf.resolve("elasticsearch.bk"), installation.envFile);
rm(tempConf);
cleanup();
}
}
}

35
qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
View File

@ -26,6 +26,8 @@ import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
import com.carrotsearch.randomizedtesting.annotations.Timeout;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.packaging.util.Archives;
import org.elasticsearch.packaging.util.Distribution;
import org.elasticsearch.packaging.util.Docker;
@ -60,9 +62,10 @@ import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
import static org.elasticsearch.packaging.util.Docker.removeContainer;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
import static org.elasticsearch.packaging.util.FileUtils.append;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
@ -389,4 +392,34 @@ public abstract class PackagingTestCase extends Assert {
return Files.createTempDirectory(getRootTempDir(), prefix, NEW_DIR_PERMS);
}
/**
* Run the given action with a temporary copy of the config directory.
*
* Files under the path passed to the action may be modified as necessary for the
* test to execute, and running Elasticsearch with {@link #startElasticsearch()} will
* use the temporary directory.
*/
public void withCustomConfig(CheckedConsumer<Path, Exception> action) throws Exception {
Path tempDir = Files.createTempDirectory(getRootTempDir(), "custom-config");
Path tempConf = tempDir.resolve("elasticsearch");
FileUtils.copyDirectory(installation.config, tempConf);
Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + tempDir));
if (distribution.isPackage()) {
Files.copy(installation.envFile, tempDir.resolve("elasticsearch.bk"));// backup
append(installation.envFile, "ES_PATH_CONF=" + tempConf + "\n");
} else {
sh.getEnv().put("ES_PATH_CONF", tempConf.toString());
}
action.accept(tempConf);
if (distribution.isPackage()) {
IOUtils.rm(installation.envFile);
Files.copy(tempDir.resolve("elasticsearch.bk"), installation.envFile);
} else {
sh.getEnv().remove("ES_PATH_CONF");
}
IOUtils.rm(tempDir);
}
}

24
qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java
View File

@ -32,10 +32,13 @@ import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileOwnerAttributeView;
@ -362,4 +365,25 @@ public class FileUtils {
}
return path.toString();
}
/**
* Recursively copy the the source directory to the target directory, preserving permissions.
*/
public static void copyDirectory(Path source, Path target) throws IOException {
Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
// this won't copy the directory contents, only the directory itself, but we use
// copy to allow copying attributes
Files.copy(dir, target.resolve(source.relativize(dir)), StandardCopyOption.COPY_ATTRIBUTES);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.copy(file, target.resolve(source.relativize(file)), StandardCopyOption.COPY_ATTRIBUTES);
return FileVisitResult.CONTINUE;
}
});
}
}