honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Autoformat :qa:os and :benchmarks (#52816) 

Add `:qa:os` and `:benchmarks` to the list of automatically formatted
projects, and apply some manual fix-ups to polish it up.

In particular, I noticed that `Files.write(...)` when passed a list will
automaticaly apply a UTF-8 encoding and write a newline after each line,
making it easier to use than FileUtils.append. It's even available from
1.8.

Also, in the Allocators class, a number of methods declared thrown exceptions that IntelliJ reported were never thrown, and as far as I could see this is true, so I removed the exceptions.
This commit is contained in:
Rory Hunter 2020-02-28 14:48:04 +00:00 committed by GitHub
parent c3a167830f
commit d863c510da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 472 additions and 520 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java
View File

@ -55,13 +55,14 @@ public class AvailableIndexFoldersBenchmark {
@Setup
public void setup() throws IOException {
Path path = Files.createTempDirectory("test");
String[] paths = new String[] {path.toString()};
String[] paths = new String[] { path.toString() };
nodePath = new NodeEnvironment.NodePath(path);
LogConfigurator.setNodeName("test");
Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), path)
.putList(Environment.PATH_DATA_SETTING.getKey(), paths).build();
.putList(Environment.PATH_DATA_SETTING.getKey(), paths)
.build();
nodeEnv = new NodeEnvironment(settings, new Environment(settings, null));
Files.createDirectories(nodePath.indicesPath);
@ -80,7 +81,6 @@ public class AvailableIndexFoldersBenchmark {
}
}
@Benchmark
public Set<String> availableIndexFolderNaive() throws IOException {
return nodeEnv.availableIndexFoldersForPath(nodePath);

5
benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java
View File

@ -41,11 +41,11 @@ import java.util.concurrent.TimeUnit;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
@State(Scope.Benchmark)
@SuppressWarnings("unused") //invoked by benchmarking framework
@SuppressWarnings("unused") // invoked by benchmarking framework
public class MemoryStatsBenchmark {
private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean();
@Param({"0", "16", "256", "4096"})
@Param({ "0", "16", "256", "4096" })
private int tokens;
@Benchmark
@ -102,4 +102,3 @@ public class MemoryStatsBenchmark {
return MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed();
}
}

26
benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
View File

@ -49,7 +49,7 @@ import java.util.concurrent.TimeUnit;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
@SuppressWarnings("unused") //invoked by benchmarking framework
@SuppressWarnings("unused") // invoked by benchmarking framework
public class AllocationBenchmark {
// Do NOT make any field final (even if it is not annotated with @Param)! See also
// http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java
@ -106,8 +106,7 @@ public class AllocationBenchmark {
" 10| 10| 2| 50",
" 100| 1| 2| 50",
" 100| 3| 2| 50",
" 100| 10| 2| 50"
})
" 100| 10| 2| 50" })
public String indicesShardsReplicasNodes = "10|1|0|1";
public int numTags = 2;
@ -124,13 +123,14 @@ public class AllocationBenchmark {
int numReplicas = toInt(params[2]);
int numNodes = toInt(params[3]);
strategy = Allocators.createAllocationService(Settings.builder()
.put("cluster.routing.allocation.awareness.attributes", "tag")
.build());
strategy = Allocators.createAllocationService(
Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build()
);
MetaData.Builder mb = MetaData.builder();
for (int i = 1; i <= numIndices; i++) {
mb.put(IndexMetaData.builder("test_" + i)
mb.put(
IndexMetaData.builder("test_" + i)
.settings(Settings.builder().put("index.version.created", Version.CURRENT))
.numberOfShards(numShards)
.numberOfReplicas(numReplicas)
@ -147,8 +147,10 @@ public class AllocationBenchmark {
nb.add(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
}
initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
.metaData(metaData).routingTable(routingTable).nodes
(nb).build();
.metaData(metaData)
.routingTable(routingTable)
.nodes(nb)
.build();
}
private int toInt(String v) {
@ -159,8 +161,10 @@ public class AllocationBenchmark {
public ClusterState measureAllocation() {
ClusterState clusterState = initialClusterState;
while (clusterState.getRoutingNodes().hasUnassignedShards()) {
clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
.shardsWithState(ShardRoutingState.INITIALIZING));
clusterState = strategy.applyStartedShards(
clusterState,
clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING)
);
clusterState = strategy.reroute(clusterState, "reroute");
}
return clusterState;

34
benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
View File

@ -36,7 +36,6 @@ import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.gateway.GatewayAllocator;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@ -67,33 +66,34 @@ public final class Allocators {
throw new AssertionError("Do not instantiate");
}
public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
IllegalAccessException, InvocationTargetException {
return createAllocationService(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings
.BUILT_IN_CLUSTER_SETTINGS));
public static AllocationService createAllocationService(Settings settings) {
return createAllocationService(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) {
return new AllocationService(
defaultAllocationDeciders(settings, clusterSettings),
NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
NoopGatewayAllocator.INSTANCE,
new BalancedShardsAllocator(settings),
EmptyClusterInfoService.INSTANCE
);
}
public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
Collection<AllocationDecider> deciders =
ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
Collection<AllocationDecider> deciders = ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
return new AllocationDeciders(deciders);
}
private static final AtomicInteger portGenerator = new AtomicInteger();
public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS,
portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE,
DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
return new DiscoveryNode(
"",
nodeId,
new TransportAddress(TransportAddress.META_ADDRESS, portGenerator.incrementAndGet()),
attributes,
Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE),
Version.CURRENT
);
}
}

2
benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java
View File

@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Benchmark)
@SuppressWarnings("unused") //invoked by benchmarking framework
@SuppressWarnings("unused") // invoked by benchmarking framework
public class DateFormatterBenchmark {
private final DateFormatter javaFormatter = DateFormatter.forPattern("8year_month_day||ordinal_date||epoch_millis");

2
benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java
View File

@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Benchmark)
@SuppressWarnings("unused") //invoked by benchmarking framework
@SuppressWarnings("unused") // invoked by benchmarking framework
public class DateFormatterFromBenchmark {
private final TemporalAccessor accessor = DateFormatter.forPattern("epoch_millis").parse("1234567890");

51
benchmarks/src/main/java/org/elasticsearch/benchmark/time/RoundingBenchmark.java
View File

@ -48,7 +48,7 @@ import static org.elasticsearch.common.Rounding.DateTimeUnit.YEAR_OF_CENTURY;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Benchmark)
@SuppressWarnings("unused") //invoked by benchmarking framework
@SuppressWarnings("unused") // invoked by benchmarking framework
public class RoundingBenchmark {
private final ZoneId zoneId = ZoneId.of("Europe/Amsterdam");
@ -56,10 +56,10 @@ public class RoundingBenchmark {
private long timestamp = 1548879021354L;
private final org.elasticsearch.common.rounding.Rounding jodaRounding =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(timeZone).build();
private final Rounding javaRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY)
.timeZone(zoneId).build();
private final org.elasticsearch.common.rounding.Rounding jodaRounding = org.elasticsearch.common.rounding.Rounding.builder(
DateTimeUnit.HOUR_OF_DAY
).timeZone(timeZone).build();
private final Rounding javaRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(zoneId).build();
@Benchmark
public long timeRoundingDateTimeUnitJoda() {
@ -71,10 +71,10 @@ public class RoundingBenchmark {
return javaRounding.round(timestamp);
}
private final org.elasticsearch.common.rounding.Rounding jodaDayOfMonthRounding =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(timeZone).build();
private final Rounding javaDayOfMonthRounding = Rounding.builder(DAY_OF_MONTH)
.timeZone(zoneId).build();
private final org.elasticsearch.common.rounding.Rounding jodaDayOfMonthRounding = org.elasticsearch.common.rounding.Rounding.builder(
DateTimeUnit.DAY_OF_MONTH
).timeZone(timeZone).build();
private final Rounding javaDayOfMonthRounding = Rounding.builder(DAY_OF_MONTH).timeZone(zoneId).build();
@Benchmark
public long timeRoundingDateTimeUnitDayOfMonthJoda() {
@ -86,10 +86,10 @@ public class RoundingBenchmark {
return javaDayOfMonthRounding.round(timestamp);
}
private final org.elasticsearch.common.rounding.Rounding timeIntervalRoundingJoda =
org.elasticsearch.common.rounding.Rounding.builder(TimeValue.timeValueMinutes(60)).timeZone(timeZone).build();
private final Rounding timeIntervalRoundingJava = Rounding.builder(TimeValue.timeValueMinutes(60))
.timeZone(zoneId).build();
private final org.elasticsearch.common.rounding.Rounding timeIntervalRoundingJoda = org.elasticsearch.common.rounding.Rounding.builder(
TimeValue.timeValueMinutes(60)
).timeZone(timeZone).build();
private final Rounding timeIntervalRoundingJava = Rounding.builder(TimeValue.timeValueMinutes(60)).timeZone(zoneId).build();
@Benchmark
public long timeIntervalRoundingJava() {
@ -101,10 +101,11 @@ public class RoundingBenchmark {
return timeIntervalRoundingJoda.round(timestamp);
}
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcDayOfMonthJoda =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.UTC).build();
private final Rounding timeUnitRoundingUtcDayOfMonthJava = Rounding.builder(DAY_OF_MONTH)
.timeZone(ZoneOffset.UTC).build();
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcDayOfMonthJoda = org.elasticsearch.common.rounding.Rounding
.builder(DateTimeUnit.DAY_OF_MONTH)
.timeZone(DateTimeZone.UTC)
.build();
private final Rounding timeUnitRoundingUtcDayOfMonthJava = Rounding.builder(DAY_OF_MONTH).timeZone(ZoneOffset.UTC).build();
@Benchmark
public long timeUnitRoundingUtcDayOfMonthJava() {
@ -118,8 +119,7 @@ public class RoundingBenchmark {
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcQuarterOfYearJoda =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.QUARTER).timeZone(DateTimeZone.UTC).build();
private final Rounding timeUnitRoundingUtcQuarterOfYearJava = Rounding.builder(QUARTER_OF_YEAR)
.timeZone(ZoneOffset.UTC).build();
private final Rounding timeUnitRoundingUtcQuarterOfYearJava = Rounding.builder(QUARTER_OF_YEAR).timeZone(ZoneOffset.UTC).build();
@Benchmark
public long timeUnitRoundingUtcQuarterOfYearJava() {
@ -131,10 +131,11 @@ public class RoundingBenchmark {
return timeUnitRoundingUtcQuarterOfYearJoda.round(timestamp);
}
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcMonthOfYearJoda =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(DateTimeZone.UTC).build();
private final Rounding timeUnitRoundingUtcMonthOfYearJava = Rounding.builder(MONTH_OF_YEAR)
.timeZone(ZoneOffset.UTC).build();
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcMonthOfYearJoda = org.elasticsearch.common.rounding.Rounding
.builder(DateTimeUnit.MONTH_OF_YEAR)
.timeZone(DateTimeZone.UTC)
.build();
private final Rounding timeUnitRoundingUtcMonthOfYearJava = Rounding.builder(MONTH_OF_YEAR).timeZone(ZoneOffset.UTC).build();
@Benchmark
public long timeUnitRoundingUtcMonthOfYearJava() {
@ -146,11 +147,9 @@ public class RoundingBenchmark {
return timeUnitRoundingUtcMonthOfYearJoda.round(timestamp);
}
private final org.elasticsearch.common.rounding.Rounding timeUnitRoundingUtcYearOfCenturyJoda =
org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(DateTimeZone.UTC).build();
private final Rounding timeUnitRoundingUtcYearOfCenturyJava = Rounding.builder(YEAR_OF_CENTURY)
.timeZone(ZoneOffset.UTC).build();
private final Rounding timeUnitRoundingUtcYearOfCenturyJava = Rounding.builder(YEAR_OF_CENTURY).timeZone(ZoneOffset.UTC).build();
@Benchmark
public long timeUnitRoundingUtcYearOfCenturyJava() {

2
build.gradle
View File

@ -108,10 +108,12 @@ subprojects {
// is greater than the number of unformatted projects, this can be
// switched to an exclude list, and eventually removed completely.
def projectPathsToFormat = [
':benchmarks',
':build-tools',
':distribution:tools:java-version-checker',
':distribution:tools:launchers',
':distribution:tools:plugin-cli',
':qa:os',
':x-pack:plugin:autoscaling',
':x-pack:plugin:enrich'
]

54
qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
View File

@ -30,8 +30,12 @@ import org.junit.BeforeClass;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.nio.file.StandardOpenOption.CREATE;
import static org.elasticsearch.packaging.util.Archives.installArchive;
import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
@ -112,7 +116,7 @@ public class ArchiveTests extends PackagingTestCase {
try {
startElasticsearch();
} catch (Exception e ){
} catch (Exception e) {
if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
logger.info("Dumping jstack of elasticsearch processb ({}) that failed to start", pid);
@ -121,9 +125,7 @@ public class ArchiveTests extends PackagingTestCase {
throw e;
}
final String gcLogName = Platforms.LINUX && distribution().hasJdk == false
? "gc.log.0.current"
: "gc.log";
final String gcLogName = Platforms.LINUX && distribution().hasJdk == false ? "gc.log.0.current" : "gc.log";
assertThat(installation.logs.resolve(gcLogName), fileExists());
ServerUtils.runElasticsearchTests();
@ -146,8 +148,7 @@ public class ArchiveTests extends PackagingTestCase {
stopElasticsearch();
String systemJavaHome1 = sh.getEnv().get("JAVA_HOME");
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"),
containsString(systemJavaHome1));
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), containsString(systemJavaHome1));
}
public void test52BundledJdkRemoved() throws Exception {
@ -170,8 +171,7 @@ public class ArchiveTests extends PackagingTestCase {
stopElasticsearch();
String systemJavaHome1 = sh.getEnv().get("JAVA_HOME");
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"),
containsString(systemJavaHome1));
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), containsString(systemJavaHome1));
} finally {
mv(relocatedJdk, installation.bundledJdk);
}
@ -186,7 +186,7 @@ public class ArchiveTests extends PackagingTestCase {
sh.getEnv().put("JAVA_HOME", "C:\\Program Files (x86)\\java");
//verify ES can start, stop and run plugin list
// verify ES can start, stop and run plugin list
startElasticsearch();
stopElasticsearch();
@ -196,7 +196,7 @@ public class ArchiveTests extends PackagingTestCase {
assertThat(result.exitCode, equalTo(0));
} finally {
//clean up sym link
// clean up sym link
if (Files.exists(Paths.get(javaPath))) {
sh.run("cmd /c rmdir '" + javaPath + "' ");
}
@ -211,7 +211,7 @@ public class ArchiveTests extends PackagingTestCase {
sh.run("ln -s \"" + systemJavaHome + "\" \"" + testJavaHome + "\"");
sh.getEnv().put("JAVA_HOME", testJavaHome);
//verify ES can start, stop and run plugin list
// verify ES can start, stop and run plugin list
startElasticsearch();
stopElasticsearch();
@ -237,11 +237,11 @@ public class ArchiveTests extends PackagingTestCase {
// we have to disable Log4j from using JMX lest it will hit a security
// manager exception before we have configured logging; this will fail
// startup since we detect usages of logging before it is configured
final String jvmOptions =
"-Xms512m\n" +
"-Xmx512m\n" +
"-Dlog4j2.disable.jmx=true\n";
append(tempConf.resolve("jvm.options"), jvmOptions);
final List<String> jvmOptions = new ArrayList<>();
jvmOptions.add("-Xms512m");
jvmOptions.add("-Xmx512m");
jvmOptions.add("-Dlog4j2.disable.jmx=true");
Files.write(tempConf.resolve("jvm.options"), jvmOptions, CREATE, APPEND);
sh.chown(tempConf);
@ -324,11 +324,8 @@ public class ArchiveTests extends PackagingTestCase {
try {
mkdir(tempConf);
Stream.of(
"elasticsearch.yml",
"log4j2.properties",
"jvm.options"
).forEach(file -> cp(installation.config(file), tempConf.resolve(file)));
Stream.of("elasticsearch.yml", "log4j2.properties", "jvm.options")
.forEach(file -> cp(installation.config(file), tempConf.resolve(file)));
append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");
@ -389,8 +386,7 @@ public class ArchiveTests extends PackagingTestCase {
Platforms.PlatformAction action = () -> {
final Result result = sh.run(bin.nodeTool + " -h");
assertThat(result.stdout,
containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
assertThat(result.stdout, containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
};
// TODO: this should be checked on all distributions
@ -420,17 +416,13 @@ public class ArchiveTests extends PackagingTestCase {
Platforms.PlatformAction action = () -> {
Result result = sh.run(bin.certutilTool + " -h");
assertThat(result.stdout,
containsString("Simplifies certificate creation for use with the Elastic Stack"));
assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
result = sh.run(bin.syskeygenTool + " -h");
assertThat(result.stdout,
containsString("system key tool"));
assertThat(result.stdout, containsString("system key tool"));
result = sh.run(bin.setupPasswordsTool + " -h");
assertThat(result.stdout,
containsString("Sets the passwords for reserved users"));
assertThat(result.stdout, containsString("Sets the passwords for reserved users"));
result = sh.run(bin.usersTool + " -h");
assertThat(result.stdout,
containsString("Manages elasticsearch file users"));
assertThat(result.stdout, containsString("Manages elasticsearch file users"));
};
// TODO: this should be checked on all distributions

51
qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java
View File

@ -31,13 +31,15 @@ import org.junit.BeforeClass;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.nio.file.StandardOpenOption.CREATE;
import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
import static org.elasticsearch.packaging.util.FileMatcher.file;
import static org.elasticsearch.packaging.util.FileMatcher.p600;
import static org.elasticsearch.packaging.util.FileUtils.append;
import static org.elasticsearch.packaging.util.FileUtils.escapePath;
import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
import static org.hamcrest.CoreMatchers.containsString;
@ -65,17 +67,18 @@ public class CertGenCliTests extends PackagingTestCase {
install();
}
public void test20Help() throws Exception {
public void test20Help() {
Shell.Result result = installation.executables().certgenTool.run("--help");
assertThat(result.stdout, containsString("Simplifies certificate creation"));
}
public void test30Generate() throws Exception {
Files.write(instancesFile, Arrays.asList(
"instances:",
" - name: \"mynode\"",
" ip:",
" - \"127.0.0.1\""));
final List<String> yaml = new ArrayList<>();
yaml.add("instances:");
yaml.add(" - name: \"mynode\"");
yaml.add(" ip:");
yaml.add(" - \"127.0.0.1\"");
Files.write(instancesFile, yaml, CREATE, APPEND);
installation.executables().certgenTool.run("--in " + instancesFile + " --out " + certificatesFile);
@ -105,19 +108,25 @@ public class CertGenCliTests extends PackagingTestCase {
// windows 2012 r2 has powershell 4.0, which lacks Expand-Archive
assumeFalse(Platforms.OS_NAME.equals("Windows Server 2012 R2"));
append(installation.config("elasticsearch.yml"), String.join("\n",
"node.name: mynode",
"xpack.security.transport.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")),
"xpack.security.transport.ssl.certificate: " + escapePath(installation.config("certs/mynode/mynode.crt")),
"xpack.security.transport.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]",
"xpack.security.http.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")),
"xpack.security.http.ssl.certificate: "+ escapePath(installation.config("certs/mynode/mynode.crt")),
"xpack.security.http.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]",
"xpack.security.transport.ssl.enabled: true",
"xpack.security.http.ssl.enabled: true"));
final String keyPath = escapePath(installation.config("certs/mynode/mynode.key"));
final String certPath = escapePath(installation.config("certs/mynode/mynode.crt"));
final String caCertPath = escapePath(installation.config("certs/ca/ca.crt"));
assertWhileRunning(() -> {
ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt"));
});
List<String> yaml = new ArrayList<>();
yaml.add("node.name: mynode");
yaml.add("xpack.security.transport.ssl.key: " + keyPath);
yaml.add("xpack.security.transport.ssl.certificate: " + certPath);
yaml.add("xpack.security.transport.ssl.certificate_authorities: [\"" + caCertPath + "\"]");
yaml.add("xpack.security.http.ssl.key: " + keyPath);
yaml.add("xpack.security.http.ssl.certificate: " + certPath);
yaml.add("xpack.security.http.ssl.certificate_authorities: [\"" + caCertPath + "\"]");
yaml.add("xpack.security.transport.ssl.enabled: true");
yaml.add("xpack.security.http.ssl.enabled: true");
Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND);
assertWhileRunning(
() -> ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt"))
);
}
}

11
qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java
View File

@ -79,10 +79,7 @@ public class DebPreservationTests extends PackagingTestCase {
// keystore was removed
assertPathsDoNotExist(
installation.config("elasticsearch.keystore"),
installation.config(".elasticsearch.keystore.initial_md5sum")
);
assertPathsDoNotExist(installation.config("elasticsearch.keystore"), installation.config(".elasticsearch.keystore.initial_md5sum"));
// doc files were removed
@ -105,11 +102,7 @@ public class DebPreservationTests extends PackagingTestCase {
assertRemoved(distribution());
assertPathsDoNotExist(
installation.config,
installation.envFile,
SYSVINIT_SCRIPT
);
assertPathsDoNotExist(installation.config, installation.envFile, SYSVINIT_SCRIPT);
assertThat(packageStatus(distribution()).exitCode, is(1));
}

72
qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
View File

@ -30,15 +30,16 @@ import org.elasticsearch.packaging.util.Shell;
import org.junit.Ignore;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static java.util.Collections.singletonList;
import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER;
import static org.elasticsearch.packaging.util.Archives.installArchive;
import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
@ -82,8 +83,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
final Installation.Executables bin = installation.executables();
Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
assertFalse("has-passwd should fail", r.isSuccess());
assertThat("has-passwd should indicate missing keystore",
r.stderr, containsString(ERROR_KEYSTORE_NOT_FOUND));
assertThat("has-passwd should indicate missing keystore", r.stderr, containsString(ERROR_KEYSTORE_NOT_FOUND));
}
/** Test initial package state */
@ -98,8 +98,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
final Installation.Executables bin = installation.executables();
Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
assertFalse("has-passwd should fail", r.isSuccess());
assertThat("has-passwd should indicate unprotected keystore",
r.stderr, containsString(ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED));
assertThat("has-passwd should indicate unprotected keystore", r.stderr, containsString(ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED));
Shell.Result r2 = bin.keystoreTool.run("list");
assertThat(r2.stdout, containsString("keystore.seed"));
}
@ -119,8 +118,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
final Installation.Executables bin = installation.executables();
Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
assertFalse("has-passwd should fail", r.isSuccess());
assertThat("has-passwd should indicate unprotected keystore",
r.stdout, containsString(ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED));
assertThat("has-passwd should indicate unprotected keystore", r.stdout, containsString(ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED));
Shell.Result r2 = bin.keystoreTool.run("list");
assertThat(r2.stdout, containsString("keystore.seed"));
}
@ -153,8 +151,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
}
public void test40KeystorePasswordOnStandardInput() throws Exception {
assumeTrue("packages will use systemd, which doesn't handle stdin",
distribution.isArchive());
assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
assumeThat(installation, is(notNullValue()));
String password = "^|<>\\&exit"; // code insertion on Windows if special characters are not escaped
@ -171,8 +168,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
}
public void test41WrongKeystorePasswordOnStandardInput() {
assumeTrue("packages will use systemd, which doesn't handle stdin",
distribution.isArchive());
assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
assumeThat(installation, is(notNullValue()));
assertPasswordProtectedKeystore();
@ -183,10 +179,8 @@ public class KeystoreManagementTests extends PackagingTestCase {
@Ignore /* Ignored for feature branch, awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
public void test42KeystorePasswordOnTty() throws Exception {
assumeTrue("expect command isn't on Windows",
distribution.platform != Distribution.Platform.WINDOWS);
assumeTrue("packages will use systemd, which doesn't handle stdin",
distribution.isArchive());
assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
assumeThat(installation, is(notNullValue()));
String password = "keystorepass";
@ -204,10 +198,8 @@ public class KeystoreManagementTests extends PackagingTestCase {
@Ignore /* Ignored for feature branch, awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
public void test43WrongKeystorePasswordOnTty() throws Exception {
assumeTrue("expect command isn't on Windows",
distribution.platform != Distribution.Platform.WINDOWS);
assumeTrue("packages will use systemd, which doesn't handle stdin",
distribution.isArchive());
assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
assumeThat(installation, is(notNullValue()));
assertPasswordProtectedKeystore();
@ -222,8 +214,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
* view help information.
*/
public void test44EncryptedKeystoreAllowsHelpMessage() throws Exception {
assumeTrue("users call elasticsearch directly in archive case",
distribution.isArchive());
assumeTrue("users call elasticsearch directly in archive case", distribution.isArchive());
String password = "keystorepass";
@ -251,9 +242,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
sh.run("sudo systemctl set-environment ES_KEYSTORE_PASSPHRASE_FILE=" + esKeystorePassphraseFile);
Files.createFile(esKeystorePassphraseFile);
Files.write(esKeystorePassphraseFile,
(password + System.lineSeparator()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.WRITE);
Files.write(esKeystorePassphraseFile, singletonList(password));
startElasticsearch();
ServerUtils.runElasticsearchTests();
@ -277,9 +266,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
}
Files.createFile(esKeystorePassphraseFile);
Files.write(esKeystorePassphraseFile,
("wrongpassword" + System.lineSeparator()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.WRITE);
Files.write(esKeystorePassphraseFile, singletonList("wrongpassword"));
Packages.JournaldWrapper journaldWrapper = new Packages.JournaldWrapper(sh);
Shell.Result result = runElasticsearchStartCommand();
@ -323,7 +310,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
String password = "password";
String passwordFilename = "password.txt";
Files.write(tempDir.resolve(passwordFilename), (password + "\n").getBytes(StandardCharsets.UTF_8));
Files.write(tempDir.resolve(passwordFilename), singletonList(password));
Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
Path dockerKeystore = installation.config("elasticsearch.keystore");
@ -342,8 +329,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
waitForElasticsearch(installation);
ServerUtils.runElasticsearchTests();
}
finally {
} finally {
if (tempDir != null) {
rm(tempDir);
}
@ -387,9 +373,12 @@ public class KeystoreManagementTests extends PackagingTestCase {
// It's very tricky to properly quote a pipeline that you're passing to
// a docker exec command, so we're just going to put a small script in the
// temp folder.
String setPasswordScript = "echo \"" + password + "\n" + password
+ "\n\" | " + installation.executables().keystoreTool.toString() + " passwd";
Files.write(tempDirectory.resolve("set-pass.sh"), setPasswordScript.getBytes(StandardCharsets.UTF_8));
List<String> setPasswordScript = new ArrayList<>();
setPasswordScript.add("echo \"" + password);
setPasswordScript.add(password);
setPasswordScript.add("\" | " + installation.executables().keystoreTool.toString() + " passwd");
Files.write(tempDirectory.resolve("set-pass.sh"), setPasswordScript);
runContainer(distribution(), volumes, null);
try {
@ -420,9 +409,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
// the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here.
// from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests.
// when we run these commands as a role user we won't have to do this
Platforms.onWindows(() -> {
sh.chown(keystore);
});
Platforms.onWindows(() -> sh.chown(keystore));
if (distribution().isDocker()) {
try {
@ -455,14 +442,11 @@ public class KeystoreManagementTests extends PackagingTestCase {
final Installation.Executables bin = installation.executables();
// set the password by passing it to stdin twice
Platforms.onLinux(() -> {
bin.keystoreTool.run("passwd", password + "\n" + password + "\n");
});
Platforms.onLinux(() -> bin.keystoreTool.run("passwd", password + "\n" + password + "\n"));
Platforms.onWindows(() -> {
sh.run("Invoke-Command -ScriptBlock {echo \'" + password + "\'; echo \'" + password + "\'} | "
+ bin.keystoreTool + " passwd");
});
Platforms.onWindows(
() -> sh.run("Invoke-Command -ScriptBlock {echo '" + password + "'; echo '" + password + "'} | " + bin.keystoreTool + " passwd")
);
}
private void assertPasswordProtectedKeystore() {

25
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java
View File

@ -26,15 +26,15 @@ import org.elasticsearch.packaging.util.Packages;
import org.elasticsearch.packaging.util.Shell.Result;
import org.junit.BeforeClass;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.util.Collections.singletonList;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
import static org.elasticsearch.packaging.util.FileUtils.append;
@ -98,8 +98,7 @@ public class PackageTests extends PackagingTestCase {
private void assertRunsWithJavaHome() throws Exception {
byte[] originalEnvFile = Files.readAllBytes(installation.envFile);
try {
Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
StandardOpenOption.APPEND);
Files.write(installation.envFile, singletonList("JAVA_HOME=" + systemJavaHome), APPEND);
startElasticsearch();
runElasticsearchTests();
stopElasticsearch();
@ -107,8 +106,7 @@ public class PackageTests extends PackagingTestCase {
Files.write(installation.envFile, originalEnvFile);
}
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "elasticsearch*.log.gz"),
containsString(systemJavaHome));
assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "elasticsearch*.log.gz"), containsString(systemJavaHome));
}
public void test32JavaHomeOverride() throws Exception {
@ -170,9 +168,12 @@ public class PackageTests extends PackagingTestCase {
String start = sh.runIgnoreExitCode("date ").stdout.trim();
startElasticsearch();
String journalEntries = sh.runIgnoreExitCode("journalctl _SYSTEMD_UNIT=elasticsearch.service " +
"--since \"" + start + "\" --output cat | grep -v \"future versions of Elasticsearch will require Java 11\" | wc -l")
.stdout.trim();
String journalEntries = sh.runIgnoreExitCode(
"journalctl _SYSTEMD_UNIT=elasticsearch.service "
+ "--since \""
+ start
+ "\" --output cat | grep -v \"future versions of Elasticsearch will require Java 11\" | wc -l"
).stdout.trim();
assertThat(journalEntries, equalTo("0"));
assertPathsExist(installation.pidDir.resolve("elasticsearch.pid"));
@ -212,9 +213,7 @@ public class PackageTests extends PackagingTestCase {
matcher.find();
final int version = Integer.parseInt(matcher.group(1));
statusExitCode = version < 231
? 3
: 4;
statusExitCode = version < 231 ? 3 : 4;
}
assertThat(sh.runIgnoreExitCode("systemctl status elasticsearch.service").exitCode, is(statusExitCode));
@ -257,7 +256,6 @@ public class PackageTests extends PackagingTestCase {
}
}
public void test72TestRuntimeDirectory() throws Exception {
try {
install();
@ -280,7 +278,6 @@ public class PackageTests extends PackagingTestCase {
// TEST CASES FOR SYSTEMD ONLY
/**
* # Simulates the behavior of a system restart:
* # the PID directory is deleted by the operating system

31
qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
View File

@ -67,14 +67,12 @@ import static org.junit.Assume.assumeTrue;
* Class that all packaging test cases should inherit from
*/
@RunWith(RandomizedRunner.class)
@TestMethodProviders({
JUnit3MethodProvider.class
})
@TestMethodProviders({ JUnit3MethodProvider.class })
@Timeout(millis = 20 * 60 * 1000) // 20 min
@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
public abstract class PackagingTestCase extends Assert {
protected final Logger logger = LogManager.getLogger(getClass());
protected final Logger logger = LogManager.getLogger(getClass());
// the distribution being tested
protected static final Distribution distribution;
@ -142,19 +140,14 @@ public abstract class PackagingTestCase extends Assert {
}
}
@Before
public void setup() throws Exception {
assumeFalse(failed); // skip rest of tests once one fails
sh.reset();
if (distribution().hasJdk == false) {
Platforms.onLinux(() -> {
sh.getEnv().put("JAVA_HOME", systemJavaHome);
});
Platforms.onWindows(() -> {
sh.getEnv().put("JAVA_HOME", systemJavaHome);
});
Platforms.onLinux(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome));
Platforms.onWindows(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome));
}
}
@ -208,15 +201,14 @@ public abstract class PackagingTestCase extends Assert {
protected void assertWhileRunning(Platforms.PlatformAction assertions) throws Exception {
try {
awaitElasticsearchStartup(runElasticsearchStartCommand());
} catch (Exception e ){
} catch (Exception e) {
if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
logger.info("Dumping jstack of elasticsearch processb ({}) that failed to start", pid);
sh.runIgnoreExitCode("jstack " + pid);
}
if (Files.exists(installation.logs.resolve("elasticsearch.log"))) {
logger.warn("Elasticsearch log:\n" +
FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
logger.warn("Elasticsearch log:\n" + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
}
if (Files.exists(installation.logs.resolve("output.out"))) {
logger.warn("Stdout:\n" + FileUtils.slurpTxtorGz(installation.logs.resolve("output.out")));
@ -230,8 +222,7 @@ public abstract class PackagingTestCase extends Assert {
try {
assertions.run();
} catch (Exception e) {
logger.warn("Elasticsearch log:\n" +
FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
logger.warn("Elasticsearch log:\n" + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
throw e;
}
stopElasticsearch();
@ -345,9 +336,11 @@ public abstract class PackagingTestCase extends Assert {
// in the background
String wrapperPid = result.stdout.trim();
sh.runIgnoreExitCode("Wait-Process -Timeout " + Archives.ES_STARTUP_SLEEP_TIME_SECONDS + " -Id " + wrapperPid);
sh.runIgnoreExitCode("Get-EventSubscriber | " +
"where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |" +
"Unregister-EventSubscriber -Force");
sh.runIgnoreExitCode(
"Get-EventSubscriber | "
+ "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |"
+ "Unregister-EventSubscriber -Force"
);
assertThat(FileUtils.slurp(Archives.getPowershellErrorPath(installation)), anyOf(stringMatchers));
} else {

30
qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java
View File

@ -28,13 +28,15 @@ import org.junit.Before;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static org.elasticsearch.packaging.util.FileUtils.append;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.collection.IsMapContaining.hasKey;
import static org.junit.Assume.assumeTrue;
@ -52,9 +54,12 @@ public class PasswordToolsTests extends PackagingTestCase {
public void test010Install() throws Exception {
install();
append(installation.config("elasticsearch.yml"),
"xpack.license.self_generated.type: trial\n" +
"xpack.security.enabled: true");
List<String> lines = new ArrayList<>();
lines.add("xpack.license.self_generated.type: trial");
lines.add("xpack.security.enabled: true");
Files.write(installation.config("elasticsearch.yml"), lines, StandardOpenOption.APPEND);
}
public void test20GeneratePasswords() throws Exception {
@ -63,7 +68,11 @@ public class PasswordToolsTests extends PackagingTestCase {
Map<String, String> userpasses = parseUsersAndPasswords(result.stdout);
for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
String response = ServerUtils.makeRequest(
Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
Request.Get("http://localhost:9200"),
userpass.getKey(),
userpass.getValue(),
null
);
assertThat(response, containsString("You Know, for Search"));
}
});
@ -112,7 +121,10 @@ public class PasswordToolsTests extends PackagingTestCase {
assertWhileRunning(() -> {
String response = ServerUtils.makeRequest(
Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
"elastic", BOOTSTRAP_PASSWORD, null);
"elastic",
BOOTSTRAP_PASSWORD,
null
);
assertThat(response, containsString("\"status\":\"green\""));
});
}
@ -125,7 +137,11 @@ public class PasswordToolsTests extends PackagingTestCase {
assertThat(userpasses, hasKey("elastic"));
for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
String response = ServerUtils.makeRequest(
Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
Request.Get("http://localhost:9200"),
userpass.getKey(),
userpass.getValue(),
null
);
assertThat(response, containsString("You Know, for Search"));
}
});

28
qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java
View File

@ -78,21 +78,12 @@ public class RpmPreservationTests extends PackagingTestCase {
verifyPackageInstallation(installation, distribution(), sh);
sh.run("echo foobar | " + installation.executables().keystoreTool + " add --stdin foo.bar");
Stream.of(
"elasticsearch.yml",
"jvm.options",
"log4j2.properties"
)
Stream.of("elasticsearch.yml", "jvm.options", "log4j2.properties")
.map(each -> installation.config(each))
.forEach(path -> append(path, "# foo"));
append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo");
if (distribution().isDefault()) {
Stream.of(
"role_mapping.yml",
"roles.yml",
"users",
"users_roles"
)
Stream.of("role_mapping.yml", "roles.yml", "users", "users_roles")
.map(each -> installation.config(each))
.forEach(path -> append(path, "# foo"));
}
@ -119,27 +110,18 @@ public class RpmPreservationTests extends PackagingTestCase {
assertThat(installation.config, fileExists());
assertThat(installation.config("elasticsearch.keystore"), fileExists());
Stream.of(
"elasticsearch.yml",
"jvm.options",
"log4j2.properties"
).forEach(this::assertConfFilePreserved);
Stream.of("elasticsearch.yml", "jvm.options", "log4j2.properties").forEach(this::assertConfFilePreserved);
assertThat(installation.config(Paths.get("jvm.options.d", "heap.options")), fileExists());
if (distribution().isDefault()) {
Stream.of(
"role_mapping.yml",
"roles.yml",
"users",
"users_roles"
).forEach(this::assertConfFilePreserved);
Stream.of("role_mapping.yml", "roles.yml", "users", "users_roles").forEach(this::assertConfFilePreserved);
}
}
private void assertConfFilePreserved(String configFile) {
final Path original = installation.config(configFile);
final Path saved = installation.config(configFile + ".rpmsave");
assertConfFilePreserved(original ,saved);
assertConfFilePreserved(original, saved);
}
private void assertConfFilePreserved(final Path original, final Path saved) {

2
qa/os/src/test/java/org/elasticsearch/packaging/test/SysVInitTests.java
View File

@ -22,7 +22,7 @@ package org.elasticsearch.packaging.test;
import org.elasticsearch.packaging.util.Platforms;
import org.elasticsearch.packaging.util.ServerUtils;
import org.elasticsearch.packaging.util.Shell;
import org.junit.BeforeClass;
import org.junit.BeforeClass;
import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
import static org.elasticsearch.packaging.util.FileUtils.fileWithGlobExist;

66
qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java
View File

@ -82,12 +82,16 @@ public class WindowsServiceTests extends PackagingTestCase {
logger.error("---- Unexpected exit code (expected " + exitCode + ", got " + result.exitCode + ") for script: " + script);
logger.error(result);
logger.error("Dumping log files\n");
Result logs = sh.run("$files = Get-ChildItem \"" + installation.logs + "\\elasticsearch.log\"; " +
"Write-Output $files; " +
"foreach ($file in $files) {" +
"Write-Output \"$file\"; " +
"Get-Content \"$file\" " +
"}");
Result logs = sh.run(
"$files = Get-ChildItem \""
+ installation.logs
+ "\\elasticsearch.log\"; "
+ "Write-Output $files; "
+ "foreach ($file in $files) {"
+ " Write-Output \"$file\"; "
+ " Get-Content \"$file\" "
+ "}"
);
logger.error(logs.stdout);
fail();
} else {
@ -105,7 +109,7 @@ public class WindowsServiceTests extends PackagingTestCase {
Path serviceExe = installation.bin("elasticsearch-service-x64.exe");
Path tmpServiceExe = serviceExe.getParent().resolve(serviceExe.getFileName() + ".tmp");
Files.move(serviceExe, tmpServiceExe);
Result result = sh.runIgnoreExitCode(serviceScript + " install");
Result result = sh.runIgnoreExitCode(serviceScript + " install");
assertThat(result.exitCode, equalTo(1));
assertThat(result.stdout, containsString("elasticsearch-service-x64.exe was not found..."));
Files.move(tmpServiceExe, serviceExe);
@ -167,28 +171,32 @@ public class WindowsServiceTests extends PackagingTestCase {
assertCommand(serviceScript + " stop");
assertService(DEFAULT_ID, "Stopped", DEFAULT_DISPLAY_NAME);
// the process is stopped async, and can become a zombie process, so we poll for the process actually being gone
assertCommand("$p = Get-Service -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;" +
"$i = 0;" +
"do {" +
"$p = Get-Process -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;" +
"echo \"$p\";" +
"if ($p -eq $Null) {" +
" Write-Host \"exited after $i seconds\";" +
" exit 0;" +
"}" +
"Start-Sleep -Seconds 1;" +
"$i += 1;" +
"} while ($i -lt 300);" +
"exit 9;");
assertCommand(
"$p = Get-Service -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;"
+ "$i = 0;"
+ "do {"
+ " $p = Get-Process -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;"
+ " echo \"$p\";"
+ " if ($p -eq $Null) {"
+ " Write-Host \"exited after $i seconds\";"
+ " exit 0;"
+ " }"
+ " Start-Sleep -Seconds 1;"
+ " $i += 1;"
+ "} while ($i -lt 300);"
+ "exit 9;"
);
assertCommand(serviceScript + " remove");
assertCommand("$p = Get-Service -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;" +
"echo \"$p\";" +
"if ($p -eq $Null) {" +
" exit 0;" +
"} else {" +
" exit 1;" +
"}");
assertCommand(
"$p = Get-Service -Name \"elasticsearch-service-x64\" -ErrorAction SilentlyContinue;"
+ "echo \"$p\";"
+ "if ($p -eq $Null) {"
+ " exit 0;"
+ "} else {"
+ " exit 1;"
+ "}"
);
}
public void test30StartStop() throws Exception {
@ -231,12 +239,12 @@ public class WindowsServiceTests extends PackagingTestCase {
Path fakeServiceMgr = serviceMgr.getParent().resolve("elasticsearch-service-mgr.bat");
Files.write(fakeServiceMgr, Arrays.asList("echo \"Fake Service Manager GUI\""));
Shell sh = new Shell();
Result result = sh.run(serviceScript + " manager");
Result result = sh.run(serviceScript + " manager");
assertThat(result.stdout, containsString("Fake Service Manager GUI"));
// check failure too
Files.write(fakeServiceMgr, Arrays.asList("echo \"Fake Service Manager GUI Failure\"", "exit 1"));
result = sh.runIgnoreExitCode(serviceScript + " manager");
result = sh.runIgnoreExitCode(serviceScript + " manager");
TestCase.assertEquals(1, result.exitCode);
TestCase.assertTrue(result.stdout, result.stdout.contains("Fake Service Manager GUI Failure"));
Files.move(tmpServiceMgr, serviceMgr);

194
qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java
View File

@ -26,6 +26,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Locale;
import java.util.stream.Stream;
import static java.util.stream.Collectors.joining;
@ -56,12 +57,10 @@ import static org.hamcrest.core.IsNot.not;
*/
public class Archives {
protected static final Logger logger = LogManager.getLogger(Archives.class);
protected static final Logger logger = LogManager.getLogger(Archives.class);
// in the future we'll run as a role user on Windows
public static final String ARCHIVE_OWNER = Platforms.WINDOWS
? System.getenv("username")
: "elasticsearch";
public static final String ARCHIVE_OWNER = Platforms.WINDOWS ? System.getenv("username") : "elasticsearch";
/** This is an arbitrarily chosen value that gives Elasticsearch time to log Bootstrap
* errors to the console if they occur before the logging framework is initialized. */
@ -91,9 +90,12 @@ public class Archives {
if (Platforms.WINDOWS == false) {
throw new IllegalStateException("Distribution " + distribution + " is not supported on linux");
}
installCommand =
"Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " +
"[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')";
installCommand = String.format(
Locale.ROOT,
"Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; [IO.Compression.ZipFile]::ExtractToDirectory('%s', '%s')",
distributionFile,
baseInstallPath
);
} else {
throw new RuntimeException("Distribution " + distribution + " is not a known archive type");
@ -129,22 +131,26 @@ public class Archives {
if (sh.runIgnoreExitCode("id elasticsearch").isSuccess() == false) {
if (isDPKG()) {
sh.run("adduser " +
"--quiet " +
"--system " +
"--no-create-home " +
"--ingroup elasticsearch " +
"--disabled-password " +
"--shell /bin/false " +
"elasticsearch");
sh.run(
"adduser "
+ "--quiet "
+ "--system "
+ "--no-create-home "
+ "--ingroup elasticsearch "
+ "--disabled-password "
+ "--shell /bin/false "
+ "elasticsearch"
);
} else {
sh.run("useradd " +
"--system " +
"-M " +
"--gid elasticsearch " +
"--shell /sbin/nologin " +
"--comment 'elasticsearch user' " +
"elasticsearch");
sh.run(
"useradd "
+ "--system "
+ "-M "
+ "--gid elasticsearch "
+ "--shell /sbin/nologin "
+ "--comment 'elasticsearch user' "
+ "elasticsearch"
);
}
}
}
@ -157,13 +163,7 @@ public class Archives {
}
private static void verifyOssInstallation(Installation es, Distribution distribution, String owner) {
Stream.of(
es.home,
es.config,
es.plugins,
es.modules,
es.logs
).forEach(dir -> assertThat(dir, file(Directory, owner, owner, p755)));
Stream.of(es.home, es.config, es.plugins, es.modules, es.logs).forEach(dir -> assertThat(dir, file(Directory, owner, owner, p755)));
assertThat(Files.exists(es.data), is(false));
@ -188,24 +188,15 @@ public class Archives {
});
if (distribution.packaging == Distribution.Packaging.ZIP) {
Stream.of(
"elasticsearch-service.bat",
"elasticsearch-service-mgr.exe",
"elasticsearch-service-x64.exe"
).forEach(executable -> assertThat(es.bin(executable), file(File, owner)));
Stream.of("elasticsearch-service.bat", "elasticsearch-service-mgr.exe", "elasticsearch-service-x64.exe")
.forEach(executable -> assertThat(es.bin(executable), file(File, owner)));
}
Stream.of(
"elasticsearch.yml",
"jvm.options",
"log4j2.properties"
).forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
Stream.of("elasticsearch.yml", "jvm.options", "log4j2.properties")
.forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
Stream.of(
"NOTICE.txt",
"LICENSE.txt",
"README.asciidoc"
).forEach(doc -> assertThat(es.home.resolve(doc), file(File, owner, owner, p644)));
Stream.of("NOTICE.txt", "LICENSE.txt", "README.asciidoc")
.forEach(doc -> assertThat(es.home.resolve(doc), file(File, owner, owner, p644)));
}
private static void verifyDefaultInstallation(Installation es, Distribution distribution, String owner) {
@ -236,13 +227,8 @@ public class Archives {
// the version through here
assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, owner, owner, p755));
Stream.of(
"users",
"users_roles",
"roles.yml",
"role_mapping.yml",
"log4j2.properties"
).forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties")
.forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
}
public static Shell.Result startElasticsearch(Installation installation, Shell sh) {
@ -254,13 +240,20 @@ public class Archives {
final Installation.Executables bin = installation.executables();
// requires the "expect" utility to be installed
String script = "expect -c \"$(cat<<EXPECT\n" +
"spawn -ignore HUP sudo -E -u " + ARCHIVE_OWNER + " " + bin.elasticsearch + " -d -p " + pidFile + "\n" +
"expect \"Elasticsearch keystore password:\"\n" +
"send \"" + keystorePassword + "\\r\"\n" +
"expect eof\n" +
"EXPECT\n" +
")\"";
String script = String.format(
Locale.ROOT,
"expect -c \"$(cat<<EXPECT\n"
+ "spawn -ignore HUP sudo -E -u %s %s -d -p %s \n"
+ "expect \"Elasticsearch keystore password:\"\n"
+ "send \"%s\\r\"\n"
+ "expect eof\n"
+ "EXPECT\n"
+ ")\"",
ARCHIVE_OWNER,
bin.elasticsearch,
pidFile,
keystorePassword
);
sh.getEnv().put("ES_STARTUP_SLEEP_TIME", ES_STARTUP_SLEEP_TIME_SECONDS);
return sh.runIgnoreExitCode(script);
@ -283,8 +276,9 @@ public class Archives {
// We need to give Elasticsearch enough time to print failures to stderr before exiting
sh.getEnv().put("ES_STARTUP_SLEEP_TIME", ES_STARTUP_SLEEP_TIME_SECONDS);
return sh.runIgnoreExitCode("sudo -E -u " + ARCHIVE_OWNER + " " + bin.elasticsearch + " -d -p " + pidFile +
" <<<'" + keystorePassword + "'");
return sh.runIgnoreExitCode(
"sudo -E -u " + ARCHIVE_OWNER + " " + bin.elasticsearch + " -d -p " + pidFile + " <<<'" + keystorePassword + "'"
);
}
final Path stdout = getPowershellOutputPath(installation);
final Path stderr = getPowershellErrorPath(installation);
@ -294,45 +288,59 @@ public class Archives {
// the tests will run as Administrator in vagrant.
// we don't want to run the server as Administrator, so we provide the current user's
// username and password to the process which has the effect of starting it not as Administrator.
powerShellProcessUserSetup =
"$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " +
"$processInfo.Username = 'vagrant'; " +
"$processInfo.Password = $password; ";
powerShellProcessUserSetup = "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; "
+ "$processInfo.Username = 'vagrant'; "
+ "$processInfo.Password = $password; ";
} else {
powerShellProcessUserSetup = "";
}
// this starts the server in the background. the -d flag is unsupported on windows
return sh.run(
"$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " +
"$processInfo.FileName = '" + bin.elasticsearch + "'; " +
"$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " +
powerShellProcessUserSetup +
"$processInfo.RedirectStandardOutput = $true; " +
"$processInfo.RedirectStandardError = $true; " +
"$processInfo.RedirectStandardInput = $true; " +
sh.env.entrySet().stream()
"$processInfo = New-Object System.Diagnostics.ProcessStartInfo; "
+ "$processInfo.FileName = '"
+ bin.elasticsearch
+ "'; "
+ "$processInfo.Arguments = '-p "
+ installation.home.resolve("elasticsearch.pid")
+ "'; "
+ powerShellProcessUserSetup
+ "$processInfo.RedirectStandardOutput = $true; "
+ "$processInfo.RedirectStandardError = $true; "
+ "$processInfo.RedirectStandardInput = $true; "
+ sh.env.entrySet()
.stream()
.map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ")
.collect(joining()) +
"$processInfo.UseShellExecute = $false; " +
"$process = New-Object System.Diagnostics.Process; " +
"$process.StartInfo = $processInfo; " +
.collect(joining())
+ "$processInfo.UseShellExecute = $false; "
+ "$process = New-Object System.Diagnostics.Process; "
+ "$process.StartInfo = $processInfo; "
+
// set up some asynchronous output handlers
"$outScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '" + stdout + "' }; " +
"$errScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '" + stderr + "' }; " +
"$stdOutEvent = Register-ObjectEvent -InputObject $process " +
"-Action $outScript -EventName 'OutputDataReceived'; " +
"$stdErrEvent = Register-ObjectEvent -InputObject $process " +
"-Action $errScript -EventName 'ErrorDataReceived'; " +
"$outScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '"
+ stdout
+ "' }; "
+ "$errScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '"
+ stderr
+ "' }; "
+ "$stdOutEvent = Register-ObjectEvent -InputObject $process "
+ "-Action $outScript -EventName 'OutputDataReceived'; "
+ "$stdErrEvent = Register-ObjectEvent -InputObject $process "
+ "-Action $errScript -EventName 'ErrorDataReceived'; "
+
"$process.Start() | Out-Null; " +
"$process.BeginOutputReadLine(); " +
"$process.BeginErrorReadLine(); " +
"$process.StandardInput.WriteLine('" + keystorePassword + "'); " +
"Wait-Process -Timeout " + ES_STARTUP_SLEEP_TIME_SECONDS + " -Id $process.Id; " +
"$process.Id;"
);
"$process.Start() | Out-Null; "
+ "$process.BeginOutputReadLine(); "
+ "$process.BeginErrorReadLine(); "
+ "$process.StandardInput.WriteLine('"
+ keystorePassword
+ "'); "
+ "Wait-Process -Timeout "
+ ES_STARTUP_SLEEP_TIME_SECONDS
+ " -Id $process.Id; "
+ "$process.Id;"
);
}
public static void assertElasticsearchStarted(Installation installation) throws Exception {
@ -356,9 +364,11 @@ public class Archives {
sh.run("Get-Process -Id " + pid + " | Stop-Process -Force; Wait-Process -Id " + pid);
// Clear the asynchronous event handlers
sh.runIgnoreExitCode("Get-EventSubscriber | " +
"where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |" +
"Unregister-EventSubscriber -Force");
sh.runIgnoreExitCode(
"Get-EventSubscriber | "
+ "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |"
+ "Unregister-EventSubscriber -Force"
);
});
if (Files.exists(pidFile)) {
Files.delete(pidFile);

23
qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java
View File

@ -62,14 +62,16 @@ public class Cleanup {
sh.runIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9");
});
Platforms.onWindows(() -> {
// the view of processes returned by Get-Process doesn't expose command line arguments, so we use WMI here
sh.runIgnoreExitCode(
"Get-WmiObject Win32_Process | " +
"Where-Object { $_.CommandLine -Match 'org.elasticsearch.bootstrap.Elasticsearch' } | " +
"ForEach-Object { $_.Terminate() }"
);
});
Platforms.onWindows(
() -> {
// the view of processes returned by Get-Process doesn't expose command line arguments, so we use WMI here
sh.runIgnoreExitCode(
"Get-WmiObject Win32_Process | "
+ "Where-Object { $_.CommandLine -Match 'org.elasticsearch.bootstrap.Elasticsearch' } | "
+ "ForEach-Object { $_.Terminate() }"
);
}
);
Platforms.onLinux(Cleanup::purgePackagesLinux);
@ -85,10 +87,7 @@ public class Cleanup {
final List<String> filesToDelete = Platforms.WINDOWS ? ELASTICSEARCH_FILES_WINDOWS : ELASTICSEARCH_FILES_LINUX;
// windows needs leniency due to asinine releasing of file locking async from a process exiting
Consumer<? super Path> rm = Platforms.WINDOWS ? FileUtils::rmWithRetries : FileUtils::rm;
filesToDelete.stream()
.map(Paths::get)
.filter(Files::exists)
.forEach(rm);
filesToDelete.stream().map(Paths::get).filter(Files::exists).forEach(rm);
// disable elasticsearch service
// todo add this for windows when adding tests for service intallation

5
qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java
View File

@ -79,10 +79,7 @@ public class Docker {
public static void ensureImageIsLoaded(Distribution distribution) {
Shell.Result result = sh.run("docker image ls --format '{{.Repository}}' " + distribution.flavor.name);
final long count = Arrays.stream(result.stdout.split("\n"))
.map(String::trim)
.filter(s -> s.isEmpty() == false)
.count();
final long count = Arrays.stream(result.stdout.split("\n")).map(String::trim).filter(s -> s.isEmpty() == false).count();
if (count != 0) {
return;

17
qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java
View File

@ -43,7 +43,10 @@ import static org.elasticsearch.packaging.util.FileUtils.getPosixFileAttributes;
*/
public class FileMatcher extends TypeSafeMatcher<Path> {
public enum Fileness { File, Directory }
public enum Fileness {
File,
Directory
}
public static final Set<PosixFilePermission> p775 = fromString("rwxrwxr-x");
public static final Set<PosixFilePermission> p770 = fromString("rwxrwx---");
@ -126,10 +129,14 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
@Override
public void describeTo(Description description) {
description.appendValue("file/directory: ").appendValue(fileness)
.appendText(" with owner ").appendValue(owner)
.appendText(" with group ").appendValue(group)
.appendText(" with posix permissions ").appendValueList("[", ",", "]", posixPermissions);
description.appendValue("file/directory: ")
.appendValue(fileness)
.appendText(" with owner ")
.appendValue(owner)
.appendText(" with group ")
.appendValue(group)
.appendText(" with posix permissions ")
.appendValueList("[", ",", "]", posixPermissions);
}
public static FileMatcher file(Fileness fileness, String owner) {

53
qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java
View File

@ -34,6 +34,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
@ -111,22 +112,21 @@ public class FileUtils {
public static Path mktempDir(Path path) {
try {
return Files.createTempDirectory(path,"tmp");
return Files.createTempDirectory(path, "tmp");
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static Path mkdir(Path path) {
try {
return Files.createDirectories(path);
} catch (IOException e) {
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
public static Path cp(Path source, Path target) {
public static Path cp(Path source, Path target) {
try {
return Files.copy(source, target);
} catch (IOException e) {
@ -142,9 +142,22 @@ public class FileUtils {
}
}
/**
* Creates or appends to the specified file, and writes the supplied string to it.
* No newline is written - if a trailing newline is required, it should be present
* in <code>text</code>, or use {@link Files#write(Path, Iterable, OpenOption...)}.
* @param file the file to create or append
* @param text the string to write
*/
public static void append(Path file, String text) {
try (BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8,
StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
try (
BufferedWriter writer = Files.newBufferedWriter(
file,
StandardCharsets.UTF_8,
StandardOpenOption.CREATE,
StandardOpenOption.APPEND
)
) {
writer.write(text);
} catch (IOException e) {
@ -204,7 +217,7 @@ public class FileUtils {
for (Path rotatedLogFile : FileUtils.lsGlob(logPath, rotatedLogFilesGlob)) {
logFileJoiner.add(FileUtils.slurpTxtorGz(rotatedLogFile));
}
return(logFileJoiner.toString());
return (logFileJoiner.toString());
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -221,14 +234,14 @@ public class FileUtils {
// gc logs are verbose and not useful in this context
.filter(file -> file.getFileName().toString().startsWith("gc.log") == false)
.forEach(file -> {
logger.info("=== Contents of `{}` ({}) ===", file, file.toAbsolutePath());
try (Stream<String> stream = Files.lines(file)) {
stream.forEach(logger::info);
} catch (IOException e) {
logger.error("Can't show contents", e);
}
logger.info("=== End of contents of `{}`===", file);
});
logger.info("=== Contents of `{}` ({}) ===", file, file.toAbsolutePath());
try (Stream<String> stream = Files.lines(file)) {
stream.forEach(logger::info);
} catch (IOException e) {
logger.error("Can't show contents", e);
}
logger.info("=== End of contents of `{}`===", file);
});
} catch (IOException e) {
logger.error("Can't list log files", e);
}
@ -284,7 +297,6 @@ public class FileUtils {
return numericPathOwnership;
}
// vagrant creates /tmp for us in windows so we use that to avoid long paths
public static Path getTempDir() {
return Paths.get("/tmp").toAbsolutePath();
@ -295,6 +307,7 @@ public class FileUtils {
}
private static final Pattern VERSION_REGEX = Pattern.compile("(\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?)");
public static String getCurrentVersion() {
// TODO: just load this once
String distroFile = System.getProperty("tests.distribution");
@ -314,12 +327,12 @@ public class FileUtils {
}
public static Matcher<Path> fileWithGlobExist(String glob) throws IOException {
return new FeatureMatcher<Path,Iterable<Path>>(not(emptyIterable()),"File with pattern exist", "file with pattern"){
return new FeatureMatcher<Path, Iterable<Path>>(not(emptyIterable()), "File with pattern exist", "file with pattern") {
@Override
protected Iterable<Path> featureValueOf(Path actual) {
try {
return Files.newDirectoryStream(actual,glob);
return Files.newDirectoryStream(actual, glob);
} catch (IOException e) {
return Collections.emptyList();
}

22
qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java
View File

@ -28,9 +28,7 @@ import java.nio.file.Paths;
public class Installation {
// in the future we'll run as a role user on Windows
public static final String ARCHIVE_OWNER = Platforms.WINDOWS
? System.getenv("username")
: "elasticsearch";
public static final String ARCHIVE_OWNER = Platforms.WINDOWS ? System.getenv("username") : "elasticsearch";
private final Shell sh;
public final Distribution distribution;
@ -46,8 +44,18 @@ public class Installation {
public final Path pidDir;
public final Path envFile;
private Installation(Shell sh, Distribution distribution, Path home, Path config, Path data, Path logs,
Path plugins, Path modules, Path pidDir, Path envFile) {
private Installation(
Shell sh,
Distribution distribution,
Path home,
Path config,
Path data,
Path logs,
Path plugins,
Path modules,
Path pidDir,
Path envFile
) {
this.sh = sh;
this.distribution = distribution;
this.home = home;
@ -147,9 +155,7 @@ public class Installation {
public final Path path;
private Executable(String name) {
final String platformExecutableName = Platforms.WINDOWS
? name + ".bat"
: name;
final String platformExecutableName = Platforms.WINDOWS ? name + ".bat" : name;
this.path = bin(platformExecutableName);
}

74
qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java
View File

@ -24,7 +24,6 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.packaging.util.Shell.Result;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -32,6 +31,7 @@ import java.nio.file.StandardOpenOption;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static java.util.Collections.singletonList;
import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory;
import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
@ -53,7 +53,7 @@ import static org.junit.Assert.assertTrue;
public class Packages {
private static final Logger logger = LogManager.getLogger(Packages.class);
private static final Logger logger = LogManager.getLogger(Packages.class);
public static final Path SYSVINIT_SCRIPT = Paths.get("/etc/init.d/elasticsearch");
public static final Path SYSTEMD_SERVICE = Paths.get("/usr/lib/systemd/system/elasticsearch.service");
@ -73,9 +73,10 @@ public class Packages {
Platforms.onDPKG(() -> {
assertThat(status.exitCode, anyOf(is(0), is(1)));
if (status.exitCode == 0) {
assertTrue("an uninstalled status should be indicated: " + status.stdout,
Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find() ||
Pattern.compile("(?m)^Status:.+ok not-installed").matcher(status.stdout).find()
assertTrue(
"an uninstalled status should be indicated: " + status.stdout,
Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find()
|| Pattern.compile("(?m)^Status:.+ok not-installed").matcher(status.stdout).find()
);
}
});
@ -108,8 +109,7 @@ public class Packages {
Installation installation = Installation.ofPackage(sh, distribution);
if (distribution.hasJdk == false) {
Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
StandardOpenOption.APPEND);
Files.write(installation.envFile, singletonList("JAVA_HOME=" + systemJavaHome), StandardOpenOption.APPEND);
}
return installation;
}
@ -124,9 +124,7 @@ public class Packages {
if (r.exitCode != 0) {
Result lockOF = sh.runIgnoreExitCode("lsof /var/lib/dpkg/lock");
if (lockOF.exitCode == 0) {
throw new RuntimeException(
"dpkg failed and the lockfile still exists. "
+ "Failure:\n" + r + "\nLockfile:\n" + lockOF);
throw new RuntimeException("dpkg failed and the lockfile still exists. " + "Failure:\n" + r + "\nLockfile:\n" + lockOF);
}
}
return r;
@ -157,7 +155,6 @@ public class Packages {
}
}
private static void verifyOssInstallation(Installation es, Distribution distribution, Shell sh) {
sh.run("id elasticsearch");
@ -167,16 +164,9 @@ public class Packages {
final Path homeDir = Paths.get(passwdResult.stdout.trim().split(":")[5]);
assertThat("elasticsearch user home directory must not exist", homeDir, fileDoesNotExist());
Stream.of(
es.home,
es.plugins,
es.modules
).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
Stream.of(es.home, es.plugins, es.modules).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
Stream.of(
es.data,
es.logs
).forEach(dir -> assertThat(dir, file(Directory, "elasticsearch", "elasticsearch", p750)));
Stream.of(es.data, es.logs).forEach(dir -> assertThat(dir, file(Directory, "elasticsearch", "elasticsearch", p750)));
// we shell out here because java's posix file permission view doesn't support special modes
assertThat(es.config, file(Directory, "root", "elasticsearch", p750));
@ -186,33 +176,18 @@ public class Packages {
assertThat(jvmOptionsDirectory, file(Directory, "root", "elasticsearch", p750));
assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
Stream.of(
"elasticsearch.keystore",
"elasticsearch.yml",
"jvm.options",
"log4j2.properties"
).forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
Stream.of("elasticsearch.keystore", "elasticsearch.yml", "jvm.options", "log4j2.properties")
.forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
assertThat(es.config(".elasticsearch.keystore.initial_md5sum"), file(File, "root", "elasticsearch", p644));
assertThat(sh.run("sudo -u elasticsearch " + es.bin("elasticsearch-keystore") + " list").stdout, containsString("keystore.seed"));
Stream.of(
es.bin,
es.lib
).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
Stream.of(es.bin, es.lib).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
Stream.of(
"elasticsearch",
"elasticsearch-plugin",
"elasticsearch-keystore",
"elasticsearch-shard",
"elasticsearch-node"
).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
Stream.of("elasticsearch", "elasticsearch-plugin", "elasticsearch-keystore", "elasticsearch-shard", "elasticsearch-node")
.forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
Stream.of(
"NOTICE.txt",
"README.asciidoc"
).forEach(doc -> assertThat(es.home.resolve(doc), file(File, "root", "root", p644)));
Stream.of("NOTICE.txt", "README.asciidoc").forEach(doc -> assertThat(es.home.resolve(doc), file(File, "root", "root", p644)));
assertThat(es.envFile, file(File, "root", "elasticsearch", p660));
@ -231,9 +206,7 @@ public class Packages {
Paths.get("/usr/lib/sysctl.d/elasticsearch.conf")
).forEach(confFile -> assertThat(confFile, file(File, "root", "root", p644)));
final String sysctlExecutable = (distribution.packaging == Distribution.Packaging.RPM)
? "/usr/sbin/sysctl"
: "/sbin/sysctl";
final String sysctlExecutable = (distribution.packaging == Distribution.Packaging.RPM) ? "/usr/sbin/sysctl" : "/sbin/sysctl";
assertThat(sh.run(sysctlExecutable + " vm.max_map_count").stdout, containsString("vm.max_map_count = 262144"));
}
@ -263,13 +236,8 @@ public class Packages {
// the version through here
assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, "root", "root", p755));
Stream.of(
"users",
"users_roles",
"roles.yml",
"role_mapping.yml",
"log4j2.properties"
).forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties")
.forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
}
/**
@ -337,8 +305,8 @@ public class Packages {
* for Elasticsearch logs and storing it in class state.
*/
public void clear() {
cursor = sh.run("sudo journalctl --unit=elasticsearch.service --lines=0 --show-cursor -o cat" +
" | sed -e 's/-- cursor: //'").stdout.trim();
final String script = "sudo journalctl --unit=elasticsearch.service --lines=0 --show-cursor -o cat | sed -e 's/-- cursor: //'";
cursor = sh.run(script).stdout.trim();
}
/**

26
qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java
View File

@ -58,12 +58,12 @@ import static org.hamcrest.Matchers.containsString;
public class ServerUtils {
private static final Logger logger = LogManager.getLogger(ServerUtils.class);
private static final Logger logger = LogManager.getLogger(ServerUtils.class);
private static String SECURITY_ENABLED = "xpack.security.enabled: true";
private static String SSL_ENABLED = "xpack.security.http.ssl.enabled: true";
// generous timeout as nested virtualization can be quite slow ...
// generous timeout as nested virtualization can be quite slow ...
private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
@ -124,9 +124,7 @@ public class ServerUtils {
connectionManager.setDefaultMaxPerRoute(100);
connectionManager.setMaxTotal(200);
connectionManager.setValidateAfterInactivity(1000);
executor = Executor.newInstance(HttpClientBuilder.create()
.setConnectionManager(connectionManager)
.build());
executor = Executor.newInstance(HttpClientBuilder.create().setConnectionManager(connectionManager).build());
}
} else {
executor = Executor.newInstance();
@ -159,13 +157,8 @@ public class ServerUtils {
throw new RuntimeException("Elasticsearch (with x-pack) did not start");
}
public static void waitForElasticsearch(
String status,
String index,
Installation installation,
String username,
String password
) throws Exception {
public static void waitForElasticsearch(String status, String index, Installation installation, String username, String password)
throws Exception {
Objects.requireNonNull(status);
@ -186,8 +179,7 @@ public class ServerUtils {
try {
final HttpResponse response = execute(
Request
.Get("http://localhost:9200/_cluster/health")
Request.Get("http://localhost:9200/_cluster/health")
.connectTimeout((int) timeoutLength)
.socketTimeout((int) timeoutLength),
username,
@ -239,11 +231,13 @@ public class ServerUtils {
public static void runElasticsearchTests() throws Exception {
makeRequest(
Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty")
.bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON));
.bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)
);
makeRequest(
Request.Post("http://localhost:9200/library/book/2?refresh=true&pretty")
.bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON));
.bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON)
);
String count = makeRequest(Request.Get("http://localhost:9200/_count?pretty"));
assertThat(count, containsString("\"count\" : 2"));

80
qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java
View File

@ -42,9 +42,8 @@ import java.util.stream.Stream;
*/
public class Shell {
public static final int TAIL_WHEN_TOO_MUCH_OUTPUT = 1000;
public static final Result NO_OP = new Shell.Result(0, "","");
protected final Logger logger = LogManager.getLogger(getClass());
public static final Result NO_OP = new Shell.Result(0, "", "");
protected final Logger logger = LogManager.getLogger(getClass());
final Map<String, String> env = new HashMap<>();
Path workingDirectory;
@ -86,20 +85,28 @@ public class Shell {
public void chown(Path path) throws Exception {
Platforms.onLinux(() -> run("chown -R elasticsearch:elasticsearch " + path));
Platforms.onWindows(() -> run(
"$account = New-Object System.Security.Principal.NTAccount '" + System.getenv("username") + "'; " +
"$pathInfo = Get-Item '" + path + "'; " +
"$toChown = @(); " +
"if ($pathInfo.PSIsContainer) { " +
" $toChown += Get-ChildItem '" + path + "' -Recurse; " +
"}" +
"$toChown += $pathInfo; " +
"$toChown | ForEach-Object { " +
"$acl = Get-Acl $_.FullName; " +
"$acl.SetOwner($account); " +
"Set-Acl $_.FullName $acl " +
"}"
));
Platforms.onWindows(
() -> run(
String.format(
Locale.ROOT,
"$account = New-Object System.Security.Principal.NTAccount '%s'; "
+ "$pathInfo = Get-Item '%s'; "
+ "$toChown = @(); "
+ "if ($pathInfo.PSIsContainer) { "
+ " $toChown += Get-ChildItem '%s' -Recurse; "
+ "}"
+ "$toChown += $pathInfo; "
+ "$toChown | ForEach-Object { "
+ " $acl = Get-Acl $_.FullName; "
+ " $acl.SetOwner($account); "
+ " Set-Acl $_.FullName $acl "
+ "}",
System.getenv("username"),
path,
path
)
)
);
}
public void extractZip(Path zipPath, Path destinationDir) throws Exception {
@ -165,22 +172,13 @@ public class Shell {
if (process.isAlive()) {
process.destroyForcibly();
}
Result result = new Result(
-1,
readFileIfExists(stdOut),
readFileIfExists(stdErr)
);
Result result = new Result(-1, readFileIfExists(stdOut), readFileIfExists(stdErr));
throw new IllegalStateException(
"Timed out running shell command: " + Arrays.toString(command) + "\n" +
"Result:\n" + result
"Timed out running shell command: " + Arrays.toString(command) + "\n" + "Result:\n" + result
);
}
Result result = new Result(
process.exitValue(),
readFileIfExists(stdOut),
readFileIfExists(stdErr)
);
Result result = new Result(process.exitValue(), readFileIfExists(stdOut), readFileIfExists(stdErr));
logger.info("Ran: {} {}", Arrays.toString(command), result);
return result;
@ -203,7 +201,7 @@ public class Shell {
if (Files.exists(path)) {
long size = Files.size(path);
if (size > 100 * 1024) {
return "<<Too large to read: " + size + " bytes>>";
return "<<Too large to read: " + size + " bytes>>";
}
try (Stream<String> lines = Files.lines(path, StandardCharsets.UTF_8)) {
return lines.collect(Collectors.joining("\n"));
@ -225,15 +223,7 @@ public class Shell {
}
public String toString() {
return new StringBuilder()
.append(" ")
.append("env = [")
.append(env)
.append("]")
.append("workingDirectory = [")
.append(workingDirectory)
.append("]")
.toString();
return String.format(Locale.ROOT, " env = [%s] workingDirectory = [%s]", env, workingDirectory);
}
public static class Result {
@ -252,17 +242,7 @@ public class Shell {
}
public String toString() {
return new StringBuilder()
.append("exitCode = [")
.append(exitCode)
.append("] ")
.append("stdout = [")
.append(stdout.trim())
.append("] ")
.append("stderr = [")
.append(stderr.trim())
.append("]")
.toString();
return String.format(Locale.ROOT, "exitCode = [%d] stdout = [%s] stderr = [%s]", exitCode, stdout.trim(), stderr.trim());
}
}