diff --git a/qa/os/bats/default/40_tar_certgen.bats b/qa/os/bats/default/40_tar_certgen.bats deleted file mode 120000 index c9a929d829e..00000000000 --- a/qa/os/bats/default/40_tar_certgen.bats +++ /dev/null @@ -1 +0,0 @@ -certgen.bash \ No newline at end of file diff --git a/qa/os/bats/default/45_package_certgen.bats b/qa/os/bats/default/45_package_certgen.bats deleted file mode 120000 index c9a929d829e..00000000000 --- a/qa/os/bats/default/45_package_certgen.bats +++ /dev/null @@ -1 +0,0 @@ -certgen.bash \ No newline at end of file diff --git a/qa/os/bats/default/certgen.bash b/qa/os/bats/default/certgen.bash deleted file mode 100644 index b52e286742c..00000000000 --- a/qa/os/bats/default/certgen.bash +++ /dev/null @@ -1,433 +0,0 @@ -#!/usr/bin/env bats - -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -load $BATS_UTILS/utils.bash -load $BATS_UTILS/plugins.bash -load $BATS_UTILS/xpack.bash - -# Description of the nodes instances -instances="/tmp/instances.yml" - -# Destination for generated certificates -certificates="/tmp/certificates.zip" - -setup() { - if [ $BATS_TEST_NUMBER == 1 ]; then - clean_before_test - fi -} - -DEFAULT_ARCHIVE_USER=elasticsearch -DEFAULT_ARCHIVE_ESHOME="/tmp/elasticsearch" -DEFAULT_ARCHIVE_UTILS=$BATS_UTILS/tar.bash - -DEFAULT_PACKAGE_USER=root -DEFAULT_PACKAGE_ESHOME="/usr/share/elasticsearch" -DEFAULT_PACKAGE_UTILS=$BATS_UTILS/packages.bash - -if [[ "$BATS_TEST_FILENAME" =~ 40_tar_certgen.bats$ ]]; then - GROUP='TAR CERTGEN' - - MASTER_USER=$DEFAULT_ARCHIVE_USER - MASTER_GROUP=$DEFAULT_ARCHIVE_USER - MASTER_DPERMS=755 - MASTER_HOME=$DEFAULT_ARCHIVE_ESHOME - MASTER_UTILS=$DEFAULT_ARCHIVE_UTILS - - DATA_USER=$DEFAULT_PACKAGE_USER - DATA_GROUP=elasticsearch - DATA_DPERMS=2755 - DATA_HOME=$DEFAULT_PACKAGE_ESHOME - DATA_UTILS=$DEFAULT_PACKAGE_UTILS - - install_master_node() { - install_node_using_archive - } - start_master_node() { - start_node_using_archive - } - install_data_node() { - install_node_using_package - } - start_data_node() { - start_node_using_package - } -else - if is_rpm; then - GROUP='RPM CERTGEN' - elif is_dpkg; then - GROUP='DEB CERTGEN' - fi - - MASTER_USER=$DEFAULT_PACKAGE_USER - MASTER_GROUP=elasticsearch - MASTER_DPERMS=2755 - MASTER_HOME=$DEFAULT_PACKAGE_ESHOME - MASTER_UTILS=$DEFAULT_PACKAGE_UTILS - - DATA_USER=$DEFAULT_ARCHIVE_USER - DATA_GROUP=$DEFAULT_ARCHIVE_USER - DATA_DPERMS=755 - DATA_HOME=$DEFAULT_ARCHIVE_ESHOME - DATA_UTILS=$DEFAULT_ARCHIVE_UTILS - - install_master_node() { - install_node_using_package - } - start_master_node() { - start_node_using_package - } - install_data_node() { - install_node_using_archive - } - start_data_node() { - start_node_using_archive - } -fi - -# Install a node with x-pack using the archive file -install_node_using_archive() { - load $BATS_UTILS/tar.bash - export ESHOME="$DEFAULT_ARCHIVE_ESHOME" - export_elasticsearch_paths - - install_archive - set_debug_logging - verify_archive_installation - - export ESPLUGIN_COMMAND_USER=$DEFAULT_ARCHIVE_USER - generate_trial_license - verify_xpack_installation -} - -# Starts a node installed using the archive -start_node_using_archive() { - load $BATS_UTILS/tar.bash - export ESHOME="$DEFAULT_ARCHIVE_ESHOME" - export_elasticsearch_paths - - run sudo -u $DEFAULT_ARCHIVE_USER "$ESHOME/bin/elasticsearch" -d -p $ESHOME/elasticsearch.pid - [ "$status" -eq "0" ] || { - echo "Failed to start node using archive: $output" - false - } -} - -# Install a node with x-pack using a package file -install_node_using_package() { - load $BATS_UTILS/packages.bash - export ESHOME="$DEFAULT_PACKAGE_ESHOME" - export_elasticsearch_paths - - install_package - set_debug_logging - verify_package_installation - - export ESPLUGIN_COMMAND_USER=$DEFAULT_PACKAGE_USER - generate_trial_license - verify_xpack_installation -} - -# Starts a node installed using a package -start_node_using_package() { - if is_systemd; then - run systemctl daemon-reload - [ "$status" -eq 0 ] - - run sudo systemctl start elasticsearch.service - [ "$status" -eq "0" ] - - elif is_sysvinit; then - run sudo service elasticsearch start - [ "$status" -eq "0" ] - fi -} - - -@test "[$GROUP] install master node" { - install_master_node -} - -@test "[$GROUP] add bootstrap password" { - load $MASTER_UTILS - export ESHOME="$MASTER_HOME" - export_elasticsearch_paths - - # For the sake of simplicity we use a bootstrap password in this test. The - # alternative would be to start the master node, use - # elasticsearch-setup-passwords and restart the node once ssl/tls is - # configured. Or use elasticsearch-setup-passwords over HTTPS with the right - # cacerts imported into a Java keystore. - run sudo -E -u $MASTER_USER bash <<"NEW_PASS" -if [[ ! -f $ESCONFIG/elasticsearch.keystore ]]; then - $ESHOME/bin/elasticsearch-keystore create -fi -echo "changeme" | $ESHOME/bin/elasticsearch-keystore add --stdin bootstrap.password -NEW_PASS - [ "$status" -eq 0 ] || { - echo "Expected elasticsearch-keystore tool exit code to be zero" - echo "$output" - false - } -} - -@test "[$GROUP] create instances file" { - rm -f /tmp/instances.yml - run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE" -cat > /tmp/instances.yml <<- EOF -instances: - - name: "node-master" - ip: - - "127.0.0.1" - - name: "node-data" - ip: - - "127.0.0.1" -EOF -CREATE_INSTANCES_FILE - - [ "$status" -eq 0 ] || { - echo "Failed to create instances file [$instances]: $output" - false - } -} - -@test "[$GROUP] create certificates" { - if [[ -f "$certificates" ]]; then - sudo rm -f "$certificates" - fi - - run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --in "$instances" --out "$certificates" - [ "$status" -eq 0 ] || { - echo "Expected elasticsearch-certgen tool exit code to be zero" - echo "$output" - false - } - - echo "$output" | grep "Certificates written to $certificates" - assert_file "$certificates" f $MASTER_USER $MASTER_USER 600 -} - -@test "[$GROUP] install certificates on master node" { - load $MASTER_UTILS - export ESHOME="$MASTER_HOME" - export_elasticsearch_paths - - certs="$ESCONFIG/certs" - if [[ -d "$certs" ]]; then - sudo rm -rf "$certs" - fi - - run sudo -E -u $MASTER_USER "unzip" $certificates -d $certs - [ "$status" -eq 0 ] || { - echo "Failed to unzip certificates in $certs: $output" - false - } - - assert_file "$certs/ca/ca.key" f $MASTER_USER $MASTER_GROUP 644 - assert_file "$certs/ca/ca.crt" f $MASTER_USER $MASTER_GROUP 644 - - assert_file "$certs/node-master" d $MASTER_USER $MASTER_GROUP $MASTER_DPERMS - assert_file "$certs/node-master/node-master.key" f $MASTER_USER $MASTER_GROUP 644 - assert_file "$certs/node-master/node-master.crt" f $MASTER_USER $MASTER_GROUP 644 - - assert_file "$certs/node-data" d $MASTER_USER $MASTER_GROUP $MASTER_DPERMS - assert_file "$certs/node-data/node-data.key" f $MASTER_USER $MASTER_GROUP 644 - assert_file "$certs/node-data/node-data.crt" f $MASTER_USER $MASTER_GROUP 644 -} - -@test "[$GROUP] update master node settings" { - load $MASTER_UTILS - export ESHOME="$MASTER_HOME" - export_elasticsearch_paths - - run sudo -E -u $MASTER_USER bash <<"MASTER_SETTINGS" -cat >> $ESCONFIG/elasticsearch.yml <<- EOF -node.name: "node-master" -node.master: true -node.data: false -discovery.seed_hosts: ["127.0.0.1:9301"] -cluster.initial_master_nodes: ["node-master"] - -xpack.security.transport.ssl.key: $ESCONFIG/certs/node-master/node-master.key -xpack.security.transport.ssl.certificate: $ESCONFIG/certs/node-master/node-master.crt -xpack.security.transport.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"] -xpack.security.http.ssl.key: $ESCONFIG/certs/node-master/node-master.key -xpack.security.http.ssl.certificate: $ESCONFIG/certs/node-master/node-master.crt -xpack.security.http.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"] - -xpack.security.transport.ssl.enabled: true -transport.port: 9300 - -xpack.security.http.ssl.enabled: true -http.port: 9200 - -EOF -MASTER_SETTINGS - - start_master_node - wait_for_xpack 127.0.0.1 9200 -} - -@test "[$GROUP] test connection to master node using HTTPS" { - load $MASTER_UTILS - export ESHOME="$MASTER_HOME" - export_elasticsearch_paths - - run sudo -E -u $MASTER_USER curl -u "elastic:changeme" --cacert "$ESCONFIG/certs/ca/ca.crt" -XGET "https://127.0.0.1:9200" - [ "$status" -eq 0 ] || { - echo "Failed to connect to master node using HTTPS:" - echo "$output" - debug_collect_logs - false - } - echo "$output" | grep "node-master" -} - -@test "[$GROUP] install data node" { - install_data_node -} - -@test "[$GROUP] install certificates on data node" { - load $DATA_UTILS - export ESHOME="$DATA_HOME" - export_elasticsearch_paths - - sudo chown $DATA_USER:$DATA_USER "$certificates" - [ -f "$certificates" ] || { - echo "Could not find certificates: $certificates" - false - } - - certs="$ESCONFIG/certs" - if [[ -d "$certs" ]]; then - sudo rm -rf "$certs" - fi - - run sudo -E -u $DATA_USER "unzip" $certificates -d $certs - [ "$status" -eq 0 ] || { - echo "Failed to unzip certificates in $certs: $output" - false - } - - assert_file "$certs/ca" d $DATA_USER $DATA_GROUP - assert_file "$certs/ca/ca.key" f $DATA_USER $DATA_GROUP 644 - assert_file "$certs/ca/ca.crt" f $DATA_USER $DATA_GROUP 644 - - assert_file "$certs/node-master" d $DATA_USER $DATA_GROUP - assert_file "$certs/node-master/node-master.key" f $DATA_USER $DATA_GROUP 644 - assert_file "$certs/node-master/node-master.crt" f $DATA_USER $DATA_GROUP 644 - - assert_file "$certs/node-data" d $DATA_USER $DATA_GROUP - assert_file "$certs/node-data/node-data.key" f $DATA_USER $DATA_GROUP 644 - assert_file "$certs/node-data/node-data.crt" f $DATA_USER $DATA_GROUP 644 -} - -@test "[$GROUP] update data node settings" { - load $DATA_UTILS - export ESHOME="$DATA_HOME" - export_elasticsearch_paths - - run sudo -E -u $DATA_USER bash <<"DATA_SETTINGS" -cat >> $ESCONFIG/elasticsearch.yml <<- EOF -node.name: "node-data" -node.master: false -node.data: true -discovery.seed_hosts: ["127.0.0.1:9300"] - -xpack.security.transport.ssl.key: $ESCONFIG/certs/node-data/node-data.key -xpack.security.transport.ssl.certificate: $ESCONFIG/certs/node-data/node-data.crt -xpack.security.transport.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"] -xpack.security.http.ssl.key: $ESCONFIG/certs/node-data/node-data.key -xpack.security.http.ssl.certificate: $ESCONFIG/certs//node-data/node-data.crt -xpack.security.http.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"] - -xpack.security.transport.ssl.enabled: true -transport.tcp.port: 9301 - -xpack.security.http.ssl.enabled: true -http.port: 9201 - -EOF -DATA_SETTINGS - - start_data_node - wait_for_xpack 127.0.0.1 9201 -} - -@test "[$GROUP] test connection to data node using HTTPS" { - load $DATA_UTILS - export ESHOME="$DATA_HOME" - export_elasticsearch_paths - - run sudo -E -u $DATA_USER curl --cacert "$ESCONFIG/certs/ca/ca.crt" -XGET "https://127.0.0.1:9201" - [ "$status" -eq 0 ] || { - echo "Failed to connect to data node using HTTPS:" - echo "$output" - false - } - echo "$output" | grep "missing authentication credentials" -} - -@test "[$GROUP] test node to node communication" { - load $MASTER_UTILS - export ESHOME="$MASTER_HOME" - export_elasticsearch_paths - - testIndex=$(sudo curl -u "elastic:changeme" \ - -H "Content-Type: application/json" \ - --cacert "$ESCONFIG/certs/ca/ca.crt" \ - -XPOST "https://127.0.0.1:9200/books/book/0?refresh" \ - -d '{"title": "Elasticsearch The Definitive Guide"}') - - debug_collect_logs - echo "$testIndex" | grep '"result":"created"' - - masterSettings=$(sudo curl -u "elastic:changeme" \ - -H "Content-Type: application/json" \ - --cacert "$ESCONFIG/certs/ca/ca.crt" \ - -XGET "https://127.0.0.1:9200/_nodes/node-master?filter_path=nodes.*.settings.xpack,nodes.*.settings.http.type,nodes.*.settings.transport.type") - - echo "$masterSettings" | grep '"http":{"ssl":{"enabled":"true"}' - echo "$masterSettings" | grep '"http":{"type":"security4"}' - echo "$masterSettings" | grep '"transport":{"ssl":{"enabled":"true"}' - echo "$masterSettings" | grep '"transport":{"type":"security4"}' - - load $DATA_UTILS - export ESHOME="$DATA_HOME" - export_elasticsearch_paths - - dataSettings=$(curl -u "elastic:changeme" \ - -H "Content-Type: application/json" \ - --cacert "$ESCONFIG/certs/ca/ca.crt" \ - -XGET "https://127.0.0.1:9200/_nodes/node-data?filter_path=nodes.*.settings.xpack,nodes.*.settings.http.type,nodes.*.settings.transport.type") - - echo "$dataSettings" | grep '"http":{"ssl":{"enabled":"true"}' - echo "$dataSettings" | grep '"http":{"type":"security4"}' - echo "$dataSettings" | grep '"transport":{"ssl":{"enabled":"true"}' - echo "$dataSettings" | grep '"transport":{"type":"security4"}' - - testSearch=$(curl -u "elastic:changeme" \ - -H "Content-Type: application/json" \ - --cacert "$ESCONFIG/certs/ca/ca.crt" \ - -XGET "https://127.0.0.1:9200/_search?q=title:guide") - - echo "$testSearch" | grep '"_index":"books"' - echo "$testSearch" | grep '"_id":"0"' -} - -@test "[$GROUP] exit code on failure" { - run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --not-a-valid-option - [ "$status" -ne 0 ] || { - echo "Expected elasticsearch-certgen tool exit code to be non-zero" - echo "$output" - false - } -} - -@test "[$GROUP] remove Elasticsearch" { - # NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed - clean_before_test -} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java new file mode 100644 index 00000000000..1893e5f682f --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.FileUtils; +import org.elasticsearch.packaging.util.Platforms; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse; +import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; +import static org.elasticsearch.packaging.util.FileMatcher.file; +import static org.elasticsearch.packaging.util.FileMatcher.p600; +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.FileUtils.escapePath; +import static org.elasticsearch.packaging.util.FileUtils.getTempDir; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assume.assumeTrue; + +public class CertGenCliTests extends PackagingTestCase { + private static final Path instancesFile = getTempDir().resolve("instances.yml"); + private static final Path certificatesFile = getTempDir().resolve("certificates.zip"); + + @Before + public void filterDistros() { + assumeTrue("only default distro", distribution.flavor == Distribution.Flavor.DEFAULT); + assumeTrue("no docker", distribution.packaging != Distribution.Packaging.DOCKER); + } + + @BeforeClass + public static void cleanupFiles() { + FileUtils.rm(instancesFile, certificatesFile); + } + + public void test10Install() throws Exception { + install(); + } + + public void test20Help() throws Exception { + Shell.Result result = installation.executables().certgenTool.run("--help"); + assertThat(result.stdout, containsString("Simplifies certificate creation")); + } + + public void test30Generate() throws Exception { + Files.write(instancesFile, Arrays.asList( + "instances:", + " - name: \"mynode\"", + " ip:", + " - \"127.0.0.1\"")); + + installation.executables().certgenTool.run("--in " + instancesFile + " --out " + certificatesFile); + + String owner = installation.getOwner(); + assertThat(certificatesFile, file(File, owner, owner, p600)); + } + + public void test31ExtractCerts() throws Exception { + // windows 2012 r2 has powershell 4.0, which lacks Expand-Archive + assumeFalse(Platforms.OS_NAME.equals("Windows Server 2012 R2")); + + Path certsDir = installation.config("certs"); + sh.extractZip(certificatesFile, certsDir); + + Path caDir = certsDir.resolve("ca"); + assertThat(caDir.resolve("ca.key"), file(File, null, null, null)); + assertThat(caDir.resolve("ca.crt"), file(File, null, null, null)); + + Path nodeDir = certsDir.resolve("mynode"); + assertThat(nodeDir.resolve("mynode.key"), file(File, null, null, null)); + assertThat(nodeDir.resolve("mynode.crt"), file(File, null, null, null)); + + FileUtils.cp(certsDir, installation.config("certs")); + } + + public void test40RunWithCert() throws Exception { + // windows 2012 r2 has powershell 4.0, which lacks Expand-Archive + assumeFalse(Platforms.OS_NAME.equals("Windows Server 2012 R2")); + + append(installation.config("elasticsearch.yml"), String.join("\n", + "node.name: mynode", + "xpack.security.transport.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")), + "xpack.security.transport.ssl.certificate: " + escapePath(installation.config("certs/mynode/mynode.crt")), + "xpack.security.transport.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]", + "xpack.security.http.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")), + "xpack.security.http.ssl.certificate: "+ escapePath(installation.config("certs/mynode/mynode.crt")), + "xpack.security.http.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]", + "xpack.security.transport.ssl.enabled: true", + "xpack.security.http.ssl.enabled: true")); + + assertWhileRunning(() -> { + ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt")); + }); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index da3f55daae0..c66bb3cb7f3 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -175,6 +175,12 @@ public abstract class PackagingTestCase extends Assert { logger.warn("Elasticsearch log:\n" + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz")); } + if (Files.exists(installation.logs.resolve("output.out"))) { + logger.warn("Stdout:\n" + FileUtils.slurpTxtorGz(installation.logs.resolve("output.out"))); + } + if (Files.exists(installation.logs.resolve("output.err"))) { + logger.warn("Stderr:\n" + FileUtils.slurpTxtorGz(installation.logs.resolve("output.err"))); + } throw e; } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java index b08b3bfe529..edec71a12a0 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java @@ -62,7 +62,8 @@ public class PasswordToolsTests extends PackagingTestCase { Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null); Map userpasses = parseUsersAndPasswords(result.stdout); for (Map.Entry userpass : userpasses.entrySet()) { - String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue()); + String response = ServerUtils.makeRequest( + Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null); assertThat(response, containsString("You Know, for Search")); } }); @@ -111,7 +112,7 @@ public class PasswordToolsTests extends PackagingTestCase { assertWhileRunning(() -> { String response = ServerUtils.makeRequest( Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"), - "elastic", BOOTSTRAP_PASSWORD); + "elastic", BOOTSTRAP_PASSWORD, null); assertThat(response, containsString("\"status\":\"green\"")); }); } @@ -123,7 +124,8 @@ public class PasswordToolsTests extends PackagingTestCase { Map userpasses = parseUsersAndPasswords(result.stdout); assertThat(userpasses, hasKey("elastic")); for (Map.Entry userpass : userpasses.entrySet()) { - String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue()); + String response = ServerUtils.makeRequest( + Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null); assertThat(response, containsString("You Know, for Search")); } }); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java index 5f4fac5f535..d261a2627a6 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java @@ -160,7 +160,7 @@ public class WindowsServiceTests extends PackagingTestCase { } // NOTE: service description is not attainable through any powershell api, so checking it is not possible... - public void assertStartedAndStop() throws IOException { + public void assertStartedAndStop() throws Exception { ServerUtils.waitForElasticsearch(installation); ServerUtils.runElasticsearchTests(); @@ -191,7 +191,7 @@ public class WindowsServiceTests extends PackagingTestCase { "}"); } - public void test30StartStop() throws IOException { + public void test30StartStop() throws Exception { sh.run(serviceScript + " install"); assertCommand(serviceScript + " start"); assertStartedAndStop(); @@ -209,7 +209,7 @@ public class WindowsServiceTests extends PackagingTestCase { assertThat(result.stdout, containsString("The service '" + DEFAULT_ID + "' has been stopped")); } - public void test33JavaChanged() throws IOException { + public void test33JavaChanged() throws Exception { final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); try { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index bdb349d111f..8c32c7edc32 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -26,7 +26,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.http.client.fluent.Request; import org.elasticsearch.common.CheckedRunnable; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFileAttributes; @@ -503,7 +502,7 @@ public class Docker { } } - public static JsonNode getJson(String path) throws IOException { + public static JsonNode getJson(String path) throws Exception { final String pluginsResponse = makeRequest(Request.Get("http://localhost:9200/" + path)); ObjectMapper mapper = new ObjectMapper(); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java index 57c74a8c68e..90edb42829d 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java @@ -62,7 +62,7 @@ public class FileMatcher extends TypeSafeMatcher { public FileMatcher(Fileness fileness, String owner, String group, Set posixPermissions) { this.fileness = Objects.requireNonNull(fileness); - this.owner = Objects.requireNonNull(owner); + this.owner = owner; this.group = group; this.posixPermissions = posixPermissions; } @@ -76,16 +76,18 @@ public class FileMatcher extends TypeSafeMatcher { if (Platforms.WINDOWS) { final BasicFileAttributes attributes = getBasicFileAttributes(path); - final String attributeViewOwner = getFileOwner(path); if (fileness.equals(Fileness.Directory) != attributes.isDirectory()) { mismatch = "Is " + (attributes.isDirectory() ? "a directory" : "a file"); return false; } - if (attributeViewOwner.contains(owner) == false) { - mismatch = "Owned by " + attributeViewOwner; - return false; + if (owner != null) { + final String attributeViewOwner = getFileOwner(path); + if (attributeViewOwner.contains(owner) == false) { + mismatch = "Owned by " + attributeViewOwner; + return false; + } } } else { final PosixFileAttributes attributes = getPosixFileAttributes(path); @@ -95,7 +97,7 @@ public class FileMatcher extends TypeSafeMatcher { return false; } - if (owner.equals(attributes.owner().getName()) == false) { + if (owner != null && owner.equals(attributes.owner().getName()) == false) { mismatch = "Owned by " + attributes.owner().getName(); return false; } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java index 2efb757037c..eb57e66239e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java @@ -286,7 +286,7 @@ public class FileUtils { // vagrant creates /tmp for us in windows so we use that to avoid long paths public static Path getTempDir() { - return Paths.get("/tmp"); + return Paths.get("/tmp").toAbsolutePath(); } public static Path getDefaultArchiveInstallPath() { @@ -339,4 +339,15 @@ public class FileUtils { } } } + + /** + * Return the given path a string suitable for using on the host system. + */ + public static String escapePath(Path path) { + if (Platforms.WINDOWS) { + // replace single backslash with forward slash, to avoid unintended escapes in scripts + return path.toString().replace('\\', '/'); + } + return path.toString(); + } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index a237da3ea19..717f15aebc6 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -114,6 +114,19 @@ public class Installation { ); } + /** + * Returns the user that owns this installation. + * + * For packages this is root, and for archives it is the user doing the installation. + */ + public String getOwner() { + if (Platforms.WINDOWS) { + // windows is always administrator, since there is no sudo + return "BUILTIN\\Administrators"; + } + return distribution.isArchive() ? ARCHIVE_OWNER : "root"; + } + public Path bin(String executableName) { return bin.resolve(executableName); } @@ -147,7 +160,7 @@ public class Installation { public Shell.Result run(String args, String input) { String command = path + " " + args; - if (distribution.isArchive() && distribution.platform != Distribution.Platform.WINDOWS) { + if (distribution.isArchive() && Platforms.WINDOWS == false) { command = "sudo -E -u " + ARCHIVE_OWNER + " " + command; } if (input != null) { @@ -163,6 +176,7 @@ public class Installation { public final Executable pluginTool = new Executable("elasticsearch-plugin"); public final Executable keystoreTool = new Executable("elasticsearch-keystore"); public final Executable certutilTool = new Executable("elasticsearch-certutil"); + public final Executable certgenTool = new Executable("elasticsearch-certgen"); public final Executable shardTool = new Executable("elasticsearch-shard"); public final Executable nodeTool = new Executable("elasticsearch-node"); public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index 2886b19c3a2..e039d67ffcc 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -291,7 +291,7 @@ public class Packages { } } - public static void assertElasticsearchStarted(Shell sh, Installation installation) throws IOException { + public static void assertElasticsearchStarted(Shell sh, Installation installation) throws Exception { waitForElasticsearch(installation); if (isSystemd()) { @@ -310,7 +310,7 @@ public class Packages { } } - public static void restartElasticsearch(Shell sh, Installation installation) throws IOException { + public static void restartElasticsearch(Shell sh, Installation installation) throws Exception { if (isSystemd()) { sh.run("systemctl restart elasticsearch.service"); } else { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 34a46b908d5..bcaef380076 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -23,17 +23,32 @@ import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.fluent.Executor; import org.apache.http.client.fluent.Request; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; import java.net.Socket; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -46,25 +61,26 @@ public class ServerUtils { private static final Logger logger = LogManager.getLogger(ServerUtils.class); private static String SECURITY_ENABLED = "xpack.security.enabled: true"; + private static String SSL_ENABLED = "xpack.security.http.ssl.enabled: true"; // generous timeout as nested virtualization can be quite slow ... private static final long waitTime = TimeUnit.MINUTES.toMillis(3); private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30); private static final long requestInterval = TimeUnit.SECONDS.toMillis(5); - public static void waitForElasticsearch(Installation installation) throws IOException { - boolean securityEnabled = false; + public static void waitForElasticsearch(Installation installation) throws Exception { + boolean xpackEnabled = false; // TODO: need a way to check if docker has security enabled, the yml config is not bind mounted so can't look from here if (installation.distribution.packaging != Distribution.Packaging.DOCKER) { Path configFilePath = installation.config("elasticsearch.yml"); // this is fragile, but currently doesn't deviate from a single line enablement and not worth the parsing effort try (Stream lines = Files.lines(configFilePath, StandardCharsets.UTF_8)) { - securityEnabled = lines.anyMatch(line -> line.contains(SECURITY_ENABLED)); + xpackEnabled = lines.anyMatch(line -> line.contains(SECURITY_ENABLED) || line.contains(SSL_ENABLED)); } } - if (securityEnabled) { + if (xpackEnabled) { // with security enabled, we may or may not have setup a user/pass, so we use a more generic port being available check. // this isn't as good as a health check, but long term all this waiting should go away when node startup does not // make the http port available until the system is really ready to serve requests @@ -80,17 +96,46 @@ public class ServerUtils { * @param request the request to execute * @param username the username to supply, or null * @param password the password to supply, or null + * @param caCert path to the ca certificate the server side ssl cert was generated from, or no if not using ssl * @return the response from the server * @throws IOException if an error occurs */ - private static HttpResponse execute(Request request, String username, String password) throws IOException { - final Executor executor = Executor.newInstance(); + private static HttpResponse execute(Request request, String username, String password, Path caCert) throws Exception { + final Executor executor; + if (caCert != null) { + try (InputStream inStream = Files.newInputStream(caCert)) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(inStream); + KeyStore truststore = KeyStore.getInstance(KeyStore.getDefaultType()); + truststore.load(null, null); + truststore.setCertificateEntry("myClusterCA", cert); + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(truststore, null); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(truststore); + SSLContext context = SSLContext.getInstance("TLSv1.2"); + context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null); + final LayeredConnectionSocketFactory ssl = new SSLConnectionSocketFactory(context); + final Registry sfr = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .register("https", ssl) + .build(); + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(sfr); + connectionManager.setDefaultMaxPerRoute(100); + connectionManager.setMaxTotal(200); + connectionManager.setValidateAfterInactivity(1000); + executor = Executor.newInstance(HttpClientBuilder.create() + .setConnectionManager(connectionManager) + .build()); + } + } else { + executor = Executor.newInstance(); + } if (username != null && password != null) { executor.auth(username, password); executor.authPreemptive(new HttpHost("localhost", 9200)); } - return executor.execute(request).returnResponse(); } @@ -120,7 +165,7 @@ public class ServerUtils { Installation installation, String username, String password - ) throws IOException { + ) throws Exception { Objects.requireNonNull(status); @@ -131,6 +176,11 @@ public class ServerUtils { boolean started = false; Throwable thrownException = null; + Path caCert = installation.config("certs/ca/ca.crt"); + if (Files.exists(caCert) == false) { + caCert = null; // no cert, so don't use ssl + } + while (started == false && timeElapsed < waitTime) { if (System.currentTimeMillis() - lastRequest > requestInterval) { try { @@ -141,7 +191,8 @@ public class ServerUtils { .connectTimeout((int) timeoutLength) .socketTimeout((int) timeoutLength), username, - password + password, + caCert ); if (response.getStatusLine().getStatusCode() >= 300) { @@ -181,11 +232,11 @@ public class ServerUtils { url = "http://localhost:9200/_cluster/health/" + index + "?wait_for_status=" + status + "&timeout=60s&pretty"; } - final String body = makeRequest(Request.Get(url), username, password); + final String body = makeRequest(Request.Get(url), username, password, caCert); assertThat("cluster health response must contain desired status", body, containsString(status)); } - public static void runElasticsearchTests() throws IOException { + public static void runElasticsearchTests() throws Exception { makeRequest( Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty") .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)); @@ -200,12 +251,12 @@ public class ServerUtils { makeRequest(Request.Delete("http://localhost:9200/_all")); } - public static String makeRequest(Request request) throws IOException { - return makeRequest(request, null, null); + public static String makeRequest(Request request) throws Exception { + return makeRequest(request, null, null, null); } - public static String makeRequest(Request request, String username, String password) throws IOException { - final HttpResponse response = execute(request, username, password); + public static String makeRequest(Request request, String username, String password, Path caCert) throws Exception { + final HttpResponse response = execute(request, username, password, caCert); final String body = EntityUtils.toString(response.getEntity()); if (response.getStatusLine().getStatusCode() >= 300) { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java index e0fae3d0955..18988a7310f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java @@ -98,7 +98,12 @@ public class Shell { )); } - public Result run( String command, Object... args) { + public void extractZip(Path zipPath, Path destinationDir) throws Exception { + Platforms.onLinux(() -> run("unzip \"" + zipPath + "\" -d \"" + destinationDir + "\"")); + Platforms.onWindows(() -> run("Expand-Archive -Path \"" + zipPath + "\" -DestinationPath \"" + destinationDir + "\"")); + } + + public Result run(String command, Object... args) { String formattedCommand = String.format(Locale.ROOT, command, args); return run(formattedCommand); }