diff --git a/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 37cc104106c..06a2a624d5e 100644
--- a/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -273,7 +273,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
         final PluginsAndModules mockPluginsAndModules = mock(PluginsAndModules.class);
         when(mockNodeInfo.getPlugins()).thenReturn(mockPluginsAndModules);
         final PluginInfo pluginInfo = new PluginInfo("_plugin", "_plugin_desc", "_plugin_version", Version.CURRENT,
-            "1.8", "_plugin_class", Collections.emptyList(), false, false);
+            "1.8", "_plugin_class", Collections.emptyList(), false);
         when(mockPluginsAndModules.getPluginInfos()).thenReturn(singletonList(pluginInfo));
 
         final OsInfo mockOsInfo = mock(OsInfo.class);
diff --git a/qa/vagrant/src/test/resources/packaging/tests/40_tar_bootstrap_password.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/40_tar_bootstrap_password.bats
rename to qa/vagrant/src/test/resources/packaging/tests/20_tar_bootstrap_password.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/20_tar_keystore.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_keystore.bats
deleted file mode 120000
index a3c545a63d0..00000000000
--- a/qa/vagrant/src/test/resources/packaging/tests/20_tar_keystore.bats
+++ /dev/null
@@ -1 +0,0 @@
-keystore.bash
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/tests/45_package_bootstrap_password.bats b/qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/45_package_bootstrap_password.bats
rename to qa/vagrant/src/test/resources/packaging/tests/25_package_bootstrap_password.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/30_package_keystore.bats b/qa/vagrant/src/test/resources/packaging/tests/30_package_keystore.bats
deleted file mode 120000
index a3c545a63d0..00000000000
--- a/qa/vagrant/src/test/resources/packaging/tests/30_package_keystore.bats
+++ /dev/null
@@ -1 +0,0 @@
-keystore.bash
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/tests/50_tar_setup_passwords.bats b/qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/50_tar_setup_passwords.bats
rename to qa/vagrant/src/test/resources/packaging/tests/30_tar_setup_passwords.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/55_package_setup_passwords.bats b/qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/55_package_setup_passwords.bats
rename to qa/vagrant/src/test/resources/packaging/tests/35_package_setup_passwords.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_tar_certgen.bats b/qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/60_tar_certgen.bats
rename to qa/vagrant/src/test/resources/packaging/tests/40_tar_certgen.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/65_package_certgen.bats b/qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats
similarity index 100%
rename from qa/vagrant/src/test/resources/packaging/tests/65_package_certgen.bats
rename to qa/vagrant/src/test/resources/packaging/tests/45_package_certgen.bats
diff --git a/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash b/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash
index b5b5d6b91ff..d5af3d7c5da 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash
+++ b/qa/vagrant/src/test/resources/packaging/tests/bootstrap_password.bash
@@ -19,7 +19,7 @@ setup() {
     fi
 }
 
-if [[ "$BATS_TEST_FILENAME" =~ 40_tar_bootstrap_password.bats$ ]]; then
+if [[ "$BATS_TEST_FILENAME" =~ 20_tar_bootstrap_password.bats$ ]]; then
     load $BATS_UTILS/tar.bash
     GROUP='TAR BOOTSTRAP PASSWORD'
     install() {
@@ -50,6 +50,9 @@ fi
     fi
 
     run sudo -E -u $ESPLUGIN_COMMAND_USER sh <<"NEW_PASS"
+if [[ ! -f $ESCONFIG/elasticsearch.keystore ]]; then
+    $ESHOME/bin/elasticsearch-keystore create
+fi
 cat /dev/urandom | tr -dc "[a-zA-Z0-9]" | fold -w 20 | head -n 1 > /tmp/bootstrap.password
 cat /tmp/bootstrap.password | $ESHOME/bin/elasticsearch-keystore add --stdin bootstrap.password
 NEW_PASS
diff --git a/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/qa/vagrant/src/test/resources/packaging/tests/certgen.bash
index 12d2797b9cb..2848b43a949 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/certgen.bash
+++ b/qa/vagrant/src/test/resources/packaging/tests/certgen.bash
@@ -28,7 +28,7 @@ DEFAULT_PACKAGE_USER=root
 DEFAULT_PACKAGE_ESHOME="/usr/share/elasticsearch"
 DEFAULT_PACKAGE_UTILS=$BATS_UTILS/packages.bash
 
-if [[ "$BATS_TEST_FILENAME" =~ 60_tar_certgen.bats$ ]]; then
+if [[ "$BATS_TEST_FILENAME" =~ 40_tar_certgen.bats$ ]]; then
     GROUP='TAR CERTGEN'
 
     MASTER_USER=$DEFAULT_ARCHIVE_USER
@@ -153,6 +153,9 @@ start_node_using_package() {
     # and restart the node once ssl/tls is configured. Or use setup-passwords over
     # HTTPS with the right cacerts imported into a Java keystore.
     run sudo -E -u $MASTER_USER sh <<"NEW_PASS"
+if [[ ! -f $ESCONFIG/elasticsearch.keystore ]]; then
+    $ESHOME/bin/elasticsearch-keystore create
+fi
 echo "changeme" | $ESHOME/bin/elasticsearch-keystore add --stdin bootstrap.password
 NEW_PASS
     [ "$status" -eq 0 ] || {
diff --git a/qa/vagrant/src/test/resources/packaging/tests/keystore.bash b/qa/vagrant/src/test/resources/packaging/tests/keystore.bash
deleted file mode 100644
index a5f855f2f55..00000000000
--- a/qa/vagrant/src/test/resources/packaging/tests/keystore.bash
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-
-load $BATS_UTILS/utils.bash
-load $BATS_UTILS/plugins.bash
-load $BATS_UTILS/xpack.bash
-
-setup() {
-    if [ $BATS_TEST_NUMBER == 1 ]; then
-        clean_before_test
-        install
-    fi
-}
-
-if [[ "$BATS_TEST_FILENAME" =~ 20_tar_keystore.bats$ ]]; then
-    load $BATS_UTILS/tar.bash
-    GROUP='TAR KEYSTORE'
-    install() {
-        install_archive
-        verify_archive_installation
-    }
-    export ESHOME=/tmp/elasticsearch
-    export_elasticsearch_paths
-    export ESPLUGIN_COMMAND_USER=elasticsearch
-else
-    load $BATS_UTILS/packages.bash
-    if is_rpm; then
-        GROUP='RPM KEYSTORE'
-    elif is_dpkg; then
-        GROUP='DEB KEYSTORE'
-    fi
-    export_elasticsearch_paths
-    export ESPLUGIN_COMMAND_USER=root
-    install() {
-        install_package
-        verify_package_installation
-    }
-fi
-
-@test "[$GROUP] keystore does not exist" {
-    assert_file_not_exist /etc/elasticsearch/elasticsearch.keystore
-}
-
-@test "[$GROUP] keystore exists after install" {
-    install_xpack
-    verify_xpack_installation
-}
diff --git a/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash b/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash
index 050b8c2f225..aa71d93f6e3 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash
+++ b/qa/vagrant/src/test/resources/packaging/tests/setup_passwords.bash
@@ -20,7 +20,7 @@ setup() {
 }
 
 
-if [[ "$BATS_TEST_FILENAME" =~ 50_tar_setup_passwords.bats$ ]]; then
+if [[ "$BATS_TEST_FILENAME" =~ 30_tar_setup_passwords.bats$ ]]; then
     load $BATS_UTILS/tar.bash
     GROUP='TAR SETUP PASSWORD'
     install() {
diff --git a/qa/vagrant/src/test/resources/packaging/utils/xpack.bash b/qa/vagrant/src/test/resources/packaging/utils/xpack.bash
index 6d4839fe9d2..73c9959ee1a 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/xpack.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/xpack.bash
@@ -69,9 +69,6 @@ verify_xpack_installation() {
     done
     assert_number_of_files "$ESCONFIG/$name/" $configFilesCount
 
-    # Verify keystore creation
-    assert_file "$ESCONFIG/elasticsearch.keystore" f $user elasticsearch 660
-
     # Read the $name.expected file that contains all the expected
     # plugins for the meta plugin
     while read plugin; do