diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java index 7e6db57be1e..35b4b8b292a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java @@ -129,11 +129,9 @@ public class HttpExporter extends Exporter { public static final String PIPELINE_CHECK_TIMEOUT_SETTING = "index.pipeline.master_timeout"; /** - * Minimum supported version of the remote monitoring cluster. - *

- * We must have support for the latest template syntax (index_patterns), which requires a minimum of 6.0. + * Minimum supported version of the remote monitoring cluster (same major). */ - public static final Version MIN_SUPPORTED_CLUSTER_VERSION = Version.V_6_0_0_alpha1; + public static final Version MIN_SUPPORTED_CLUSTER_VERSION = Version.V_7_0_0_alpha1; /** * The {@link RestClient} automatically pools connections and keeps them alive as necessary. diff --git a/plugin/src/test/java/org/elasticsearch/AbstractOldXPackIndicesBackwardsCompatibilityTestCase.java b/plugin/src/test/java/org/elasticsearch/AbstractOldXPackIndicesBackwardsCompatibilityTestCase.java deleted file mode 100644 index 4f7cc043422..00000000000 --- a/plugin/src/test/java/org/elasticsearch/AbstractOldXPackIndicesBackwardsCompatibilityTestCase.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch; - -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.TestUtil; -import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.license.PutLicenseAction; -import org.elasticsearch.license.PutLicenseRequest; -import org.elasticsearch.license.PutLicenseResponse; -import org.elasticsearch.license.TestUtils; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.VersionUtils; -import org.junit.Before; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -import static org.elasticsearch.test.OldIndexUtils.copyIndex; -import static org.elasticsearch.test.OldIndexUtils.loadDataFilesList; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; - -/** - * Base class for tests against clusters coming from old versions of xpack and Elasticsearch. - */ -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) // We'll start the nodes manually -public abstract class AbstractOldXPackIndicesBackwardsCompatibilityTestCase extends SecurityIntegTestCase { - - /** - * Set to true when it is ok to start a node. We don't want to start nodes at unexpected times. - */ - private boolean okToStartNode = false; - private List dataFiles; - - @Override - protected final boolean ignoreExternalCluster() { - return true; - } - - @Override - protected boolean shouldAssertXPackIsInstalled() { - return false; // Skip asserting that the xpack is installed because it tries to start the cluter. - } - - @Override - protected void ensureClusterSizeConsistency() { - // We manage our nodes ourselves. At this point no node should be running anyway and this would start a new one! - } - - @Override - protected void ensureClusterStateConsistency() throws IOException { - // We manage our nodes ourselves. At this point no node should be running anyway and this would start a new one! - } - - @Before - public final void initIndexesList() throws Exception { - dataFiles = loadDataFilesList("x-pack", getBwcIndicesPath()); - } - - @Override - public Settings nodeSettings(int ord) { - if (false == okToStartNode) { - throw new IllegalStateException("Starting nodes must only happen in setupCluster"); - } - // speed up recoveries - return Settings.builder() - .put(super.nodeSettings(ord)) - .put(ThrottlingAllocationDecider - .CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 30) - .put(ThrottlingAllocationDecider - .CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 30) - .build(); - } - - @Override - protected int maxNumberOfNodes() { - try { - return SecurityIntegTestCase.defaultMaxNumberOfNodes() + loadDataFilesList("x-pack", getBwcIndicesPath()).size(); - } catch (IOException e) { - throw new RuntimeException("couldn't enumerate bwc indices", e); - } - } - - public void testAllVersionsTested() throws Exception { - SortedSet expectedVersions = new TreeSet<>(); - for (Version v : VersionUtils.allReleasedVersions()) { - if (v.isRelease()) { - // no guarantees for prereleases - expectedVersions.add("x-pack-" + v.toString() + ".zip"); - } - } - expectedVersions.removeAll(dataFiles); - if (expectedVersions.isEmpty() == false) { - StringBuilder msg = new StringBuilder("Old index tests are missing indexes:"); - for (String expected : expectedVersions) { - msg.append("\n" + expected); - } - fail(msg.toString()); - } - } - - public void testOldIndexes() throws Exception { - assertSecurityIndexWriteable(); - Collections.shuffle(dataFiles, random()); - for (String dataFile : dataFiles) { - Version version = Version.fromString(dataFile.replace("x-pack-", "").replace(".zip", "")); - long clusterStartTime = System.nanoTime(); - setupCluster(dataFile); - ensureYellow(); - long testStartTime = System.nanoTime(); - try { - checkVersion(version); - } catch (Throwable t) { - throw new AssertionError("Failed while checking [" + version + "]", t); - } - logger.info("--> Done testing [{}]. Setting up cluster took [{}] millis and testing took [{}] millis", version, - TimeUnit.NANOSECONDS.toMillis(testStartTime - clusterStartTime), - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - testStartTime)); - } - } - - /** - * Actually test this version. - */ - protected abstract void checkVersion(Version version) throws Exception; - - private void setupCluster(String pathToZipFile) throws Exception { - // shutdown any nodes from previous zip files - while (internalCluster().size() > 0) { - internalCluster().stopRandomNode(s -> true); - } - // first create the data directory and unzip the data there - // we put the whole cluster state and indexes because if we only copy indexes and import them as dangling then - // the native realm services will start because there is no security index and nothing is recovering - // but we want them to not start! - Path dataPath = createTempDir(); - Settings.Builder nodeSettings = Settings.builder() - .put("path.data", dataPath.toAbsolutePath()); - // unzip data - Path backwardsIndex = getBwcIndicesPath().resolve(pathToZipFile); - // decompress the index - try (InputStream stream = Files.newInputStream(backwardsIndex)) { - logger.info("unzipping {}", backwardsIndex.toString()); - TestUtil.unzip(stream, dataPath); - // now we need to copy the whole thing so that it looks like an actual data path - try (Stream unzippedFiles = Files.list(dataPath.resolve("data"))) { - Path dataDir = unzippedFiles.findFirst().get(); - // this is not actually an index but the copy does the job anyway - int zipIndex = pathToZipFile.indexOf(".zip"); - Version version = Version.fromString(pathToZipFile.substring("x-pack-".length(), zipIndex)); - if (version.before(Version.V_5_0_0_alpha1)) { - // the bwc scripts packs the indices under this path before 5.0 - dataDir = dataDir.resolve("nodes"); - } - copyIndex(logger, dataDir, "nodes", dataPath); - // remove the original unzipped directory - } - IOUtils.rm(dataPath.resolve("data")); - } - - // check it is unique - assertTrue(Files.exists(dataPath)); - Path[] list = FileSystemUtils.files(dataPath); - if (list.length != 1) { - throw new IllegalStateException("Backwards index must contain exactly one node"); - } - - // start the node - logger.info("--> Data path for importing node: {}", dataPath); - okToStartNode = true; - String importingNodeName = internalCluster().startNode(nodeSettings.build()); - okToStartNode = false; - Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, importingNodeName).nodeDataPaths(); - assertEquals(1, nodePaths.length); - PutLicenseResponse putLicenseResponse = client().execute(PutLicenseAction.INSTANCE, - new PutLicenseRequest().license(TestUtils.generateSignedLicense("platinum", TimeValue.timeValueHours(24L)))).get(); - assertAcked(putLicenseResponse); - } -} diff --git a/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 1994add267a..3b63a35d2fa 100644 --- a/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.test; -import org.elasticsearch.AbstractOldXPackIndicesBackwardsCompatibilityTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -183,9 +182,6 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase { @Before //before methods from the superclass are run before this, which means that the current cluster is ready to go public void assertXPackIsInstalled() { - if (false == shouldAssertXPackIsInstalled()) { - return; - } NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).get(); for (NodeInfo nodeInfo : nodeInfos.getNodes()) { // TODO: disable this assertion for now, due to random runs with mock plugins. perhaps run without mock plugins? @@ -197,17 +193,6 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase { } } - /** - * Should this test assert that x-pack is installed? You might want to skip this assertion if the test itself validates the installation - * and running the assertion would significantly affect the performance or function of the test. For example - * {@link AbstractOldXPackIndicesBackwardsCompatibilityTestCase} disables the assertion because the assertion would force starting a - * node which it will then just shut down. That would slow the test down significantly and causes spurious failures because watcher - * needs to be shut down with kid gloves. - */ - protected boolean shouldAssertXPackIsInstalled() { - return true; - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/OldMonitoringIndicesBackwardsCompatibilityTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/OldMonitoringIndicesBackwardsCompatibilityTests.java deleted file mode 100644 index a284c8f1d8a..00000000000 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/OldMonitoringIndicesBackwardsCompatibilityTests.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.monitoring; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.AbstractOldXPackIndicesBackwardsCompatibilityTestCase; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.xpack.XPackSettings; -import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc; -import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc; -import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc; -import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc; -import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc; -import org.elasticsearch.xpack.monitoring.exporter.Exporters; -import org.hamcrest.Matcher; -import org.joda.time.format.DateTimeFormat; - -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.is; - -/** - * Tests for monitoring indexes created before {@link Version#CURRENT}. - */ -//Give ourselves 30 seconds instead of 5 to shut down. Sometimes it takes a while, especially on weak hardware. But we do get there. -@ThreadLeakLingering(linger = 30000) -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/4314") -public class OldMonitoringIndicesBackwardsCompatibilityTests extends AbstractOldXPackIndicesBackwardsCompatibilityTestCase { - - private final boolean httpExporter = randomBoolean(); - - @Override - public Settings nodeSettings(int ord) { - Settings.Builder settings = Settings.builder().put(super.nodeSettings(ord)) - .put(XPackSettings.MONITORING_ENABLED.getKey(), true) - // Don't clean old monitoring indexes - we want to make sure we can load them - .put(Monitoring.HISTORY_DURATION.getKey(), TimeValue.timeValueHours(1000 * 365 * 24).getStringRep()) - // Do not start monitoring exporters at startup - .put(MonitoringService.INTERVAL.getKey(), "-1"); - - if (httpExporter) { - /* If we want to test the http exporter we have to create it but disable it. We need to create it so we don't use the default - * local exporter and we have to disable it because we don't yet know the port we've bound to. We can only get that once - * Elasticsearch starts so we'll enable the exporter then. */ - settings.put(NetworkModule.HTTP_ENABLED.getKey(), true); - setupHttpExporter(settings, null); - } - return settings.build(); - } - - private void setupHttpExporter(Settings.Builder settings, Integer port) { - Map httpExporter = new HashMap<>(); - httpExporter.put("type", "http"); - httpExporter.put("enabled", port == null ? "false" : "true"); - httpExporter.put("host", "http://localhost:" + (port == null ? "does_not_matter" : port)); - httpExporter.put("auth.username", SecuritySettingsSource.TEST_USER_NAME); - httpExporter.put("auth.password", SecuritySettingsSource.TEST_PASSWORD); - - settings.putProperties(httpExporter, k -> Exporters.EXPORTERS_SETTINGS.getKey() + "my_exporter." + k); - } - - @Override - protected void checkVersion(Version version) throws Exception { - try { - logger.info("--> Start testing version [{}]", version); - if (httpExporter) { - // If we're using the http exporter we need to update the port and enable it - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); - TransportAddress publishAddress = nodeInfos.getNodes().get(0).getHttp().address().publishAddress(); - InetSocketAddress address = publishAddress.address(); - Settings.Builder settings = Settings.builder(); - setupHttpExporter(settings, address.getPort()); - - logger.info("--> Enabling http exporter pointing to [localhost:{}]", address.getPort()); - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get()); - } - - // Monitoring can now start to collect new data - Settings.Builder settings = Settings.builder().put(MonitoringService.INTERVAL.getKey(), timeValueSeconds(3).getStringRep()); - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get()); - - final String prefix = ".monitoring-" + MonitoredSystem.ES.getSystem() + "-" + TEMPLATE_VERSION + "-"; - - // And we wait until data have been indexed locally using either by the local or http exporter - final String dateTime = DateTimeFormat.forPattern("YYYY.MM.dd").print(System.currentTimeMillis()); - final String expectedIndex = prefix + dateTime; - final String indexPattern = prefix + "*"; - - logger.info("--> {} Waiting for [{}] to be ready", Thread.currentThread().getName(), expectedIndex); - assertBusy(() -> { - assertTrue(client().admin().indices().prepareExists(expectedIndex).get().isExists()); - - NumShards numShards = getNumShards(expectedIndex); - ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth(expectedIndex) - .setWaitForActiveShards(numShards.numPrimaries) - .get(); - assertThat(clusterHealth.getIndices().get(expectedIndex).getActivePrimaryShards(), equalTo(numShards.numPrimaries)); - }); - - SearchResponse firstIndexStats = - search(indexPattern, IndexStatsMonitoringDoc.TYPE, greaterThanOrEqualTo(10L)); - - // All the other aliases should have been created by now so we can assert that we have the data we saved in the bwc indexes - SearchResponse firstShards = search(indexPattern, ShardMonitoringDoc.TYPE, greaterThanOrEqualTo(10L)); - SearchResponse firstIndices = search(indexPattern, IndicesStatsMonitoringDoc.TYPE, greaterThanOrEqualTo(3L)); - SearchResponse firstNode = search(indexPattern, NodeStatsMonitoringDoc.TYPE, greaterThanOrEqualTo(3L)); - SearchResponse firstState = search(indexPattern, ClusterStatsMonitoringDoc.TYPE, greaterThanOrEqualTo(3L)); - - ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().clear().setNodes(true).get(); - final String masterNodeId = clusterStateResponse.getState().getNodes().getMasterNodeId(); - - // Verify some stuff about the stuff in the backwards compatibility indexes - Arrays.stream(firstIndexStats.getHits().getHits()).forEach(hit -> checkIndexStats(version, hit.getSourceAsMap())); - Arrays.stream(firstShards.getHits().getHits()).forEach(hit -> checkShards(version, hit.getSourceAsMap())); - Arrays.stream(firstIndices.getHits().getHits()).forEach(hit -> checkIndicesStats(version, hit.getSourceAsMap())); - Arrays.stream(firstNode.getHits().getHits()).forEach(hit -> checkNodeStats(version, masterNodeId, hit.getSourceAsMap())); - Arrays.stream(firstState.getHits().getHits()).forEach(hit -> checkClusterState(version, hit.getSourceAsMap())); - - // Create some docs - indexRandom(true, client().prepareIndex("test-1", "doc", "1").setSource("field", 1), - client().prepareIndex("test-2", "doc", "2").setSource("field", 2)); - - // Wait for monitoring to accumulate some data about the current cluster - long indexStatsCount = firstIndexStats.getHits().getTotalHits(); - assertBusy(() -> search(indexPattern, IndexStatsMonitoringDoc.TYPE, - greaterThan(indexStatsCount)), 1, TimeUnit.MINUTES); - assertBusy(() -> search(indexPattern, ShardMonitoringDoc.TYPE, - greaterThan(firstShards.getHits().getTotalHits())), 1, TimeUnit.MINUTES); - assertBusy(() -> search(indexPattern, IndicesStatsMonitoringDoc.TYPE, - greaterThan(firstIndices.getHits().getTotalHits())), 1, TimeUnit.MINUTES); - assertBusy(() -> search(indexPattern, NodeStatsMonitoringDoc.TYPE, - greaterThan(firstNode.getHits().getTotalHits())), 1, TimeUnit.MINUTES); - assertBusy(() -> search(indexPattern, ClusterStatsMonitoringDoc.TYPE, - greaterThan(firstState.getHits().getTotalHits())), 1, TimeUnit.MINUTES); - - } finally { - /* Now we stop monitoring and disable the HTTP exporter. We also delete all data and checks multiple times - if they have not been re created by some in flight monitoring bulk request */ - internalCluster().getInstances(MonitoringService.class).forEach(MonitoringService::stop); - - Settings.Builder settings = Settings.builder().put(MonitoringService.INTERVAL.getKey(), "-1"); - if (httpExporter) { - logger.info("--> Disabling http exporter after test"); - setupHttpExporter(settings, null); - } - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get()); - - logger.info("--> Waiting for indices deletion"); - CountDown retries = new CountDown(10); - assertBusy(() -> { - String[] indices = new String[]{".monitoring-*"}; - IndicesExistsResponse existsResponse = client().admin().indices().prepareExists(indices).get(); - if (existsResponse.isExists()) { - assertAcked(client().admin().indices().prepareDelete(indices)); - } else { - retries.countDown(); - } - assertThat(retries.isCountedDown(), is(true)); - }); - logger.info("--> End testing version [{}]", version); - } - } - - private SearchResponse search(String indexPattern, String type, Matcher hitCount) { - SearchResponse response = client().prepareSearch(indexPattern).setTypes(type).get(); - assertThat(response.getHits().getTotalHits(), hitCount); - return response; - } - - private void checkIndexStats(final Version version, Map indexStats) { - checkMonitoringElement(indexStats); - checkSourceNode(version, indexStats); - Map stats = (Map) indexStats.get("index_stats"); - assertThat(stats, hasKey("index")); - Map total = (Map) stats.get("total"); - Map docs = (Map) total.get("docs"); - // These might have been taken before all the documents were added so we can't assert a whole lot about the number - assertThat((Integer) docs.get("count"), greaterThanOrEqualTo(0)); - } - - private void checkShards(final Version version, Map shards) { - checkMonitoringElement(shards); - Map shard = (Map) shards.get("shard"); - assertThat(shard, allOf(hasKey("index"), hasKey("state"), hasKey("primary"), hasKey("node"))); - } - - private void checkIndicesStats(final Version version, Map indicesStats) { - checkMonitoringElement(indicesStats); - checkSourceNode(version, indicesStats); - Map stats = (Map) indicesStats.get("indices_stats"); - Map all = (Map) stats.get("_all"); - Map primaries = (Map) all.get("primaries"); - Map docs = (Map) primaries.get("docs"); - // These might have been taken before all the documents were added so we can't assert a whole lot about the number - assertThat((Integer) docs.get("count"), greaterThanOrEqualTo(0)); - } - - private void checkNodeStats(final Version version, final String masterNodeId, Map nodeStats) { - checkMonitoringElement(nodeStats); - checkSourceNode(version, nodeStats); - Map stats = (Map) nodeStats.get("node_stats"); - - // Those fields are expected in every node stats documents - Set mandatoryKeys = new HashSet<>(); - mandatoryKeys.add("node_id"); - mandatoryKeys.add("node_master"); - mandatoryKeys.add("mlockall"); - mandatoryKeys.add("indices"); - mandatoryKeys.add("os"); - mandatoryKeys.add("fs"); - mandatoryKeys.add("process"); - mandatoryKeys.add("jvm"); - mandatoryKeys.add("thread_pool"); - - // disk_threshold_* fields have been removed in 5.0 alpha5, we only check for them if the - // current tested version is less than or equal to alpha4. Also, the current master node - // might have collected its own node stats through the Monitoring plugin, and since it is - // running under Version.CURRENT there's no chance to find these fields. - if (version.onOrBefore(Version.V_5_0_0_alpha4)) { - if (masterNodeId.equals((String) stats.get("node_id")) == false) { - mandatoryKeys.add("disk_threshold_enabled"); - mandatoryKeys.add("disk_threshold_watermark_high"); - } - } - - for (String key : mandatoryKeys) { - assertThat("Expecting [" + key + "] to be present for bwc index in version [" + version + "]", stats, hasKey(key)); - } - - Set keys = new HashSet<>(stats.keySet()); - keys.removeAll(mandatoryKeys); - assertTrue("Found unexpected fields [" + Strings.collectionToCommaDelimitedString(keys) + "] " + - "for bwc index in version [" + version + "]", keys.isEmpty()); - } - - private void checkClusterState(final Version version, Map clusterState) { - checkMonitoringElement(clusterState); - checkSourceNode(version, clusterState); - Map stats = (Map) clusterState.get("cluster_state"); - assertThat(stats, allOf(hasKey("status"), hasKey("version"), hasKey("state_uuid"), hasKey("master_node"), hasKey("nodes"))); - } - - private void checkMonitoringElement(Map element) { - assertThat(element, allOf(hasKey("cluster_uuid"), hasKey("timestamp"))); - } - - private void checkSourceNode(final Version version, Map element) { - assertThat(element, hasKey("source_node")); - } -} diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 342fe0ae167..6401a48aa3a 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.common.text.TextTemplate; +import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils; import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase; import org.elasticsearch.xpack.security.support.IndexLifecycleManager; import org.elasticsearch.xpack.test.rest.XPackRestTestCase; @@ -175,6 +176,22 @@ public class FullClusterRestartIT extends ESRestTestCase { } } + @SuppressWarnings("unchecked") + public void testMonitoring() throws Exception { + waitForYellow(".monitoring-es-*"); + + if (runningAgainstOldCluster == false) { + waitForMonitoringTemplates(); + } + + // ensure that monitoring [re]starts and creates the core monitoring document, cluster_stats, for the current cluster + final Map response = toMap(client().performRequest("GET", "/")); + final Map version = (Map) response.get("version"); + final String expectedVersion = (String) version.get("number"); + + waitForClusterStats(expectedVersion); + } + public void testWatcher() throws Exception { if (runningAgainstOldCluster) { logger.info("Adding a watch on old cluster"); @@ -364,6 +381,43 @@ public class FullClusterRestartIT extends ESRestTestCase { }, 30, TimeUnit.SECONDS); } + @SuppressWarnings("unchecked") + private void waitForMonitoringTemplates() throws Exception { + assertBusy(() -> { + final Map templates = toMap(client().performRequest("GET", "/_template/.monitoring-*")); + + // in earlier versions, we published legacy templates in addition to the current ones to support transitioning + assertThat(templates.size(), greaterThanOrEqualTo(MonitoringTemplateUtils.TEMPLATE_IDS.length)); + + // every template should be updated to whatever the current version is + for (final String templateId : MonitoringTemplateUtils.TEMPLATE_IDS) { + final String templateName = MonitoringTemplateUtils.templateName(templateId); + final Map template = (Map) templates.get(templateName); + + assertThat(template.get("version"), is(MonitoringTemplateUtils.LAST_UPDATED_VERSION)); + } + }, 30, TimeUnit.SECONDS); + } + + @SuppressWarnings("unchecked") + private void waitForClusterStats(final String expectedVersion) throws Exception { + assertBusy(() -> { + final Map params = new HashMap<>(3); + params.put("q", "type:cluster_stats"); + params.put("size", "1"); + params.put("sort", "timestamp:desc"); + + final Map response = toMap(client().performRequest("GET", "/.monitoring-es-*/_search", params)); + final Map hits = (Map) response.get("hits"); + + assertThat("No cluster_stats documents found.", (int)hits.get("total"), greaterThanOrEqualTo(1)); + + final Map hit = (Map) ((List) hits.get("hits")).get(0); + final Map source = (Map) hit.get("_source"); + assertThat(source.get("version"), is(expectedVersion)); + }, 30, TimeUnit.SECONDS); + } + static Map toMap(Response response) throws IOException { return toMap(EntityUtils.toString(response.getEntity())); } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 451218c208f..8412f0e9584 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -122,6 +122,11 @@ subprojects { minimumMasterNodes = { 2 } clusterName = 'rolling-upgrade' waitCondition = waitWithAuth + setting 'xpack.monitoring.collection.interval', '-1' + setting 'xpack.monitoring.exporters._http.type', 'http' + setting 'xpack.monitoring.exporters._http.enabled', 'false' + setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' + setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.ssl.keystore.path', 'testnode.jks' @@ -161,6 +166,11 @@ subprojects { minimumMasterNodes = { 2 } dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir } waitCondition = waitWithAuth + setting 'xpack.monitoring.collection.interval', '-1' + setting 'xpack.monitoring.exporters._http.type', 'http' + setting 'xpack.monitoring.exporters._http.enabled', 'false' + setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' + setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.ssl.keystore.path', 'testnode.jks' keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode' @@ -194,6 +204,11 @@ subprojects { minimumMasterNodes = { 2 } dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir } waitCondition = waitWithAuth + setting 'xpack.monitoring.collection.interval', '-1' + setting 'xpack.monitoring.exporters._http.type', 'http' + setting 'xpack.monitoring.exporters._http.enabled', 'false' + setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' + setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.ssl.keystore.path', 'testnode.jks' keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode' diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 785ef22f8cf..0dcbdc7ac59 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry; import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase; import org.elasticsearch.xpack.test.rest.XPackRestTestCase; @@ -20,6 +21,13 @@ import org.junit.Before; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.is; @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYamlTestCase { @@ -32,6 +40,34 @@ public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYa XPackRestTestCase.waitForMlTemplates(); } + /** + * Enables an HTTP exporter for monitoring so that we can test the production-level exporter (not the local exporter). + * + * The build.gradle file disables data collection, so the expectation is that any monitoring rest tests will use the + * "_xpack/monitoring/_bulk" endpoint to lazily setup the templates on-demand and fill in data without worrying about + * timing. + */ + @Before + public void waitForMonitoring() throws Exception { + final String[] nodes = System.getProperty("tests.rest.cluster").split(","); + final Map settings = new HashMap<>(); + + settings.put("xpack.monitoring.exporters._http.enabled", true); + // only select the last node to avoid getting the "old" node in a mixed cluster + // if we ever randomize the order that the nodes are restarted (or add more nodes), then we need to verify which node we select + settings.put("xpack.monitoring.exporters._http.host", nodes[nodes.length - 1]); + + assertBusy(() -> { + final ClientYamlTestResponse response = + getAdminExecutionContext().callApi("cluster.put_settings", + emptyMap(), + singletonList(singletonMap("transient", settings)), + emptyMap()); + + assertThat(response.evaluate("acknowledged"), is(true)); + }); + } + @Override protected boolean preserveIndicesUponCompletion() { return true; diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_monitoring.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_monitoring.yml new file mode 100644 index 00000000000..d8de3d57064 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_monitoring.yml @@ -0,0 +1,44 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Index monitoring data and search on the mixed cluster": + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "old_cluster" } } } + - match: { hits.total: 2 } + + - do: + xpack.monitoring.bulk: + system_id: "kibana" + system_api_version: "6" + interval: "123456ms" + type: "mixed_cluster" + body: + - '{"index": {}}' + - '{"field": "value_3"}' + - '{"index": {}}' + - '{"field": "value_4"}' + - '{"index": {}}' + - '{"field": "value_5"}' + + - is_false: errors + + - do: + indices.refresh: {} + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "old_cluster" } } } + - match: { hits.total: 2 } + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "mixed_cluster" } } } + - match: { hits.total: 3 } diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_monitoring.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_monitoring.yml new file mode 100644 index 00000000000..7ada9b43e3c --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_monitoring.yml @@ -0,0 +1,30 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Index monitoring data and search on the old cluster": + - do: + xpack.monitoring.bulk: + system_id: "kibana" + system_api_version: "6" + interval: "123456ms" + type: "old_cluster" + body: + - '{"index": {}}' + - '{"field": "value_1"}' + - '{"index": {}}' + - '{"field": "value_2"}' + + - is_false: errors + + - do: + indices.refresh: {} + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "old_cluster" } } } + - match: { hits.total: 2 } diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_monitoring.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_monitoring.yml new file mode 100644 index 00000000000..e81a120750d --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_monitoring.yml @@ -0,0 +1,50 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Index monitoring data and search on the upgraded cluster": + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "old_cluster" } } } + - match: { hits.total: 2 } + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "mixed_cluster" } } } + - match: { hits.total: 3 } + + - do: + xpack.monitoring.bulk: + system_id: "kibana" + system_api_version: "6" + interval: "123456ms" + type: "upgraded_cluster" + body: + - '{"index": {}}' + - '{"field": "value_6"}' + - '{"index": {}}' + - '{"field": "value_7"}' + - '{"index": {}}' + - '{"field": "value_8"}' + + - is_false: errors + + - do: + indices.refresh: {} + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "terms" : { "type": [ "old_cluster", "mixed_cluster" ] } } } + - match: { hits.total: 5 } + + - do: + search: + index: .monitoring-kibana-* + body: { "query": { "term" : { "type": "upgraded_cluster" } } } + - match: { hits.total: 3 }