Monitoring: Move watcher tests for repository split preparations (elastic/x-pack-elasticsearch#3183)

Created a smoke-test-monitoring-with-watcher project that runs REST
tests with watcher enabled to ensure that the proper watcher are
installed either when the local or the HTTP exporter are set up.

Also removed two more watcher imports in the tests.

Relates elastic/x-pack-elasticsearch#2925

Original commit: elastic/x-pack-elasticsearch@0a9abc3185
This commit is contained in:
Alexander Reelsen 2017-12-01 13:20:05 +01:00 committed by GitHub
parent 55a19ed394
commit f816b2e850
10 changed files with 237 additions and 181 deletions

View File

@ -12,13 +12,13 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.rest.yaml.ObjectPath;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackFeatureSet.Usage;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.Exporters;
import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
import org.junit.Before;
import java.util.ArrayList;
@ -106,26 +106,26 @@ public class MonitoringFeatureSetTests extends ESTestCase {
for (XPackFeatureSet.Usage usage : Arrays.asList(monitoringUsage, serializedUsage)) {
assertThat(usage.name(), is(featureSet.name()));
assertThat(usage.enabled(), is(featureSet.enabled()));
XContentSource source;
ObjectPath source;
try (XContentBuilder builder = jsonBuilder()) {
usage.toXContent(builder, ToXContent.EMPTY_PARAMS);
source = new XContentSource(builder);
source = ObjectPath.createFromXContent(builder.contentType().xContent(), builder.bytes());
}
assertThat(source.getValue("enabled_exporters"), is(notNullValue()));
assertThat(source.evaluate("enabled_exporters"), is(notNullValue()));
if (localCount > 0) {
assertThat(source.getValue("enabled_exporters.local"), is(localCount));
assertThat(source.evaluate("enabled_exporters.local"), is(localCount));
} else {
assertThat(source.getValue("enabled_exporters.local"), is(nullValue()));
assertThat(source.evaluate("enabled_exporters.local"), is(nullValue()));
}
if (httpCount > 0) {
assertThat(source.getValue("enabled_exporters.http"), is(httpCount));
assertThat(source.evaluate("enabled_exporters.http"), is(httpCount));
} else {
assertThat(source.getValue("enabled_exporters.http"), is(nullValue()));
assertThat(source.evaluate("enabled_exporters.http"), is(nullValue()));
}
if (xCount > 0) {
assertThat(source.getValue("enabled_exporters." + xType), is(xCount));
assertThat(source.evaluate("enabled_exporters." + xType), is(xCount));
} else {
assertThat(source.getValue("enabled_exporters." + xType), is(nullValue()));
assertThat(source.evaluate("enabled_exporters." + xType), is(nullValue()));
}
}
}

View File

@ -22,11 +22,12 @@ import org.joda.time.format.DateTimeFormatter;
import java.util.Locale;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION;
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTestCase {
static Integer INDEX_TEMPLATE_VERSION = null;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings = Settings.builder()
@ -196,7 +197,10 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
* Creates a watcher history index from the current version.
*/
protected void createWatcherHistoryIndex(final DateTime creationDate) {
createWatcherHistoryIndex(creationDate, INDEX_TEMPLATE_VERSION);
if (INDEX_TEMPLATE_VERSION == null) {
INDEX_TEMPLATE_VERSION = randomIntBetween(1, 20);
}
createWatcherHistoryIndex(creationDate, String.valueOf(INDEX_TEMPLATE_VERSION));
}
/**

View File

@ -28,7 +28,6 @@ public abstract class LocalExporterIntegTestCase extends MonitoringIntegTestCase
protected final String exporterName = "_local";
private static ThreadPool THREADPOOL;
private static Boolean ENABLE_WATCHER;
@BeforeClass
public static void setupThreadPool() {
@ -37,38 +36,18 @@ public abstract class LocalExporterIntegTestCase extends MonitoringIntegTestCase
@AfterClass
public static void cleanUpStatic() throws Exception {
ENABLE_WATCHER = null;
if (THREADPOOL != null) {
terminate(THREADPOOL);
}
}
@Override
protected boolean enableWatcher() {
if (ENABLE_WATCHER == null) {
ENABLE_WATCHER = randomBoolean();
}
return ENABLE_WATCHER;
}
/**
* Override to disable the creation of Watches because of the setting (regardless of Watcher being enabled).
*
* @return Always {@code true} by default.
*/
protected boolean useClusterAlerts() {
return true;
}
protected Settings localExporterSettings() {
return Settings.builder()
.put(MonitoringService.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters." + exporterName + ".type", LocalExporter.TYPE)
.put("xpack.monitoring.exporters." + exporterName + ".enabled", false)
.put("xpack.monitoring.exporters." + exporterName + "." + CLUSTER_ALERTS_MANAGEMENT_SETTING, useClusterAlerts())
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.put("xpack.monitoring.exporters." + exporterName + "." + CLUSTER_ALERTS_MANAGEMENT_SETTING, false)
.put(XPackSettings.WATCHER_ENABLED.getKey(), false)
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
.build();
}

View File

@ -24,17 +24,12 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.XPackClient;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringService;
import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkRequestBuilder;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchRequest;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchResponse;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
@ -48,7 +43,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@ -180,7 +174,6 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
checkMonitoringTemplates();
checkMonitoringPipelines();
checkMonitoringWatches();
checkMonitoringDocs();
} finally {
stopMonitoring();
@ -252,23 +245,6 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
assertTrue("monitoring ingest pipeline not found", response.isFound());
}
/**
* Checks that the local exporter correctly creates Watches.
*/
private void checkMonitoringWatches() throws ExecutionException, InterruptedException {
if (enableWatcher()) {
final XPackClient xpackClient = new XPackClient(client());
final WatcherClient watcher = xpackClient.watcher();
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
final GetWatchResponse response = watcher.getWatch(new GetWatchRequest(uniqueWatchId)).get();
assertTrue("watch [" + watchId + "] should exist", response.isFound());
}
}
}
/**
* Checks that the monitoring documents all have the cluster_uuid, timestamp and source_node
* fields and belongs to the right data or timestamped index.

View File

@ -8,29 +8,15 @@ package org.elasticsearch.xpack.monitoring.exporter.local;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
import org.elasticsearch.xpack.watcher.client.WatchSourceBuilders;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.NeverCondition;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchResponse;
import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchRequest;
import org.elasticsearch.xpack.watcher.trigger.manual.ManualTrigger;
import org.elasticsearch.xpack.watcher.watch.Payload;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
@ -45,14 +31,8 @@ import static org.hamcrest.Matchers.notNullValue;
numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase {
private final boolean useClusterAlerts = enableWatcher() && randomBoolean();
private final MonitoredSystem system = randomFrom(MonitoredSystem.values());
@Override
protected boolean useClusterAlerts() {
return useClusterAlerts;
}
public void testCreateWhenResourcesNeedToBeAddedOrUpdated() throws Exception {
// sometimes they need to be added; sometimes they need to be replaced
if (randomBoolean()) {
@ -70,7 +50,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
// these were "newer" or at least the same version, so they shouldn't be replaced
assertTemplateNotUpdated();
assertPipelinesNotUpdated();
assertWatchesNotUpdated();
}
private void createResources() throws Exception {
@ -118,7 +97,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
putTemplate(version);
putPipelines(version);
putWatches(version);
}
private void putTemplate(final Integer version) throws Exception {
@ -138,49 +116,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
assertAcked(client().admin().cluster().preparePutPipeline(pipelineName, replaceablePipeline(version), XContentType.JSON).get());
}
private void putWatches(final Integer version) throws Exception {
if (enableWatcher()) {
// wait until the cluster is ready so that we can get the cluster's UUID
assertBusy(() -> assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION)));
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
putWatch(ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId), version);
}
}
}
private void putWatch(final String watchName, final Integer version) throws Exception {
final WatcherClient watcherClient = new WatcherClient(client());
assertThat(watcherClient.putWatch(new PutWatchRequest(watchName, replaceableWatch(version))).get().isCreated(), is(true));
}
/**
* Create a Watch with nothing in it that will never fire its action, with a metadata field named after {@linkplain #getTestName()}.
*
* @param version Version to add to the watch, if any
* @return Never {@code null}.
*/
private WatchSourceBuilder replaceableWatch(final Integer version) {
final WatchSourceBuilder builder = WatchSourceBuilders.watchBuilder();
final Map<String, Object> metadata = new HashMap<>();
// add a marker so that we know this test made the watch
metadata.put(getTestName(), true);
if (version != null) {
metadata.put("xpack", MapBuilder.<String, Object>newMapBuilder().put("version_created", version).map());
}
builder.metadata(metadata);
builder.trigger(new ManualTrigger());
builder.input(new SimpleInput(Payload.EMPTY));
builder.condition(NeverCondition.INSTANCE);
builder.addAction("_action", LoggingAction.builder(new TextTemplate("never runs")));
return builder;
}
/**
* Create a pipeline with nothing in it whose description is literally "test".
*
@ -239,20 +174,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
}
}
private void assertWatchesExist() {
if (enableWatcher()) {
final WatcherClient watcherClient = new WatcherClient(client());
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
final String watchName = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
final GetWatchResponse response = watcherClient.prepareGetWatch(watchName).get();
// this just ensures that it's set; not who set it
assertThat(response.isFound(), is(true));
}
}
}
private void assertResourcesExist() throws Exception {
createResources();
@ -261,11 +182,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
assertBusy(() -> {
assertTemplatesExist();
assertPipelinesExist();
if (useClusterAlerts) {
assertWatchesExist();
} else {
assertWatchesNotUpdated();
}
});
}
@ -287,22 +203,4 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
assertThat(description, equalTo(getTestName()));
}
}
private void assertWatchesNotUpdated() throws Exception {
if (enableWatcher()) {
final WatcherClient watcherClient = new WatcherClient(client());
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
final String watchName = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
final GetWatchResponse response = watcherClient.prepareGetWatch(watchName).get();
// if we didn't find the watch, then we certainly didn't update it
if (response.isFound()) {
// ensure that the watch still contains a value associated with the test rather than the real watch
assertThat(response.getSource().getValue("metadata." + getTestName()), notNullValue());
}
}
}
}
}

View File

@ -39,7 +39,6 @@ import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.security.Security;
import org.elasticsearch.xpack.security.authc.file.FileRealm;
import org.elasticsearch.xpack.security.authc.support.Hasher;
import org.elasticsearch.xpack.watcher.WatcherLifeCycleService;
import org.junit.After;
import org.junit.Before;
@ -90,7 +89,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.put(XPackSettings.WATCHER_ENABLED.getKey(), false)
// Disable native ML autodetect_process as the c++ controller won't be available
.put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
@ -136,12 +135,12 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
.put(Security.USER_SETTING.getKey(), "test:" + SecuritySettings.TEST_PASSWORD)
.put(NetworkModule.TRANSPORT_TYPE_KEY, Security.NAME4)
.put(NetworkModule.HTTP_TYPE_KEY, Security.NAME4)
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.put(XPackSettings.WATCHER_ENABLED.getKey(), false)
.build();
}
return Settings.builder().put(super.transportClientSettings())
.put(XPackSettings.SECURITY_ENABLED.getKey(), false)
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.put(XPackSettings.WATCHER_ENABLED.getKey(), false)
.build();
}
@ -197,26 +196,10 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
@After
public void tearDown() throws Exception {
if (enableWatcher()) {
internalCluster().getInstances(WatcherLifeCycleService.class)
.forEach(w -> w.stop("tearing down watcher as part of monitoring test case"));
}
stopMonitoringService();
super.tearDown();
}
/**
* Override and return {@code false} to force running without Watcher.
*
* Ensure that this method always returns the same value during a test run, do not put randomBoolean() in here
* as it is called more than once
*/
protected boolean enableWatcher() {
// Once randomDefault() becomes the default again, then this should only be actively disabled when
// trying to figure out exactly how many indices are at play
return false;
}
protected void startMonitoringService() {
internalCluster().getInstances(MonitoringService.class).forEach(MonitoringService::start);
}

View File

@ -0,0 +1,25 @@
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
dependencies {
testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime')
}
integTestCluster {
plugin ':x-pack-elasticsearch:plugin'
setting 'xpack.monitoring.enabled', 'true'
setting 'xpack.watcher.enabled', 'true'
setting 'xpack.security.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
// exporter settings are configured dynamically in our tests
// configure a local exporter, the HTTP exporter is configured via dynamic settings change
//setting 'xpack.monitoring.exporters.my_local.type', 'local'
//setting 'xpack.monitoring.exporters.my_local.index.name.time_format', 'YYYY'
//setting 'xpack.monitoring.exporters.my_http.type', 'http'
//setting 'xpack.monitoring.exporters.my_http.host', 'http'
//setting 'xpack.monitoring.exporters.my_http.index.name.time_format', 'YYYY-MM'
// one of the exporters should configure cluster alerts
// setting 'xpack.monitoring.exporters.my_http.cluster_alerts.management.enabled', 'true'
}

View File

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.smoketest;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class MonitoringWithWatcherClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public MonitoringWithWatcherClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
}

View File

@ -0,0 +1,80 @@
---
teardown:
- do:
cluster.put_settings:
body:
transient:
xpack.monitoring.exporters.*: null
# delete all watcher indices, so we will start clean again
- do:
indices.delete:
index: .watch*
---
"Watches are installed on startup with local exporter":
- do:
cluster.state:
metric: [ metadata ]
- set: { metadata.cluster_uuid : cluster_uuid }
- do:
xpack.watcher.put_watch:
id: ${cluster_uuid}_kibana_version_mismatch
body: >
{
"trigger" : {
"schedule": {
"interval" : "10m"
}
},
"input" : {
"simple" : {}
},
"actions" : {
"logme" : {
"logging" : {
"text" : "{{ctx}}"
}
}
}
}
- do:
cluster.put_settings:
body:
transient:
xpack.monitoring.exporters.my_local_exporter.type: "local"
xpack.monitoring.exporters.my_local_exporter.cluster_alerts.management.enabled: true
flat_settings: true
- match: {transient: {
"xpack.monitoring.exporters.my_local_exporter.type": "local",
"xpack.monitoring.exporters.my_local_exporter.cluster_alerts.management.enabled": "true"
}}
# sleep
- do:
catch: request_timeout
cluster.health:
wait_for_nodes: 99
timeout: 5s
- match: { "timed_out": true }
- do:
indices.refresh:
index: [ ".watches" ]
- do:
search:
index: .watches
- match: { hits.total: 5 }
- do:
xpack.watcher.get_watch:
id: ${cluster_uuid}_kibana_version_mismatch
# different interval than above means the watch was correctly replaced
- match: { watch.trigger.schedule.interval: "1m" }

View File

@ -0,0 +1,87 @@
---
teardown:
- do:
cluster.put_settings:
body:
transient:
xpack.monitoring.exporters.*: null
# delete all watcher indices, so we will start clean again
- do:
indices.delete:
index: .watch*
---
"Watches are installed on startup with http exporter":
- do:
cluster.state: {}
- set: { metadata.cluster_uuid : cluster_uuid }
- set: { master_node: master }
- do:
nodes.info: {}
- set: { nodes.$master.http.publish_address: http_host }
# install a watch that is going to be overwritten
- do:
xpack.watcher.put_watch:
id: ${cluster_uuid}_elasticsearch_cluster_status
body: >
{
"trigger" : {
"schedule": {
"interval" : "10m"
}
},
"input" : {
"simple" : {}
},
"actions" : {
"logme" : {
"logging" : {
"text" : "{{ctx}}"
}
}
}
}
- do:
cluster.put_settings:
body:
transient:
xpack.monitoring.exporters.my_http_exporter.type: "http"
xpack.monitoring.exporters.my_http_exporter.host: $http_host
xpack.monitoring.exporters.my_http_exporter.cluster_alerts.management.enabled: true
flat_settings: true
- match: {transient: {
"xpack.monitoring.exporters.my_http_exporter.type": "http",
"xpack.monitoring.exporters.my_http_exporter.host": "$http_host",
"xpack.monitoring.exporters.my_http_exporter.cluster_alerts.management.enabled": "true"
}}
# sleep
- do:
catch: request_timeout
cluster.health:
wait_for_nodes: 99
timeout: 10s
- match: { "timed_out": true }
- do:
indices.refresh:
index: [ ".watches" ]
- do:
search:
index: .watches
- match: { hits.total: 5 }
- do:
xpack.watcher.get_watch:
id: ${cluster_uuid}_elasticsearch_cluster_status
# different interval than above means the watch was correctly replaced
- match: { watch.trigger.schedule.interval: "1m" }