honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-24 17:09:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Monitoring] Upgrade Indices to remove usage of _type (elastic/x-pack-elasticsearch#1616)

Browse Source 
This is just the culmination of all of the minor PRs associated with 1068. It will:

- Drop the `.monitoring-data-N` index
- Drop use of `_type` in all cases (replaced by `doc` and a new `type` field)
- Drop the API version from the template name (e.g., instead of `.monitoring-es-6` we now use `.monitoring-es`).
- Change API version to `-6-` from `-2-`.
- Both exporters handle versioned resources (templates, pipelines, and watches)
- HTTP exporters will optionally (true by default) publish placeholders for the old, `-2` templates.

When this is backported, it will need to:

- Change `index_patterns` to `template` within the templates.
- Downgrade the version requirements for the templates, pipeline, and watches _and_ the HTTP exporter itself (all require 6.0)

This is a companion to the feature branch in X-Pack Kibana elastic/x-pack-kibana/pull/1318 and they need to be merged at the same time.

Original commit: elastic/x-pack-elasticsearch@6031cfffa4
This commit is contained in:
Chris Earle 2017-06-06 14:29:52 -04:00 committed by GitHub
parent 2d893df7e9
commit e5ee80c292
98 changed files with 3655 additions and 2908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java
View File

@ -168,8 +168,7 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
@Nullable
private String assignmentExplanation;
JobStats(String jobId, DataCounts dataCounts, @Nullable ModelSizeStats modelSizeStats, JobState state,
public JobStats(String jobId, DataCounts dataCounts, @Nullable ModelSizeStats modelSizeStats, JobState state,
@Nullable DiscoveryNode node, @Nullable String assignmentExplanation, @Nullable TimeValue opentime) {
this.jobId = Objects.requireNonNull(jobId);
this.dataCounts = Objects.requireNonNull(dataCounts);
@ -220,7 +219,15 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// TODO: Have callers wrap the content with an object as they choose rather than forcing it upon them
builder.startObject();
{
toUnwrappedXContent(builder);
}
return builder.endObject();
}
public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOException {
builder.field(Job.ID.getPreferredName(), jobId);
builder.field(DATA_COUNTS, dataCounts);
if (modelSizeStats != null) {
@ -247,7 +254,6 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
if (openTime != null) {
builder.field("open_time", openTime.getStringRep());
}
builder.endObject();
return builder;
}

9
plugin/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsCollecto
import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryCollector;
import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsCollector;
import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsCollector;
import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsCollector;
import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsCollector;
import org.elasticsearch.xpack.monitoring.collector.shards.ShardsCollector;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
@ -109,9 +110,8 @@ public class Monitoring implements ActionPlugin {
final ClusterSettings clusterSettings = clusterService.getClusterSettings();
final MonitoringSettings monitoringSettings = new MonitoringSettings(settings, clusterSettings);
final CleanerService cleanerService = new CleanerService(settings, clusterSettings, threadPool, licenseState);
// TODO: https://github.com/elastic/x-plugins/issues/3117 (remove dynamic need with static exporters)
final SSLService dynamicSSLService = sslService.createDynamicSSLService();
Map<String, Exporter.Factory> exporterFactories = new HashMap<>();
exporterFactories.put(HttpExporter.TYPE, config -> new HttpExporter(config, dynamicSSLService, threadPool.getThreadContext()));
exporterFactories.put(LocalExporter.TYPE, config -> new LocalExporter(config, client, cleanerService));
@ -125,8 +125,9 @@ public class Monitoring implements ActionPlugin {
collectors.add(new ShardsCollector(settings, clusterService, monitoringSettings, licenseState));
collectors.add(new NodeStatsCollector(settings, clusterService, monitoringSettings, licenseState, client));
collectors.add(new IndexRecoveryCollector(settings, clusterService, monitoringSettings, licenseState, client));
final MonitoringService monitoringService =
new MonitoringService(settings, clusterSettings, threadPool, collectors, exporters);
collectors.add(new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client));
final MonitoringService monitoringService = new MonitoringService(settings, clusterSettings, threadPool, collectors, exporters);
return Arrays.asList(monitoringService, monitoringSettings, exporters, cleanerService);
}

28
plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java
View File

@ -52,13 +52,13 @@ public class MonitoringSettings extends AbstractComponent {
Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting index statistics (default to 10m)
* Timeout value when collecting index statistics (default to 10s)
*/
public static final Setting<TimeValue> INDEX_STATS_TIMEOUT =
timeSetting(collectionKey("index.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting total indices statistics (default to 10m)
* Timeout value when collecting total indices statistics (default to 10s)
*/
public static final Setting<TimeValue> INDICES_STATS_TIMEOUT =
timeSetting(collectionKey("indices.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
@ -70,19 +70,25 @@ public class MonitoringSettings extends AbstractComponent {
listSetting(collectionKey("indices"), Collections.emptyList(), Function.identity(), Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting the cluster state (default to 10m)
* Timeout value when collecting the cluster state (default to 10s)
*/
public static final Setting<TimeValue> CLUSTER_STATE_TIMEOUT =
timeSetting(collectionKey("cluster.state.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting the recovery information (default to 10m)
* Timeout value when collecting the recovery information (default to 10s)
*/
public static final Setting<TimeValue> CLUSTER_STATS_TIMEOUT =
timeSetting(collectionKey("cluster.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting the recovery information (default to 10m)
* Timeout value when collecting ML job statistics (default to 10s)
*/
public static final Setting<TimeValue> JOB_STATS_TIMEOUT =
timeSetting(collectionKey("ml.job.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
/**
* Timeout value when collecting the recovery information (default to 10s)
*/
public static final Setting<TimeValue> INDEX_RECOVERY_TIMEOUT =
timeSetting(collectionKey("index.recovery.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
@ -125,6 +131,7 @@ public class MonitoringSettings extends AbstractComponent {
INDEX_RECOVERY_ACTIVE_ONLY,
CLUSTER_STATE_TIMEOUT,
CLUSTER_STATS_TIMEOUT,
JOB_STATS_TIMEOUT,
HISTORY_DURATION,
EXPORTERS_SETTINGS);
}
@ -139,6 +146,7 @@ public class MonitoringSettings extends AbstractComponent {
private volatile TimeValue clusterStateTimeout;
private volatile TimeValue clusterStatsTimeout;
private volatile TimeValue recoveryTimeout;
private volatile TimeValue jobStatsTimeout;
private volatile boolean recoveryActiveOnly;
private volatile String[] indices;
@ -155,6 +163,8 @@ public class MonitoringSettings extends AbstractComponent {
clusterSettings.addSettingsUpdateConsumer(CLUSTER_STATE_TIMEOUT, this::setClusterStateTimeout);
setClusterStatsTimeout(CLUSTER_STATS_TIMEOUT.get(settings));
clusterSettings.addSettingsUpdateConsumer(CLUSTER_STATS_TIMEOUT, this::setClusterStatsTimeout);
setJobStatsTimeout(JOB_STATS_TIMEOUT.get(settings));
clusterSettings.addSettingsUpdateConsumer(JOB_STATS_TIMEOUT, this::setJobStatsTimeout);
setRecoveryTimeout(INDEX_RECOVERY_TIMEOUT.get(settings));
clusterSettings.addSettingsUpdateConsumer(INDEX_RECOVERY_TIMEOUT, this::setRecoveryTimeout);
setRecoveryActiveOnly(INDEX_RECOVERY_ACTIVE_ONLY.get(settings));
@ -179,6 +189,10 @@ public class MonitoringSettings extends AbstractComponent {
return clusterStatsTimeout;
}
public TimeValue jobStatsTimeout() {
return jobStatsTimeout;
}
public TimeValue recoveryTimeout() {
return recoveryTimeout;
}
@ -203,6 +217,10 @@ public class MonitoringSettings extends AbstractComponent {
this.clusterStatsTimeout = clusterStatsTimeout;
}
private void setJobStatsTimeout(TimeValue jobStatsTimeout) {
this.jobStatsTimeout = jobStatsTimeout;
}
private void setRecoveryTimeout(TimeValue recoveryTimeout) {
this.recoveryTimeout = recoveryTimeout;
}

15
plugin/src/main/java/org/elasticsearch/xpack/monitoring/action/MonitoringIndex.java
View File

@ -20,12 +20,12 @@ import java.io.IOException;
public enum MonitoringIndex implements Writeable {
/**
* Data that drives information about the "cluster" (e.g., a node or instance).
* A formerly used index format, which is no longer relevant. This is maintained to allow BWC for older clients.
*/
DATA {
IGNORED_DATA {
@Override
public boolean matchesIndexName(String indexName) {
return "_data".equals(indexName);
return false;
}
},
@ -64,10 +64,11 @@ public enum MonitoringIndex implements Writeable {
* @throws IllegalArgumentException if {@code indexName} is unrecognized
*/
public static MonitoringIndex from(String indexName) {
for (MonitoringIndex index : values()) {
if (index.matchesIndexName(indexName)) {
return index;
}
if (TIMESTAMPED.matchesIndexName(indexName)) {
return TIMESTAMPED;
} else if ("_data".equals(indexName)) {
// we explicitly ignore this where it's used to maintain binary BWC
return IGNORED_DATA;
}
throw new IllegalArgumentException("unrecognized index name [" + indexName + "]");

64
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterInfoMonitoringDoc.java
View File

@ -1,64 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.List;
/**
* Monitoring document collected by {@link ClusterStatsCollector} and indexed in the
* monitoring data index. It contains all information about the current cluster, mostly
* for enabling/disabling features on Kibana side according to the license and also for
* the "phone home" feature.
*/
public class ClusterInfoMonitoringDoc extends MonitoringDoc {
public static final String TYPE = "cluster_info";
private final String clusterName;
private final String version;
private final License license;
private final List<XPackFeatureSet.Usage> usage;
private final ClusterStatsResponse clusterStats;
public ClusterInfoMonitoringDoc(String monitoringId, String monitoringVersion,
String clusterUUID, long timestamp, DiscoveryNode node,
String clusterName, String version, License license,
List<XPackFeatureSet.Usage> usage,
ClusterStatsResponse clusterStats) {
super(monitoringId, monitoringVersion, TYPE, clusterUUID, clusterUUID, timestamp, node);
this.clusterName = clusterName;
this.version = version;
this.license = license;
this.usage = usage;
this.clusterStats = clusterStats;
}
public String getClusterName() {
return clusterName;
}
public String getVersion() {
return version;
}
public License getLicense() {
return license;
}
public List<XPackFeatureSet.Usage> getUsage() {
return usage;
}
public ClusterStatsResponse getClusterStats() {
return clusterStats;
}
}

27
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java
View File

@ -8,8 +8,6 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
@ -18,10 +16,8 @@ import org.elasticsearch.xpack.monitoring.collector.Collector;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.security.InternalClient;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* Collector for cluster state.
@ -47,30 +43,15 @@ public class ClusterStateCollector extends Collector {
@Override
protected Collection<MonitoringDoc> doCollect() throws Exception {
List<MonitoringDoc> results = new ArrayList<>(3);
ClusterState clusterState = clusterService.state();
String clusterUUID = clusterState.metaData().clusterUUID();
String stateUUID = clusterState.stateUUID();
long timestamp = System.currentTimeMillis();
DiscoveryNode sourceNode = localNode();
ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth()
.get(monitoringSettings.clusterStateTimeout());
ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().get(monitoringSettings.clusterStateTimeout());
// Adds a cluster_state document with associated status
results.add(new ClusterStateMonitoringDoc(monitoringId(), monitoringVersion(), clusterUUID,
timestamp, sourceNode, clusterState, clusterHealth.getStatus()));
DiscoveryNodes nodes = clusterState.nodes();
if (nodes != null) {
for (DiscoveryNode node : nodes) {
// Adds a document for every node in the monitoring timestamped index
results.add(new ClusterStateNodeMonitoringDoc(monitoringId(), monitoringVersion(),
clusterUUID, timestamp, sourceNode, stateUUID, node.getId()));
}
}
return Collections.unmodifiableCollection(results);
return Collections.singleton(
new ClusterStateMonitoringDoc(monitoringId(), monitoringVersion(), clusterUUID,
timestamp, localNode(), clusterState, clusterHealth.getStatus()));
}
}

37
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateNodeMonitoringDoc.java
View File

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.cluster;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
/**
* Monitoring document collected by {@link ClusterStateCollector} that contains the id of
* every node of the cluster.
*/
public class ClusterStateNodeMonitoringDoc extends MonitoringDoc {
public static final String TYPE = "node";
private final String stateUUID;
private final String nodeId;
public ClusterStateNodeMonitoringDoc(String monitoringId, String monitoringVersion,
String clusterUUID, long timestamp, DiscoveryNode node,
String stateUUID, String nodeId) {
super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
this.stateUUID = stateUUID;
this.nodeId = nodeId;
}
public String getStateUUID() {
return stateUUID;
}
public String getNodeId() {
return nodeId;
}
}

20
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.xpack.monitoring.collector.Collector;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.security.InternalClient;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@ -50,7 +49,7 @@ public class ClusterStatsCollector extends Collector {
MonitoringSettings monitoringSettings,
XPackLicenseState licenseState, InternalClient client,
LicenseService licenseService) {
super(settings, "cluster-stats", clusterService, monitoringSettings, licenseState);
super(settings, ClusterStatsMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
this.client = client;
this.licenseService = licenseService;
}
@ -79,20 +78,11 @@ public class ClusterStatsCollector extends Collector {
final License license = licenseService.getLicense();
final List<XPackFeatureSet.Usage> usage = collect(usageSupplier);
final List<MonitoringDoc> results = new ArrayList<>(1);
// Adds a cluster info document
results.add(new ClusterInfoMonitoringDoc(monitoringId(), monitoringVersion(),
clusterUUID, timestamp, sourceNode, clusterName, version, license, usage,
clusterStats));
// Adds a cluster stats document
if (super.shouldCollect()) {
results.add(new ClusterStatsMonitoringDoc(monitoringId(), monitoringVersion(),
clusterUUID, timestamp, sourceNode, clusterStats));
}
return Collections.unmodifiableCollection(results);
return Collections.singleton(
new ClusterStatsMonitoringDoc(monitoringId(), monitoringVersion(),
clusterUUID, timestamp, sourceNode, clusterName, version, license, usage,
clusterStats));
}
@Nullable

39
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
View File

@ -7,25 +7,60 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.List;
/**
* Monitoring document collected by {@link ClusterStatsCollector} that contains the current
* cluster stats.
* Monitoring document collected by {@link ClusterStatsCollector}.
* <p>
* It contains all information about the current cluster, mostly for enabling/disabling features on Kibana side according to the license
* and also for the "phone home" feature.
* <p>
* In the future, the usage stats (and possibly the license) may be collected <em>less</em> frequently and therefore
* split into a separate monitoring document, but keeping them here simplifies the code.
*/
public class ClusterStatsMonitoringDoc extends MonitoringDoc {
public static final String TYPE = "cluster_stats";
private final String clusterName;
private final String version;
private final License license;
private final List<XPackFeatureSet.Usage> usage;
private final ClusterStatsResponse clusterStats;
public ClusterStatsMonitoringDoc(String monitoringId, String monitoringVersion,
String clusterUUID, long timestamp, DiscoveryNode node,
String clusterName, String version, License license,
List<XPackFeatureSet.Usage> usage,
ClusterStatsResponse clusterStats) {
super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
this.clusterName = clusterName;
this.version = version;
this.license = license;
this.usage = usage;
this.clusterStats = clusterStats;
}
public String getClusterName() {
return clusterName;
}
public String getVersion() {
return version;
}
public License getLicense() {
return license;
}
public List<XPackFeatureSet.Usage> getUsage() {
return usage;
}
public ClusterStatsResponse getClusterStats() {
return clusterStats;
}

75
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.ml;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.XPackClient;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction;
import org.elasticsearch.xpack.ml.client.MachineLearningClient;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.Collector;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.security.InternalClient;
import java.util.List;
import java.util.stream.Collectors;
/**
* Collector for Machine Learning Job Stats.
* <p>
* This collector runs on the master node because it's the only dependable place that requires it when X-Pack ML is enabled.
* If ML is not enabled or if it is not allowed to run because of the license, then this will not collect results.
* <p>
* Each Job Stats returned is used to create a separate {@link JobStatsMonitoringDoc}.
*/
public class JobStatsCollector extends Collector {
private final MachineLearningClient client;
public JobStatsCollector(final Settings settings, final ClusterService clusterService,
final MonitoringSettings monitoringSettings,
final XPackLicenseState licenseState, final InternalClient client) {
this(settings, clusterService, monitoringSettings, licenseState, new XPackClient(client).machineLearning());
}
JobStatsCollector(final Settings settings, final ClusterService clusterService,
final MonitoringSettings monitoringSettings,
final XPackLicenseState licenseState, final MachineLearningClient client) {
super(settings, JobStatsMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
this.client = client;
}
@Override
protected boolean shouldCollect() {
// This can only run when monitoring is allowed + ML is enabled/allowed, but also only on the elected master node
return super.shouldCollect() &&
XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) && licenseState.isMachineLearningAllowed() &&
isLocalNodeMaster();
}
@Override
protected List<MonitoringDoc> doCollect() throws Exception {
// fetch details about all jobs
final GetJobsStatsAction.Response jobs =
client.getJobsStats(new GetJobsStatsAction.Request(Job.ALL))
.actionGet(monitoringSettings.jobStatsTimeout());
final long timestamp = System.currentTimeMillis();
final String clusterUuid = clusterUUID();
final DiscoveryNode node = localNode();
return jobs.getResponse().results().stream()
.map(jobStats -> new JobStatsMonitoringDoc(clusterUuid, timestamp, node, jobStats))
.collect(Collectors.toList());
}
}

38
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDoc.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.ml;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.Objects;
/**
* Monitoring document collected by {@link JobStatsCollector}.
* <p>
* The collected details provide stats for an individual Machine Learning Job rather than the entire payload.
*/
public class JobStatsMonitoringDoc extends MonitoringDoc {
public static final String TYPE = "job_stats";
private final JobStats jobStats;
public JobStatsMonitoringDoc(final String clusterUuid, final long timestamp, final DiscoveryNode node,
final JobStats jobStats) {
super(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString(), TYPE, null, clusterUuid, timestamp, node);
this.jobStats = Objects.requireNonNull(jobStats);
}
public JobStats getJobStats() {
return jobStats;
}
}

9
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java
View File

@ -5,17 +5,16 @@
*/
package org.elasticsearch.xpack.monitoring.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.io.Streams;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;
/**
@ -46,6 +45,12 @@ public class ClusterAlertsUtil {
private static final Pattern UNIQUE_WATCH_ID_PROPERTY =
Pattern.compile(Pattern.quote("${monitoring.watch.unique_id}"));
/**
* The last time that all watches were updated. For now, all watches have been updated in the same version and should all be replaced
* together.
*/
public static final int LAST_UPDATED_VERSION = Version.V_6_0_0_alpha2.id;
/**
* An unsorted list of Watch IDs representing resource files for Monitoring Cluster Alerts.
*/

37
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java
View File

@ -7,22 +7,13 @@ package org.elasticsearch.xpack.monitoring.exporter;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class Exporter implements AutoCloseable {
/**
* The pipeline name passed with any <em>direct</em> indexing operation in order to support future API revisions.
*/
public static final String EXPORT_PIPELINE_NAME = "xpack_monitoring_" + MonitoringTemplateUtils.TEMPLATE_VERSION;
public static final String INDEX_NAME_TIME_FORMAT_SETTING = "index.name.time_format";
/**
* Every {@code Exporter} adds the ingest pipeline to bulk requests, but they should, at the exporter level, allow that to be disabled.
* <p>
@ -82,34 +73,6 @@ public abstract class Exporter implements AutoCloseable {
return MonitoringSettings.EXPORTERS_SETTINGS.getKey() + config.name + "." + setting;
}
/**
* Create an empty pipeline.
* <pre><code>
* {
* "description" : "2: This is a placeholder pipeline ...",
* "processors": [ ]
* }
* </code></pre>
* The expectation is that you will call either {@link XContentBuilder#string()} or {@link XContentBuilder#bytes()}}.
*
* @param type The type of data you want to format for the request
* @return Never {@code null}. Always an ended-object.
*/
public static XContentBuilder emptyPipeline(XContentType type) {
try {
// For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
// to pipelines, then it would better serve this use case
return XContentBuilder.builder(type.xContent()).startObject()
.field("description", MonitoringTemplateUtils.TEMPLATE_VERSION +
": This is a placeholder pipeline for Monitoring API version " +
MonitoringTemplateUtils.TEMPLATE_VERSION + " so that future versions may fix breaking changes.")
.startArray("processors").endArray()
.endObject();
} catch (IOException e) {
throw new RuntimeException("Failed to create empty pipeline", e);
}
}
public static class Config {
private final String name;

216
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtils.java
View File

@ -5,8 +5,12 @@
*/
package org.elasticsearch.xpack.monitoring.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.template.TemplateUtils;
import java.io.IOException;
import java.util.Locale;
import java.util.regex.Pattern;
@ -15,16 +19,22 @@ public final class MonitoringTemplateUtils {
private static final String TEMPLATE_FILE = "/monitoring-%s.json";
private static final String TEMPLATE_VERSION_PROPERTY = Pattern.quote("${monitoring.template.version}");
/** Current version of es and data templates **/
public static final String TEMPLATE_VERSION = "2";
/**
* The name of the non-timestamped data index.
* The last version of X-Pack that updated the templates and pipelines.
* <p>
* It may be possible for this to diverge between templates and pipelines, but for now they're the same.
*/
public static final String DATA_INDEX = ".monitoring-data-" + TEMPLATE_VERSION;
public static final int LAST_UPDATED_VERSION = Version.V_6_0_0_alpha2.id;
/**
* Data types that should be supported by the {@linkplain #DATA_INDEX data index} that were not by the initial release.
* Current version of templates used in their name to differentiate from breaking changes (separate from product version).
*/
public static final String[] NEW_DATA_TYPES = { "kibana", "logstash", "beats" };
public static final String TEMPLATE_VERSION = "6";
/**
* The previous version of templates, which we still support via the REST _xpack/monitoring/_bulk endpoint because
* nothing changed for those documents.
*/
public static final String OLD_TEMPLATE_VERSION = "2";
/**
* IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate} that are not managed by a Resolver.
@ -33,22 +43,204 @@ public final class MonitoringTemplateUtils {
*/
public static final String[] TEMPLATE_IDS = { "alerts" };
private MonitoringTemplateUtils() {
}
/**
* IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a
* Resolver.
* <p>
* These should only be used by the HTTP Exporter to create old templates so that older versions can be properly upgraded. Older
* instances will attempt to create a named template based on the templates that they expect (e.g., ".monitoring-es-2") and not the
* ones that we are creating.
*/
public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash", "alerts" };
/**
* IDs of pipelines that can be used with
*/
public static final String[] PIPELINE_IDS = { TEMPLATE_VERSION, OLD_TEMPLATE_VERSION };
private MonitoringTemplateUtils() { }
/**
* Get a template name for any template ID.
*
* @param id The template identifier.
* @return Never {@code null} {@link String} prefixed by ".monitoring-" and the
* @return Never {@code null} {@link String} prefixed by ".monitoring-".
* @see #TEMPLATE_IDS
*/
public static String templateName(String id) {
return ".monitoring-" + id + "-" + TEMPLATE_VERSION;
public static String templateName(final String id) {
return ".monitoring-" + id;
}
public static String loadTemplate(String id) {
/**
* Get a template name for any template ID for old templates in the previous version.
*
* @param id The template identifier.
* @return Never {@code null} {@link String} prefixed by ".monitoring-" and ended by the {@code OLD_TEMPLATE_VERSION}.
* @see #OLD_TEMPLATE_IDS
*/
public static String oldTemplateName(final String id) {
return ".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION;
}
public static String loadTemplate(final String id) {
String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id);
return TemplateUtils.loadTemplate(resource, TEMPLATE_VERSION, TEMPLATE_VERSION_PROPERTY);
}
/**
* Create a template that does nothing but exist and provide a newer {@code version} so that we know that <em>we</em> created it.
*
* @param id The template identifier.
* @return Never {@code null}.
* @see #OLD_TEMPLATE_IDS
* @see #OLD_TEMPLATE_VERSION
*/
public static String createEmptyTemplate(final String id) {
// e.g., { "index_patterns": [ ".monitoring-data-2*" ], "version": 6000002 }
return "{\"index_patterns\":[\".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION + "*\"],\"version\":" + LAST_UPDATED_VERSION + "}";
}
/**
* Get a pipeline name for any template ID.
*
* @param id The template identifier.
* @return Never {@code null} {@link String} prefixed by "xpack_monitoring_" and the {@code id}.
* @see #TEMPLATE_IDS
*/
public static String pipelineName(String id) {
return "xpack_monitoring_" + id;
}
/**
* Create a pipeline that allows documents for different template versions to be upgraded.
* <p>
* The expectation is that you will call either {@link XContentBuilder#string()} or {@link XContentBuilder#bytes()}}.
*
* @param id The API version (e.g., "2") to use
* @param type The type of data you want to format for the request
* @return Never {@code null}. Always an ended-object.
* @throws IllegalArgumentException if {@code apiVersion} is unrecognized
* @see #PIPELINE_IDS
*/
public static XContentBuilder loadPipeline(final String id, final XContentType type) {
switch (id) {
case TEMPLATE_VERSION:
return emptyPipeline(type);
case OLD_TEMPLATE_VERSION:
return pipelineForApiVersion2(type);
}
throw new IllegalArgumentException("unrecognized pipeline API version [" + id + "]");
}
/**
* Create a pipeline to upgrade documents from {@link MonitoringTemplateUtils#OLD_TEMPLATE_VERSION}
* <pre><code>
* {
* "description" : "This pipeline upgrades documents ...",
* "version": 6000001,
* "processors": [ ]
* }
* </code></pre>
* The expectation is that you will call either {@link XContentBuilder#string()} or {@link XContentBuilder#bytes()}}.
*
* @param type The type of data you want to format for the request
* @return Never {@code null}. Always an ended-object.
* @see #LAST_UPDATED_VERSION
*/
static XContentBuilder pipelineForApiVersion2(final XContentType type) {
try {
// For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
// to pipelines, then it would better serve this use case
return XContentBuilder.builder(type.xContent()).startObject()
.field("description", "This pipeline upgrades documents from the older version of the Monitoring API to " +
"the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
"changes in those older documents before they are indexed from the older version (" +
OLD_TEMPLATE_VERSION + ").")
.field("version", LAST_UPDATED_VERSION)
.startArray("processors")
.startObject()
// Drop the .monitoring-data-2 index and effectively drop unnecessary data (duplicate or simply unused)
.startObject("script")
.field("inline",
"boolean legacyIndex = ctx._index == '.monitoring-data-2';" +
"if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {" +
"if (ctx._type == 'cluster_info') {" +
"ctx._type = 'cluster_stats';" +
"ctx._id = null;" +
"} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {" +
"String index = ctx._index;" +
"Object clusterUuid = ctx.cluster_uuid;" +
"Object timestamp = ctx.timestamp;" +
"ctx.clear();" +
"ctx._id = 'xpack_monitoring_2_drop_bucket';" +
"ctx._index = index;" +
"ctx._type = 'legacy_data';" +
"ctx.timestamp = timestamp;" +
"ctx.cluster_uuid = clusterUuid;" +
"}" +
"if (legacyIndex) {" +
"ctx._index = '<.monitoring-es-" + TEMPLATE_VERSION + "-{now}>';" +
"}" +
"}")
.endObject()
.endObject()
.startObject()
.startObject("rename")
.field("field", "_type")
.field("target_field", "type")
.endObject()
.endObject()
.startObject()
.startObject("set")
.field("field", "_type")
.field("value", "doc")
.endObject()
.endObject()
.startObject()
.startObject("gsub")
.field("field", "_index")
.field("pattern", "(.monitoring-\\w+-)2(-.+)")
.field("replacement", "$1" + TEMPLATE_VERSION + "$2")
.endObject()
.endObject()
.endArray()
.endObject();
} catch (final IOException e) {
throw new RuntimeException("Failed to create pipeline to upgrade from older version [" + OLD_TEMPLATE_VERSION +
"] to the newer version [" + TEMPLATE_VERSION + "].", e);
}
}
/**
* Create an empty pipeline.
* <pre><code>
* {
* "description" : "This is a placeholder pipeline ...",
* "version": 6000001,
* "processors": [ ]
* }
* </code></pre>
* The expectation is that you will call either {@link XContentBuilder#string()} or {@link XContentBuilder#bytes()}}.
*
* @param type The type of data you want to format for the request
* @return Never {@code null}. Always an ended-object.
* @see #LAST_UPDATED_VERSION
*/
public static XContentBuilder emptyPipeline(final XContentType type) {
try {
// For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
// to pipelines, then it would better serve this use case
return XContentBuilder.builder(type.xContent()).startObject()
.field("description", "This is a placeholder pipeline for Monitoring API version " + TEMPLATE_VERSION +
" so that future versions may fix breaking changes.")
.field("version", LAST_UPDATED_VERSION)
.startArray("processors").endArray()
.endObject();
} catch (final IOException e) {
throw new RuntimeException("Failed to create empty pipeline", e);
}
}
}

72
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java
View File

@ -5,17 +5,27 @@
*/
package org.elasticsearch.xpack.monitoring.exporter.http;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.XPackLicenseState;
import java.util.Objects;
import java.util.function.Supplier;
import static org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil.LAST_UPDATED_VERSION;
/**
* {@code ClusterAlertHttpResource}s allow the checking, uploading, and deleting of Watches to a remote cluster based on the current license
* state.
@ -24,6 +34,12 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
private static final Logger logger = Loggers.getLogger(ClusterAlertHttpResource.class);
/**
* Use this to retrieve the version of Cluster Alert in the Watch's JSON response from a request.
*/
public static final Map<String, String> CLUSTER_ALERT_VERSION_PARAMETERS =
Collections.singletonMap("filter_path", "metadata.xpack.version_created");
/**
* License State is used to determine if we should even be add or delete our watches.
*/
@ -48,7 +64,7 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
final XPackLicenseState licenseState,
final Supplier<String> watchId, final Supplier<String> watch) {
// Watcher does not support master_timeout
super(resourceOwnerName, null, PublishableHttpResource.NO_BODY_PARAMETERS);
super(resourceOwnerName, null, CLUSTER_ALERT_VERSION_PARAMETERS);
this.licenseState = Objects.requireNonNull(licenseState);
this.watchId = Objects.requireNonNull(watchId);
@ -62,9 +78,13 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
protected CheckResponse doCheck(final RestClient client) {
// if we should be adding, then we need to check for existence
if (licenseState.isMonitoringClusterAlertsAllowed()) {
return simpleCheckForResource(client, logger,
"/_xpack/watcher/watch", watchId.get(), "monitoring cluster alert",
resourceOwnerName, "monitoring cluster");
final CheckedFunction<Response, Boolean, IOException> watchChecker =
(response) -> shouldReplaceClusterAlert(response, XContentType.JSON.xContent(), LAST_UPDATED_VERSION);
return versionCheckForResource(client, logger,
"/_xpack/watcher/watch", watchId.get(), "monitoring cluster alert",
resourceOwnerName, "monitoring cluster",
watchChecker);
}
// if we should be deleting, then just try to delete it (same level of effort as checking)
@ -94,4 +114,48 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
return new StringEntity(watch.get(), ContentType.APPLICATION_JSON);
}
/**
* Determine if the {@code response} contains a Watch whose value
*
* <p>
* This expects a response like:
* <pre><code>
* {
* "metadata": {
* "xpack": {
* "version": 6000002
* }
* }
* }
* </code></pre>
*
* @param response The filtered response from the Get Watcher API
* @param xContent The XContent parser to use
* @param minimumVersion The minimum version allowed without being replaced (expected to be the last updated version).
* @return {@code true} represents that it should be replaced. {@code false} that it should be left alone.
* @throws IOException if any issue occurs while parsing the {@code xContent} {@code response}.
* @throws RuntimeException if the response format is changed.
*/
boolean shouldReplaceClusterAlert(final Response response, final XContent xContent, final int minimumVersion) throws IOException {
// no named content used; so EMPTY is fine
final Map<String, Object> resources = XContentHelper.convertToMap(xContent, response.getEntity().getContent(), false);
// if it's empty, then there's no version in the response thanks to filter_path
if (resources.isEmpty() == false) {
@SuppressWarnings("unchecked")
final Map<String, Object> metadata = (Map<String, Object>) resources.get("metadata");
@SuppressWarnings("unchecked")
final Map<String, Object> xpack = metadata != null ? (Map<String, Object>) metadata.get("xpack") : null;
final Object version = xpack != null ? xpack.get("version_created") : null;
// if we don't have it (perhaps more fields were returned), then we need to replace it
if (version instanceof Number) {
// the version in the cluster alert is expected to include the alpha/beta/rc codes as well
return ((Number) version).intValue() < minimumVersion;
}
}
return true;
}
}

98
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/DataTypeMappingHttpResource.java
View File

@ -1,98 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.exporter.http;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.unit.TimeValue;
import java.util.Collections;
import java.util.Objects;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.DATA_INDEX;
/**
* {@linkplain DataTypeMappingHttpResource}s allow the checking and adding of index mapping's for new types that did not exist in previous
* versions.
* <p>
* This allows the use of Monitoring's REST endpoint to publish Kibana data to the data index even if the "kibana" type did not
* exist in their existing index mapping (e.g., they started with an early alpha release). Additionally, this also enables future types to
* be added without issue.
* <p>
* The root need for this is because the index mapping started with an index setting: "index.mapper.dynamic" set to false. This prevents
* new types from being dynamically added, which is obviously needed as new components (e.g., Kibana and Logstash) are monitored.
* Unfortunately, this setting cannot be flipped without also closing and reopening the index, so the fix is to manually add any new types.
*/
public class DataTypeMappingHttpResource extends PublishableHttpResource {
private static final Logger logger = Loggers.getLogger(DataTypeMappingHttpResource.class);
/**
* The name of the type that is created in the mappings on the remote cluster.
*/
private final String typeName;
/**
* Create a new {@link DataTypeMappingHttpResource}.
*
* @param resourceOwnerName The user-recognizable name
* @param masterTimeout Master timeout to use with any request.
* @param typeName The name of the mapping type (e.g., "kibana").
*/
public DataTypeMappingHttpResource(final String resourceOwnerName, @Nullable final TimeValue masterTimeout,
final String typeName) {
// we need to inspect the mappings, so we don't use filter_path to get rid of them
super(resourceOwnerName, masterTimeout, Collections.emptyMap());
this.typeName = Objects.requireNonNull(typeName);
}
/**
* Determine if the current {@linkplain #typeName type} exists.
*/
@Override
protected CheckResponse doCheck(final RestClient client) {
final Tuple<CheckResponse, Response> resource =
checkForResource(client, logger,
"/" + DATA_INDEX + "/_mapping", typeName, "monitoring mapping type",
resourceOwnerName, "monitoring cluster", GET_EXISTS, GET_DOES_NOT_EXIST);
// depending on the content, we need to flip the actual response
CheckResponse checkResponse = resource.v1();
if (checkResponse == CheckResponse.EXISTS && resource.v2().getEntity().getContentLength() <= 2) {
// it "exists" if the index exists at all; it doesn't guarantee that the mapping exists
// the content will be "{}" if no mapping exists
checkResponse = CheckResponse.DOES_NOT_EXIST;
} else if (checkResponse == CheckResponse.DOES_NOT_EXIST) {
// DNE indicates that the entire index is missing, which means the template will create it; we only add types!
checkResponse = CheckResponse.EXISTS;
}
return checkResponse;
}
/**
* Add the current {@linkplain #typeName type} to the index's mappings.
*/
@Override
protected boolean doPublish(final RestClient client) {
// this could be a class-level constant, but it does not need to live the entire duration of ES; only the few times it is used
final HttpEntity disabledEntity = new StringEntity("{\"enabled\":false}", ContentType.APPLICATION_JSON);
return putResource(client, logger,
"/" + DATA_INDEX + "/_mapping", typeName, () -> disabledEntity, "monitoring mapping type",
resourceOwnerName, "monitoring cluster");
}
}

31
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
View File

@ -13,6 +13,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseListener;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
@ -93,11 +95,28 @@ class HttpExportBulk extends ExportBulk {
} else if (payload.length != 0) {
final HttpEntity body = new ByteArrayEntity(payload, ContentType.APPLICATION_JSON);
client.performRequestAsync("POST", "/_bulk", params, body, HttpExportBulkResponseListener.INSTANCE);
client.performRequestAsync("POST", "/_bulk", params, body, new ResponseListener() {
@Override
public void onSuccess(Response response) {
try {
HttpExportBulkResponseListener.INSTANCE.onSuccess(response);
} finally {
listener.onResponse(null);
}
}
@Override
public void onFailure(Exception exception) {
try {
HttpExportBulkResponseListener.INSTANCE.onFailure(exception);
} finally {
listener.onFailure(exception);
}
}
});
// free the memory
payload = null;
listener.onResponse(null);
}
}
@ -117,7 +136,6 @@ class HttpExportBulk extends ExportBulk {
if (resolver != null) {
String index = resolver.index(doc);
String type = doc.getType();
String id = doc.getId();
try (XContentBuilder builder = new XContentBuilder(xContent, out)) {
@ -125,7 +143,7 @@ class HttpExportBulk extends ExportBulk {
builder.startObject();
builder.startObject("index");
builder.field("_index", index);
builder.field("_type", type);
builder.field("_type", "doc");
if (id != null) {
builder.field("_id", id);
}
@ -143,10 +161,9 @@ class HttpExportBulk extends ExportBulk {
// Adds final bulk separator
out.write(xContent.streamSeparator());
logger.trace("added index request [index={}, type={}, id={}]", index, type, id);
logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id);
} else {
logger.error("no resolver found for monitoring document [class={}]",
doc.getClass().getName());
logger.error("no resolver found for monitoring document [class={}]", doc.getClass().getName());
}
return BytesReference.toBytes(out.bytes());

58
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
View File

@ -124,6 +124,10 @@ public class HttpExporter extends Exporter {
* ES level timeout used when checking and writing templates (used to speed up tests)
*/
public static final String TEMPLATE_CHECK_TIMEOUT_SETTING = "index.template.master_timeout";
/**
* A boolean setting to enable or disable whether to create placeholders for the old templates.
*/
public static final String TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING = "index.template.create_legacy_templates";
/**
* ES level timeout used when checking and writing pipelines (used to speed up tests)
*/
@ -136,9 +140,9 @@ public class HttpExporter extends Exporter {
/**
* Minimum supported version of the remote monitoring cluster.
* <p>
* We must have support for ingest pipelines, which requires a minimum of 5.0.
* We must have support for the latest template syntax (index_patterns), which requires a minimum of 6.0.
*/
public static final Version MIN_SUPPORTED_CLUSTER_VERSION = Version.V_5_0_0_beta1;
public static final Version MIN_SUPPORTED_CLUSTER_VERSION = Version.V_6_0_0_alpha1;
/**
* The {@link RestClient} automatically pools connections and keeps them alive as necessary.
@ -514,7 +518,7 @@ public class HttpExporter extends Exporter {
// allow the use of ingest pipelines to be completely optional
if (settings.getAsBoolean(USE_INGEST_PIPELINE_SETTING, true)) {
params.put("pipeline", EXPORT_PIPELINE_NAME);
params.put("pipeline", MonitoringTemplateUtils.pipelineName(MonitoringTemplateUtils.TEMPLATE_VERSION));
}
// widdle down the response to just what we care to check
@ -533,16 +537,10 @@ public class HttpExporter extends Exporter {
*/
private static void configureTemplateResources(final Config config, final ResolversRegistry resolvers, final String resourceOwnerName,
final List<HttpResource> resources) {
final TimeValue templateTimeout = config.settings().getAsTime(TEMPLATE_CHECK_TIMEOUT_SETTING, null);
final Settings settings = config.settings();
final TimeValue templateTimeout = settings.getAsTime(TEMPLATE_CHECK_TIMEOUT_SETTING, null);
final Set<String> templateNames = new HashSet<>();
// add a resource to check the index mappings of the .monitoring-data-# index
// We ensure (and add if it's not) that the kibana type is there for the index for those few customers that upgraded from alphas;
// this step makes it very easy to add logstash in 5.2+ (and eventually beats)
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
resources.add(new DataTypeMappingHttpResource(resourceOwnerName, templateTimeout, type));
}
// add templates not managed by resolvers
for (final String templateId : MonitoringTemplateUtils.TEMPLATE_IDS) {
final String templateName = MonitoringTemplateUtils.templateName(templateId);
@ -560,6 +558,16 @@ public class HttpExporter extends Exporter {
resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, resolver::template));
}
}
// add old templates, like ".monitoring-data-2" and ".monitoring-es-2" so that other versions can continue to work
if (settings.getAsBoolean(TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING, true)) {
for (final String templateId : MonitoringTemplateUtils.OLD_TEMPLATE_IDS) {
final String templateName = MonitoringTemplateUtils.oldTemplateName(templateId);
final Supplier<String> templateLoader = () -> MonitoringTemplateUtils.createEmptyTemplate(templateId);
resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, templateLoader));
}
}
}
/**
@ -576,10 +584,16 @@ public class HttpExporter extends Exporter {
// don't require pipelines if we're not using them
if (settings.getAsBoolean(USE_INGEST_PIPELINE_SETTING, true)) {
final TimeValue pipelineTimeout = settings.getAsTime(PIPELINE_CHECK_TIMEOUT_SETTING, null);
// lazily load the pipeline
final Supplier<byte[]> pipeline = () -> BytesReference.toBytes(emptyPipeline(XContentType.JSON).bytes());
resources.add(new PipelineHttpResource(resourceOwnerName, pipelineTimeout, EXPORT_PIPELINE_NAME, pipeline));
// add all pipelines
for (final String pipelineId : MonitoringTemplateUtils.PIPELINE_IDS) {
final String pipelineName = MonitoringTemplateUtils.pipelineName(pipelineId);
// lazily load the pipeline
final Supplier<byte[]> pipeline =
() -> BytesReference.toBytes(MonitoringTemplateUtils.loadPipeline(pipelineId, XContentType.JSON).bytes());
resources.add(new PipelineHttpResource(resourceOwnerName, pipelineTimeout, pipelineName, pipeline));
}
}
}
@ -614,8 +628,12 @@ public class HttpExporter extends Exporter {
}
}
@Override
public HttpExportBulk openBulk() {
/**
* Determine if this {@link HttpExporter} is ready to use.
*
* @return {@code true} if it is ready. {@code false} if not.
*/
boolean isExporterReady() {
final boolean canUseClusterAlerts = config.licenseState().isMonitoringClusterAlertsAllowed();
// if this changes between updates, then we need to add OR remove the watches
@ -624,7 +642,13 @@ public class HttpExporter extends Exporter {
}
// block until all resources are verified to exist
if (resource.checkAndPublishIfDirty(client)) {
return resource.checkAndPublishIfDirty(client);
}
@Override
public HttpExportBulk openBulk() {
// block until all resources are verified to exist
if (isExporterReady()) {
return new HttpExportBulk(settingFQN(config), client, defaultParams, resolvers, threadContext);
}

11
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResource.java
View File

@ -13,6 +13,8 @@ import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import java.util.Objects;
import java.util.function.Supplier;
@ -46,7 +48,7 @@ public class PipelineHttpResource extends PublishableHttpResource {
*/
public PipelineHttpResource(final String resourceOwnerName, @Nullable final TimeValue masterTimeout,
final String pipelineName, final Supplier<byte[]> pipeline) {
super(resourceOwnerName, masterTimeout, PublishableHttpResource.NO_BODY_PARAMETERS);
super(resourceOwnerName, masterTimeout, PublishableHttpResource.RESOURCE_VERSION_PARAMETERS);
this.pipelineName = Objects.requireNonNull(pipelineName);
this.pipeline = Objects.requireNonNull(pipeline);
@ -57,9 +59,10 @@ public class PipelineHttpResource extends PublishableHttpResource {
*/
@Override
protected CheckResponse doCheck(final RestClient client) {
return simpleCheckForResource(client, logger,
"/_ingest/pipeline", pipelineName, "monitoring pipeline",
resourceOwnerName, "monitoring cluster");
return versionCheckForResource(client, logger,
"/_ingest/pipeline", pipelineName, "monitoring pipeline",
resourceOwnerName, "monitoring cluster",
XContentType.JSON.xContent(), MonitoringTemplateUtils.LAST_UPDATED_VERSION);
}
/**

148
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java
View File

@ -12,10 +12,13 @@ import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
@ -60,11 +63,20 @@ public abstract class PublishableHttpResource extends HttpResource {
* A value that will never match anything in the JSON response body, thus limiting it to "{}".
*/
public static final String FILTER_PATH_NONE = "$NONE";
/**
* A value that will match any top-level key and an inner "version" field, like '{"any-key":{"version":123}}'.
*/
public static final String FILTER_PATH_RESOURCE_VERSION = "*.version";
/**
* Use this to avoid getting any JSON response from a request.
*/
public static final Map<String, String> NO_BODY_PARAMETERS = Collections.singletonMap("filter_path", FILTER_PATH_NONE);
/**
* Use this to retrieve the version of template and pipeline resources in their JSON response from a request.
*/
public static final Map<String, String> RESOURCE_VERSION_PARAMETERS =
Collections.singletonMap("filter_path", FILTER_PATH_RESOURCE_VERSION);
/**
* The default set of acceptable exists response codes for GET requests.
@ -180,6 +192,100 @@ public abstract class PublishableHttpResource extends HttpResource {
.v1();
}
/**
* Determine if the current {@code resourceName} exists at the {@code resourceBasePath} endpoint with a version greater than or equal
* to the expected version.
* <p>
* This provides the base-level check for any resource that does not need to care about its response beyond existence (and likely does
* not need to inspect its contents).
* <p>
* This expects responses in the form of:
* <pre><code>
* {
* "resourceName": {
* "version": 6000002
* }
* }
* </code></pre>
*
* @param client The REST client to make the request(s).
* @param logger The logger to use for status messages.
* @param resourceBasePath The base path/endpoint to check for the resource (e.g., "/_template").
* @param resourceName The name of the resource (e.g., "template123").
* @param resourceType The type of resource (e.g., "monitoring template").
* @param resourceOwnerName The user-recognizeable resource owner.
* @param resourceOwnerType The type of resource owner being dealt with (e.g., "monitoring cluster").
* @param xContent The XContent used to parse the response.
* @param minimumVersion The minimum version allowed without being replaced (expected to be the last updated version).
* @return Never {@code null}.
*/
protected CheckResponse versionCheckForResource(final RestClient client, final Logger logger,
final String resourceBasePath,
final String resourceName, final String resourceType,
final String resourceOwnerName, final String resourceOwnerType,
final XContent xContent, final int minimumVersion) {
final CheckedFunction<Response, Boolean, IOException> responseChecker =
(response) -> shouldReplaceResource(response, xContent, resourceName, minimumVersion);
return versionCheckForResource(client, logger, resourceBasePath, resourceName, resourceType, resourceOwnerName, resourceOwnerType,
responseChecker);
}
/**
* Determine if the current {@code resourceName} exists at the {@code resourceBasePath} endpoint with a version greater than or equal
* to the expected version.
* <p>
* This provides the base-level check for any resource that does not need to care about its response beyond existence (and likely does
* not need to inspect its contents).
* <p>
* This expects responses in the form of:
* <pre><code>
* {
* "resourceName": {
* "version": 6000002
* }
* }
* </code></pre>
*
* @param client The REST client to make the request(s).
* @param logger The logger to use for status messages.
* @param resourceBasePath The base path/endpoint to check for the resource (e.g., "/_template").
* @param resourceName The name of the resource (e.g., "template123").
* @param resourceType The type of resource (e.g., "monitoring template").
* @param resourceOwnerName The user-recognizeable resource owner.
* @param resourceOwnerType The type of resource owner being dealt with (e.g., "monitoring cluster").
* @param responseChecker Determine if the resource should be replaced given the response.
* @return Never {@code null}.
*/
protected CheckResponse versionCheckForResource(final RestClient client, final Logger logger,
final String resourceBasePath,
final String resourceName, final String resourceType,
final String resourceOwnerName, final String resourceOwnerType,
final CheckedFunction<Response, Boolean, IOException> responseChecker) {
final Tuple<CheckResponse, Response> response =
checkForResource(client, logger, resourceBasePath, resourceName, resourceType, resourceOwnerName, resourceOwnerType,
GET_EXISTS, GET_DOES_NOT_EXIST);
final CheckResponse checkResponse = response.v1();
// if the template exists, then we also need to check its version
if (checkResponse == CheckResponse.EXISTS) {
try {
// replace the resource because the version is below what was required
if (responseChecker.apply(response.v2())) {
return CheckResponse.DOES_NOT_EXIST;
}
} catch (IOException | RuntimeException e) {
logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to parse [{}/{}] on the [{}]",
resourceBasePath, resourceName, resourceOwnerName), e);
return CheckResponse.ERROR;
}
}
return checkResponse;
}
/**
* Determine if the current {@code resourceName} exists at the {@code resourceBasePath} endpoint.
* <p>
@ -350,4 +456,46 @@ public abstract class PublishableHttpResource extends HttpResource {
return success;
}
/**
* Determine if the current resource should replaced the checked one based on its version (or lack thereof).
* <p>
* This expects a response like (where {@code resourceName} is replaced with its value):
* <pre><code>
* {
* "resourceName": {
* "version": 6000002
* }
* }
* </code></pre>
*
* @param response The filtered response from the _template/{name} or _ingest/pipeline/{name} resource APIs
* @param xContent The XContent parser to use
* @param resourceName The name of the looked up resource, which is expected to be the top-level key
* @param minimumVersion The minimum version allowed without being replaced (expected to be the last updated version).
* @return {@code true} represents that it should be replaced. {@code false} that it should be left alone.
* @throws IOException if any issue occurs while parsing the {@code xContent} {@code response}.
* @throws RuntimeException if the response format is changed.
*/
protected boolean shouldReplaceResource(final Response response, final XContent xContent,
final String resourceName, final int minimumVersion)
throws IOException {
// no named content used; so EMPTY is fine
final Map<String, Object> resources = XContentHelper.convertToMap(xContent, response.getEntity().getContent(), false);
// if it's empty, then there's no version in the response thanks to filter_path
if (resources.isEmpty() == false) {
@SuppressWarnings("unchecked")
final Map<String, Object> resource = (Map<String, Object>) resources.get(resourceName);
final Object version = resource != null ? resource.get("version") : null;
// if we don't have it (perhaps more fields were returned), then we need to replace it
if (version instanceof Number) {
// the version in the template is expected to include the alpha/beta/rc codes as well
return ((Number)version).intValue() < minimumVersion;
}
}
return true;
}
}

15
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
View File

@ -13,6 +13,8 @@ import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import java.util.Objects;
import java.util.function.Supplier;
@ -47,20 +49,23 @@ public class TemplateHttpResource extends PublishableHttpResource {
*/
public TemplateHttpResource(final String resourceOwnerName, @Nullable final TimeValue masterTimeout,
final String templateName, final Supplier<String> template) {
super(resourceOwnerName, masterTimeout, PublishableHttpResource.NO_BODY_PARAMETERS);
super(resourceOwnerName, masterTimeout, PublishableHttpResource.RESOURCE_VERSION_PARAMETERS);
this.templateName = Objects.requireNonNull(templateName);
this.template = Objects.requireNonNull(template);
}
/**
* Determine if the current {@linkplain #templateName template} exists.
* Determine if the current {@linkplain #templateName template} exists with a relevant version (&gt;= to expected).
*
* @see MonitoringTemplateUtils#LAST_UPDATED_VERSION
*/
@Override
protected CheckResponse doCheck(final RestClient client) {
return simpleCheckForResource(client, logger,
"/_template", templateName, "monitoring template",
resourceOwnerName, "monitoring cluster");
return versionCheckForResource(client, logger,
"/_template", templateName, "monitoring template",
resourceOwnerName, "monitoring cluster",
XContentType.JSON.xContent(), MonitoringTemplateUtils.LAST_UPDATED_VERSION);
}
/**

7
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
import org.elasticsearch.xpack.monitoring.exporter.ExportException;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
import org.elasticsearch.xpack.security.InternalClient;
@ -22,8 +23,6 @@ import org.elasticsearch.xpack.security.InternalClient;
import java.util.Arrays;
import java.util.Collection;
import static org.elasticsearch.xpack.monitoring.exporter.Exporter.EXPORT_PIPELINE_NAME;
/**
* LocalBulk exports monitoring data in the local cluster using bulk requests. Its usage is not thread safe since the
* {@link LocalBulk#add(Collection)}, {@link LocalBulk#flush(org.elasticsearch.action.ActionListener)} and
@ -61,7 +60,7 @@ public class LocalBulk extends ExportBulk {
try {
MonitoringIndexNameResolver<MonitoringDoc> resolver = resolvers.getResolver(doc);
IndexRequest request = new IndexRequest(resolver.index(doc), doc.getType());
IndexRequest request = new IndexRequest(resolver.index(doc), "doc");
if (Strings.hasText(doc.getId())) {
request.id(doc.getId());
}
@ -69,7 +68,7 @@ public class LocalBulk extends ExportBulk {
// allow the use of ingest pipelines to be completely optional
if (usePipeline) {
request.setPipeline(EXPORT_PIPELINE_NAME);
request.setPipeline(MonitoringTemplateUtils.pipelineName(MonitoringTemplateUtils.TEMPLATE_VERSION));
}
requestBuilder.add(request);

212
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.monitoring.exporter.local;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
@ -17,8 +16,6 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasA
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.ingest.PutPipelineRequest;
@ -29,11 +26,11 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.inject.internal.Nullable;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.regex.Regex;
@ -43,12 +40,12 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.ingest.IngestMetadata;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.XPackClient;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
@ -79,6 +76,10 @@ import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.loadPipeline;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName;
public class LocalExporter extends Exporter implements ClusterStateListener, CleanerService.Listener {
@ -91,6 +92,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
private final XPackLicenseState licenseState;
private final ResolversRegistry resolvers;
private final CleanerService cleanerService;
private final boolean useIngest;
private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
private final AtomicBoolean installingSomething = new AtomicBoolean(false);
@ -102,6 +104,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
this.client = client;
this.clusterService = config.clusterService();
this.licenseState = config.licenseState();
this.useIngest = config.settings().getAsBoolean(USE_INGEST_PIPELINE_SETTING, true);
this.cleanerService = cleanerService;
this.resolvers = new ResolversRegistry(config.settings());
@ -135,9 +138,21 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
watcherSetup.set(false);
}
/**
* Determine if this {@link LocalExporter} is ready to use.
*
* @return {@code true} if it is ready. {@code false} if not.
*/
boolean isExporterReady() {
// forces the setup to occur if it hasn't already
final boolean running = resolveBulk(clusterService.state(), false) != null;
return running && installingSomething.get() == false;
}
@Override
public ExportBulk openBulk() {
if (state.get() != State.RUNNING) {
public LocalBulk openBulk() {
if (state.get() != State.RUNNING) {
return null;
}
return resolveBulk(clusterService.state(), false);
@ -205,7 +220,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
clusterService.removeListener(this);
}
return new LocalBulk(name(), logger, client, resolvers, config.settings().getAsBoolean(USE_INGEST_PIPELINE_SETTING, true));
return new LocalBulk(name(), logger, client, resolvers, useIngest);
}
/**
@ -218,27 +233,24 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
* @return {@code true} indicates that all resources are available and the exporter can be used. {@code false} to stop and wait.
*/
private boolean setupIfNotElectedMaster(final ClusterState clusterState, final Set<String> templates) {
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
if (hasMappingType(type, clusterState) == false) {
// the required type is not yet there in the given cluster state, we'll wait.
logger.debug("monitoring index mapping [{}] does not exist in [{}], so service cannot start",
type, MonitoringTemplateUtils.DATA_INDEX);
return false;
}
}
// any required template is not yet installed in the given cluster state, we'll wait.
for (final String template : templates) {
if (hasTemplate(template, clusterState) == false) {
// the required template is not yet installed in the given cluster state, we'll wait.
logger.debug("monitoring index template [{}] does not exist, so service cannot start", template);
if (hasTemplate(clusterState, template) == false) {
logger.debug("monitoring index template [{}] does not exist, so service cannot start (waiting on master)",
template);
return false;
}
}
// if we don't have the ingest pipeline, then it's going to fail anyway
if (hasIngestPipelines(clusterState) == false) {
logger.debug("monitoring ingest pipeline [{}] does not exist, so service cannot start", EXPORT_PIPELINE_NAME);
return false;
if (useIngest) {
for (final String pipelineId : PIPELINE_IDS) {
if (hasIngestPipeline(clusterState, pipelineId) == false) {
logger.debug("monitoring ingest pipeline [{}] does not exist, so service cannot start (waiting on master)",
pipelineName(pipelineId));
return false;
}
}
}
if (null != prepareAddAliasesTo2xIndices(clusterState)) {
@ -279,25 +291,12 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
final List<Runnable> asyncActions = new ArrayList<>();
final AtomicInteger pendingResponses = new AtomicInteger(0);
// Check that all necessary types exist for _xpack/monitoring/_bulk usage
final List<String> missingMappingTypes = Arrays.stream(MonitoringTemplateUtils.NEW_DATA_TYPES)
.filter((type) -> hasMappingType(type, clusterState) == false)
.collect(Collectors.toList());
// Check that each required template exist, installing it if needed
// Check that each required template exists, installing it if needed
final List<Entry<String, String>> missingTemplates = templates.entrySet()
.stream()
.filter((e) -> hasTemplate(e.getKey(), clusterState) == false)
.filter((e) -> hasTemplate(clusterState, e.getKey()) == false)
.collect(Collectors.toList());
if (missingMappingTypes.isEmpty() == false) {
logger.debug((Supplier<?>) () -> new ParameterizedMessage("type {} not found",
missingMappingTypes.stream().collect(Collectors.toList())));
for (final String type : missingMappingTypes) {
asyncActions.add(() -> putMappingType(type, new ResponseActionListener<>("type", type, pendingResponses)));
}
}
if (missingTemplates.isEmpty() == false) {
logger.debug((Supplier<?>) () -> new ParameterizedMessage("template {} not found",
missingTemplates.stream().map(Map.Entry::getKey).collect(Collectors.toList())));
@ -307,12 +306,24 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
}
}
// if we don't have the ingest pipeline, then install it
if (hasIngestPipelines(clusterState) == false) {
logger.debug("pipeline [{}] not found", EXPORT_PIPELINE_NAME);
asyncActions.add(() -> putIngestPipeline(new ResponseActionListener<>("pipeline", EXPORT_PIPELINE_NAME, pendingResponses)));
} else {
logger.trace("pipeline [{}] found", EXPORT_PIPELINE_NAME);
if (useIngest) {
final List<String> missingPipelines = Arrays.stream(PIPELINE_IDS)
.filter(id -> hasIngestPipeline(clusterState, id) == false)
.collect(Collectors.toList());
// if we don't have the ingest pipeline, then install it
if (missingPipelines.isEmpty() == false) {
for (final String pipelineId : missingPipelines) {
final String pipelineName = pipelineName(pipelineId);
logger.debug("pipeline [{}] not found", pipelineName);
asyncActions.add(() -> putIngestPipeline(pipelineId,
new ResponseActionListener<>("pipeline",
pipelineName,
pendingResponses)));
}
} else {
logger.trace("all pipelines found");
}
}
IndicesAliasesRequest addAliasesTo2xIndices = prepareAddAliasesTo2xIndices(clusterState);
@ -391,50 +402,24 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
}
/**
* Determine if the mapping {@code type} exists in the {@linkplain MonitoringTemplateUtils#DATA_INDEX data index}.
*
* @param type The data type to check (e.g., "kibana")
* @param clusterState The current cluster state
* @return {@code false} if the type mapping needs to be added.
*/
private boolean hasMappingType(final String type, final ClusterState clusterState) {
final IndexMetaData dataIndex = clusterState.getMetaData().getIndices().get(MonitoringTemplateUtils.DATA_INDEX);
// if the index does not exist, then the template will add it and the type; if the index does exist, then we need the type
return dataIndex == null || dataIndex.getMappings().containsKey(type);
}
/**
* Add the mapping {@code type} to the {@linkplain MonitoringTemplateUtils#DATA_INDEX data index}.
*
* @param type The data type to check (e.g., "kibana")
* @param listener The listener to use for handling the response
*/
private void putMappingType(final String type, final ActionListener<PutMappingResponse> listener) {
logger.debug("adding mapping type [{}] to [{}]", type, MonitoringTemplateUtils.DATA_INDEX);
final PutMappingRequest putMapping = new PutMappingRequest(MonitoringTemplateUtils.DATA_INDEX);
putMapping.type(type);
// avoid mapping at all; we use this index as a data cache rather than for search
putMapping.source("{\"enabled\":false}", XContentType.JSON);
client.admin().indices().putMapping(putMapping, listener);
}
/**
* Determine if the ingest pipeline for {@link #EXPORT_PIPELINE_NAME} exists in the cluster or not.
* Determine if the ingest pipeline for {@code pipelineId} exists in the cluster or not with an appropriate minimum version.
*
* @param clusterState The current cluster state
* @return {@code true} if the {@code clusterState} contains a pipeline with {@link #EXPORT_PIPELINE_NAME}
* @param pipelineId The ID of the pipeline to check (e.g., "3")
* @return {@code true} if the {@code clusterState} contains the pipeline with an appropriate minimum version
*/
private boolean hasIngestPipelines(ClusterState clusterState) {
private boolean hasIngestPipeline(final ClusterState clusterState, final String pipelineId) {
final String pipelineName = MonitoringTemplateUtils.pipelineName(pipelineId);
final IngestMetadata ingestMetadata = clusterState.getMetaData().custom(IngestMetadata.TYPE);
// NOTE: this will need to become a more advanced check once we actually supply a meaningful pipeline
// because we will want to _replace_ older pipelines so that they go from (e.g., monitoring-2 doing nothing to
// monitoring-2 becoming monitoring-3 documents)
return ingestMetadata != null && ingestMetadata.getPipelines().containsKey(EXPORT_PIPELINE_NAME);
// we ensure that we both have the pipeline and its version represents the current (or later) version
if (ingestMetadata != null) {
final PipelineConfiguration pipeline = ingestMetadata.getPipelines().get(pipelineName);
return pipeline != null && hasValidVersion(pipeline.getConfigAsMap().get("version"), LAST_UPDATED_VERSION);
}
return false;
}
/**
@ -444,54 +429,33 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
* This should only be invoked by the <em>elected</em> master node.
* <p>
* Whenever we eventually make a backwards incompatible change, then we need to override any pipeline that already exists that is
* older than this one. Currently, we prepend the API version as the first part of the description followed by a <code>:</code>.
* older than this one. This uses the Elasticsearch version, down to the alpha portion, to determine the version of the last change.
* <pre><code>
* {
* "description": "2: This is a placeholder ...",
* "pipelines" : [ ... ]
* "description": "...",
* "pipelines" : [ ... ],
* "version": 6000001
* }
* </code></pre>
* That should be used (until something better exists) to ensure that we do not override <em>newer</em> pipelines with our own.
*/
private void putIngestPipeline(ActionListener<WritePipelineResponse> listener) {
logger.debug("installing ingest pipeline [{}]", EXPORT_PIPELINE_NAME);
private void putIngestPipeline(final String pipelineId, final ActionListener<WritePipelineResponse> listener) {
final String pipelineName = pipelineName(pipelineId);
final BytesReference pipeline = loadPipeline(pipelineId, XContentType.JSON).bytes();
final PutPipelineRequest request = new PutPipelineRequest(pipelineName, pipeline, XContentType.JSON);
final BytesReference emptyPipeline = emptyPipeline(XContentType.JSON).bytes();
final PutPipelineRequest request = new PutPipelineRequest(EXPORT_PIPELINE_NAME, emptyPipeline, XContentType.JSON);
logger.debug("installing ingest pipeline [{}]", pipelineName);
client.admin().cluster().putPipeline(request, listener);
}
/**
* List templates that exists in cluster state metadata and that match a given template name pattern.
*/
private ImmutableOpenMap<String, Integer> findTemplates(String templatePattern, ClusterState state) {
if (state == null || state.getMetaData() == null || state.getMetaData().getTemplates().isEmpty()) {
return ImmutableOpenMap.of();
}
private boolean hasTemplate(final ClusterState clusterState, final String templateName) {
final IndexTemplateMetaData template = clusterState.getMetaData().getTemplates().get(templateName);
ImmutableOpenMap.Builder<String, Integer> templates = ImmutableOpenMap.builder();
for (ObjectCursor<String> template : state.metaData().templates().keys()) {
if (Regex.simpleMatch(templatePattern, template.value)) {
try {
Integer version = Integer.parseInt(template.value.substring(templatePattern.length() - 1));
templates.put(template.value, version);
logger.debug("found index template [{}] in version [{}]", template.value, version);
} catch (NumberFormatException e) {
logger.warn("cannot extract version number for template [{}]", template.value);
}
}
}
return templates.build();
}
private boolean hasTemplate(String templateName, ClusterState state) {
ImmutableOpenMap<String, Integer> templates = findTemplates(templateName, state);
return templates.size() > 0;
return template != null && hasValidVersion(template.getVersion(), LAST_UPDATED_VERSION);
}
private void putTemplate(String template, String source, ActionListener<PutIndexTemplateResponse> listener) {
logger.debug("installing template [{}]",template);
logger.debug("installing template [{}]", template);
PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source, XContentType.JSON);
assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
@ -500,6 +464,17 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
client.admin().indices().putTemplate(request, listener);
}
/**
* Determine if the {@code version} is defined and greater than or equal to the {@code minimumVersion}.
*
* @param version The version to check
* @param minimumVersion The minimum version required to be a "valid" version
* @return {@code true} if the version exists and it's &gt;= to the minimum version. {@code false} otherwise.
*/
private boolean hasValidVersion(final Object version, final long minimumVersion) {
return version instanceof Number && ((Number)version).intValue() >= minimumVersion;
}
/**
* Install Cluster Alerts (Watches) into the cluster
*
@ -721,7 +696,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
@Override
public void onResponse(GetWatchResponse response) {
if (response.isFound()) {
if (response.isFound() &&
hasValidVersion(response.getSource().getValue("metadata.xpack.version_created"), ClusterAlertsUtil.LAST_UPDATED_VERSION)) {
logger.trace("found monitoring watch [{}]", uniqueWatchId);
responseReceived(countDown, true, watcherSetup);

45
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/MonitoringIndexNameResolver.java
View File

@ -24,7 +24,6 @@ import org.joda.time.format.DateTimeFormatter;
import java.io.IOException;
import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import java.util.function.Function;
@ -68,6 +67,7 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
builder.field(Fields.CLUSTER_UUID, document.getClusterUUID());
DateTime timestampDateTime = new DateTime(document.getTimestamp(), DateTimeZone.UTC);
builder.field(Fields.TIMESTAMP, timestampDateTime.toString());
builder.field(Fields.TYPE, document.getType());
MonitoringDoc.Node sourceNode = document.getSourceNode();
if (sourceNode != null) {
@ -105,49 +105,10 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
public static final class Fields {
public static final String CLUSTER_UUID = "cluster_uuid";
public static final String TIMESTAMP = "timestamp";
public static final String TYPE = "type";
public static final String SOURCE_NODE = "source_node";
}
/**
* Data index name resolvers are used used to index documents in
* the monitoring data index (.monitoring-data-{VERSION})
*/
public abstract static class Data<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
public static final String DATA = "data";
private final String index;
public Data() {
this(MonitoringTemplateUtils.TEMPLATE_VERSION);
}
// Used in tests
protected Data(String version) {
this.index = String.join(DELIMITER, PREFIX, DATA, version);
}
@Override
public String index(T document) {
return index;
}
@Override
public String indexPattern() {
return index;
}
@Override
public String templateName() {
return String.format(Locale.ROOT, "%s-%s-%s", PREFIX, DATA, MonitoringTemplateUtils.TEMPLATE_VERSION);
}
@Override
public String template() {
return MonitoringTemplateUtils.loadTemplate(DATA);
}
}
/**
* Timestamped index name resolvers are used used to index documents in
* a timestamped index (.monitoring-{ID}-{VERSION}-YYYY.MM.dd)
@ -189,7 +150,7 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
@Override
public String templateName() {
return String.format(Locale.ROOT, "%s-%s-%s", PREFIX, getId(), MonitoringTemplateUtils.TEMPLATE_VERSION);
return MonitoringTemplateUtils.templateName(getId());
}
@Override

40
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java
View File

@ -9,30 +9,29 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.bulk.MonitoringBulkDataResolver;
import org.elasticsearch.xpack.monitoring.resolver.bulk.MonitoringBulkTimestampedResolver;
import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterInfoResolver;
import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateNodeResolver;
import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateResolver;
import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStatsResolver;
import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateResolver;
import org.elasticsearch.xpack.monitoring.resolver.indices.IndexRecoveryResolver;
import org.elasticsearch.xpack.monitoring.resolver.indices.IndexStatsResolver;
import org.elasticsearch.xpack.monitoring.resolver.indices.IndicesStatsResolver;
import org.elasticsearch.xpack.monitoring.resolver.ml.JobStatsResolver;
import org.elasticsearch.xpack.monitoring.resolver.node.NodeStatsResolver;
import org.elasticsearch.xpack.monitoring.resolver.shards.ShardsResolver;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
@ -46,18 +45,22 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
// register built-in defaults resolvers
registerBuiltIn(MonitoredSystem.ES, settings);
final List<String> supportedApiVersions = Arrays.asList(
MonitoringTemplateUtils.TEMPLATE_VERSION,
MonitoringTemplateUtils.OLD_TEMPLATE_VERSION
);
// register resolvers for monitored systems
registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings);
registerMonitoredSystem(MonitoredSystem.BEATS, settings);
registerMonitoredSystem(MonitoredSystem.KIBANA, settings, supportedApiVersions);
registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings, supportedApiVersions);
// Beats did not report data in the 5.x timeline, so it should never send the original version
registerMonitoredSystem(MonitoredSystem.BEATS, settings, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
}
/**
* Registers resolvers for elasticsearch documents collected by the monitoring plugin
*/
private void registerBuiltIn(MonitoredSystem id, Settings settings) {
registrations.add(resolveByClass(ClusterInfoMonitoringDoc.class, new ClusterInfoResolver()));
registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, settings)));
registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, settings)));
registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings)));
registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings)));
@ -65,21 +68,18 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings)));
registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, settings)));
registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, settings)));
registrations.add(resolveByClass(JobStatsMonitoringDoc.class, new JobStatsResolver(id, settings)));
}
/**
* Registers resolvers for monitored systems
*/
private void registerMonitoredSystem(MonitoredSystem id, Settings settings) {
final MonitoringBulkDataResolver dataResolver = new MonitoringBulkDataResolver();
private void registerMonitoredSystem(final MonitoredSystem id, final Settings settings, final List<String> supportedApiVersions) {
final MonitoringBulkTimestampedResolver timestampedResolver = new MonitoringBulkTimestampedResolver(id, settings);
final String currentApiVersion = MonitoringTemplateUtils.TEMPLATE_VERSION;
// Note: We resolve requests by the API version that is supplied; this allows us to translate and up-convert any older
// requests that come through the _xpack/monitoring/_bulk endpoint
registrations.add(resolveByClassSystemVersion(id, dataResolver, MonitoringIndex.DATA, currentApiVersion));
registrations.add(resolveByClassSystemVersion(id, timestampedResolver, MonitoringIndex.TIMESTAMPED, currentApiVersion));
registrations.add(resolveByClassSystemVersion(id, timestampedResolver, MonitoringIndex.TIMESTAMPED, supportedApiVersions));
}
/**
@ -104,7 +104,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
}
static Registration resolveByClassSystemVersion(MonitoredSystem system, MonitoringIndexNameResolver resolver, MonitoringIndex index,
String apiVersion) {
List<String> supportedApiVersion) {
return new Registration(resolver, doc -> {
try {
if (doc instanceof MonitoringBulkDoc == false || index != ((MonitoringBulkDoc)doc).getIndex()) {
@ -113,7 +113,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
if (system != MonitoredSystem.fromSystem(doc.getMonitoringId())) {
return false;
}
return apiVersion.equals(doc.getMonitoringVersion());
return supportedApiVersion.contains(doc.getMonitoringVersion());
} catch (Exception e) {
return false;
}

26
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/bulk/MonitoringBulkDataResolver.java
View File

@ -1,26 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.bulk;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
public class MonitoringBulkDataResolver extends MonitoringIndexNameResolver.Data<MonitoringBulkDoc> {
@Override
protected void buildXContent(MonitoringBulkDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
BytesReference source = document.getSource();
if (source != null && source.length() > 0) {
builder.rawField(document.getType(), source, document.getXContentType());
}
}
}

70
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoResolver.java
View File

@ -1,70 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data<ClusterInfoMonitoringDoc> {
@Override
protected void buildXContent(ClusterInfoMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.field("cluster_name", document.getClusterName());
builder.field("version", document.getVersion());
final License license = document.getLicense();
if (license != null) {
builder.startObject("license");
Map<String, String> extraParams = new MapBuilder<String, String>()
.put(License.REST_VIEW_MODE, "true")
.map();
params = new ToXContent.DelegatingMapParams(extraParams, params);
license.toInnerXContent(builder, params);
builder.field("hkey", hash(license, document.getClusterUUID()));
builder.endObject();
}
final ClusterStatsResponse clusterStats = document.getClusterStats();
if (clusterStats != null) {
builder.startObject("cluster_stats");
clusterStats.toXContent(builder, params);
builder.endObject();
}
final List<XPackFeatureSet.Usage> usages = document.getUsage();
if (usages != null) {
// in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
// it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
builder.startObject("stack_stats").startObject("xpack");
for (final XPackFeatureSet.Usage usage : usages) {
builder.field(usage.name(), usage);
}
builder.endObject().endObject();
}
}
public static String hash(License license, String clusterName) {
return hash(license.status().label(), license.uid(), license.type(), String.valueOf(license.expiryDate()), clusterName);
}
public static String hash(String licenseStatus, String licenseUid, String licenseType, String licenseExpiryDate, String clusterUUID) {
String toHash = licenseStatus + licenseUid + licenseType + licenseExpiryDate + clusterUUID;
return MessageDigests.toHexString(MessageDigests.sha256().digest(toHash.getBytes(StandardCharsets.UTF_8)));
}
}

37
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateNodeResolver.java
View File

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
public class ClusterStateNodeResolver extends MonitoringIndexNameResolver.Timestamped<ClusterStateNodeMonitoringDoc> {
public ClusterStateNodeResolver(MonitoredSystem id, Settings settings) {
super(id, settings);
}
@Override
protected void buildXContent(ClusterStateNodeMonitoringDoc document,
XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.field(Fields.STATE_UUID, document.getStateUUID());
builder.startObject(Fields.NODE);
builder.field(Fields.ID, document.getNodeId());
builder.endObject();
}
static final class Fields {
static final String STATE_UUID = "state_uuid";
static final String NODE = "node";
static final String ID = "id";
}
}

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateResolver.java
View File

@ -28,6 +28,7 @@ public class ClusterStateResolver extends MonitoringIndexNameResolver.Timestampe
Set<String> filters = Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"cluster_state.version",
"cluster_state.master_node",

89
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsResolver.java
View File

@ -6,64 +6,71 @@
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
public class ClusterStatsResolver extends MonitoringIndexNameResolver.Timestamped<ClusterStatsMonitoringDoc> {
public static final String TYPE = "cluster_stats";
static final Set<String> FILTERS;
static {
Set<String> filters = Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"cluster_stats.nodes.count.total",
"cluster_stats.indices.count",
"cluster_stats.indices.shards.total",
"cluster_stats.indices.shards.primaries",
"cluster_stats.indices.docs.count",
"cluster_stats.indices.store.size_in_bytes",
"cluster_stats.nodes.fs.total_in_bytes",
"cluster_stats.nodes.fs.free_in_bytes",
"cluster_stats.nodes.fs.available_in_bytes",
"cluster_stats.nodes.jvm.max_uptime_in_millis",
"cluster_stats.nodes.jvm.mem.heap_max_in_bytes",
"cluster_stats.nodes.jvm.mem.heap_used_in_bytes",
"cluster_stats.nodes.versions");
FILTERS = Collections.unmodifiableSet(filters);
}
public ClusterStatsResolver(MonitoredSystem id, Settings settings) {
super(id, settings);
}
@Override
public Set<String> filters() {
return FILTERS;
public ClusterStatsResolver(MonitoredSystem system, Settings settings) {
super(system, settings);
}
@Override
protected void buildXContent(ClusterStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.CLUSTER_STATS);
ClusterStatsResponse clusterStats = document.getClusterStats();
if (clusterStats != null) {
clusterStats.toXContent(builder, params);
builder.field("cluster_name", document.getClusterName());
builder.field("version", document.getVersion());
final License license = document.getLicense();
if (license != null) {
builder.startObject("license");
Map<String, String> extraParams = new MapBuilder<String, String>()
.put(License.REST_VIEW_MODE, "true")
.map();
params = new ToXContent.DelegatingMapParams(extraParams, params);
license.toInnerXContent(builder, params);
builder.field("hkey", hash(license, document.getClusterUUID()));
builder.endObject();
}
final ClusterStatsResponse clusterStats = document.getClusterStats();
if (clusterStats != null) {
builder.startObject("cluster_stats");
clusterStats.toXContent(builder, params);
builder.endObject();
}
final List<XPackFeatureSet.Usage> usages = document.getUsage();
if (usages != null) {
// in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
// it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
builder.startObject("stack_stats").startObject("xpack");
for (final XPackFeatureSet.Usage usage : usages) {
builder.field(usage.name(), usage);
}
builder.endObject().endObject();
}
builder.endObject();
}
static final class Fields {
static final String CLUSTER_STATS = TYPE;
public static String hash(License license, String clusterName) {
return hash(license.status().label(), license.uid(), license.type(), String.valueOf(license.expiryDate()), clusterName);
}
public static String hash(String licenseStatus, String licenseUid, String licenseType, String licenseExpiryDate, String clusterUUID) {
String toHash = licenseStatus + licenseUid + licenseType + licenseExpiryDate + clusterUUID;
return MessageDigests.toHexString(MessageDigests.sha256().digest(toHash.getBytes(StandardCharsets.UTF_8)));
}
}

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexStatsResolver.java
View File

@ -27,6 +27,7 @@ public class IndexStatsResolver extends MonitoringIndexNameResolver.Timestamped<
Set<String> filters = Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"index_stats.index",
"index_stats.primaries.docs.count",

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsResolver.java
View File

@ -27,6 +27,7 @@ public class IndicesStatsResolver extends MonitoringIndexNameResolver.Timestampe
Set<String> filters = Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"indices_stats._all.primaries.docs.count",
"indices_stats._all.primaries.indexing.index_time_in_millis",

35
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ml/JobStatsResolver.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.ml;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
public class JobStatsResolver extends MonitoringIndexNameResolver.Timestamped<JobStatsMonitoringDoc> {
public JobStatsResolver(MonitoredSystem system, Settings settings) {
super(system, settings);
}
@Override
protected void buildXContent(JobStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
final JobStats jobStats = document.getJobStats();
builder.startObject(document.getType());
{
jobStats.toUnwrappedXContent(builder);
}
builder.endObject();
}
}

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsResolver.java
View File

@ -28,6 +28,7 @@ public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped<N
// Common information
"cluster_uuid",
"timestamp",
"type",
"source_node",
// Extra information
"node_stats.node_id",

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/shards/ShardsResolver.java
View File

@ -27,6 +27,7 @@ public class ShardsResolver extends MonitoringIndexNameResolver.Timestamped<Shar
Set<String> filters = Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"state_uuid",
"shard.state",

2
plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java
View File

@ -49,7 +49,7 @@ public class ReservedRolesStore {
.put("remote_monitoring_agent", new RoleDescriptor("remote_monitoring_agent",
new String[] {
"manage_index_templates", "manage_ingest_pipelines", "monitor",
"cluster:admin/xpack/watcher/watch/get",
"cluster:monitor/xpack/watcher/watch/get",
"cluster:admin/xpack/watcher/watch/put",
"cluster:admin/xpack/watcher/watch/delete",
},

9
plugin/src/main/resources/monitoring-alerts.json
View File

@ -1,15 +1,16 @@
{
"index_patterns": ".monitoring-alerts-${monitoring.template.version}",
"version": 6000002,
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 1,
"codec": "best_compression",
"mapping.single_type": false
"codec": "best_compression"
}
},
"mappings": {
"doc": {
"dynamic": false,
"properties": {
"timestamp": {
"type": "date"
@ -30,7 +31,6 @@
"type": "text"
},
"metadata": {
"dynamic": false,
"properties": {
"cluster_uuid": {
"type": "keyword"
@ -54,6 +54,5 @@
}
}
}
},
"version": 60000
}
}

28
plugin/src/main/resources/monitoring-beats.json
View File

@ -1,13 +1,14 @@
{
"index_patterns": ".monitoring-beats-${monitoring.template.version}-*",
"version": 6000002,
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapping.single_type": false
"index.codec": "best_compression"
},
"mappings": {
"beats_stats": {
"doc": {
"dynamic": false,
"properties": {
"cluster_uuid": {
"type": "keyword"
@ -16,6 +17,9 @@
"type": "date",
"format": "date_time"
},
"type": {
"type": "keyword"
},
"source_node": {
"properties": {
"uuid": {
@ -32,20 +36,6 @@
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
},
@ -76,10 +66,6 @@
"type": "keyword"
}
}
},
"metrics": {
"properties": {
}
}
}
}

32
plugin/src/main/resources/monitoring-data.json
View File

@ -1,32 +0,0 @@
{
"index_patterns": ".monitoring-data-${monitoring.template.version}",
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapping.single_type": false
},
"mappings": {
"_default_": {
"enabled": false
},
"cluster_info": {
"enabled": false,
"_meta": {
"xpack.version": "${project.version}"
}
},
"kibana": {
"enabled": false
},
"node": {
"enabled": false
},
"logstash": {
"enabled": false
},
"beats": {
"enabled": false
}
}
}

355
plugin/src/main/resources/monitoring-es.json
View File

@ -1,23 +1,29 @@
{
"index_patterns": ".monitoring-es-${monitoring.template.version}-*",
"version": 6000002,
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapper.dynamic": false,
"index.mapping.single_type": false
"index.codec": "best_compression"
},
"mappings": {
"_default_": {
"doc": {
"date_detection": false,
"dynamic": false,
"properties": {
"cluster_uuid": {
"type": "keyword"
},
"state_uuid": {
"type": "keyword"
},
"timestamp": {
"type": "date",
"format": "date_time"
},
"type": {
"type": "keyword"
},
"source_node": {
"properties": {
"uuid": {
@ -34,27 +40,9 @@
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
}
}
},
"indices_stats": {
"properties": {
},
"indices_stats": {
"properties": {
"_all": {
@ -67,6 +55,26 @@
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
},
"index_time_in_millis": {
"type": "long"
}
}
},
"search": {
"properties": {
"query_total": {
"type": "long"
},
"query_time_in_millis": {
"type": "long"
}
}
}
}
},
@ -78,17 +86,33 @@
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
},
"index_time_in_millis": {
"type": "long"
}
}
},
"search": {
"properties": {
"query_total": {
"type": "long"
},
"query_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
}
}
},
"index_stats": {
"properties": {
},
"index_stats": {
"properties": {
"index": {
@ -193,6 +217,9 @@
"terms_memory_in_bytes": {
"type": "long"
},
"points_memory_in_bytes": {
"type": "long"
},
"stored_fields_memory_in_bytes": {
"type": "long"
},
@ -324,6 +351,9 @@
"terms_memory_in_bytes": {
"type": "long"
},
"points_memory_in_bytes": {
"type": "long"
},
"stored_fields_memory_in_bytes": {
"type": "long"
},
@ -357,11 +387,7 @@
}
}
}
}
}
},
"cluster_stats": {
"properties": {
},
"cluster_stats": {
"properties": {
"nodes": {
@ -371,11 +397,7 @@
"type": "object"
}
}
}
}
},
"cluster_state": {
"properties": {
},
"cluster_state": {
"properties": {
"version": {
@ -391,31 +413,13 @@
"type": "keyword"
},
"nodes": {
"enabled": false
"type": "object"
},
"shards": {
"type": "object"
}
}
}
}
},
"node": {
"properties": {
"state_uuid": {
"type": "keyword"
},
"node": {
"properties": {
"id": {
"type": "keyword"
}
}
}
}
},
"node_stats": {
"properties": {
"node_stats": {
"properties": {
"node_id": {
@ -512,6 +516,9 @@
"terms_memory_in_bytes": {
"type": "long"
},
"points_memory_in_bytes": {
"type": "long"
},
"stored_fields_memory_in_bytes": {
"type": "long"
},
@ -546,6 +553,19 @@
},
"fs": {
"properties": {
"total": {
"properties": {
"total_in_bytes": {
"type": "long"
},
"free_in_bytes": {
"type": "long"
},
"available_in_bytes": {
"type": "long"
}
}
},
"data": {
"properties": {
"spins": {
@ -634,29 +654,166 @@
}
},
"process": {
"type": "object"
"properties": {
"open_file_descriptors": {
"type": "long"
},
"max_file_descriptors": {
"type": "long"
},
"cpu": {
"properties": {
"percent": {
"type": "half_float"
}
}
}
}
},
"jvm": {
"type": "object"
"properties": {
"mem": {
"properties": {
"heap_used_in_bytes": {
"type": "long"
},
"heap_used_percent": {
"type": "half_float"
},
"heap_max_in_bytes": {
"type": "long"
}
}
},
"gc": {
"properties": {
"collectors": {
"properties": {
"young": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
},
"old": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
},
"thread_pool": {
"type": "object"
"properties": {
"bulk": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"generic": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"get": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"index": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"maanagement": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"search": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
},
"watcher": {
"properties": {
"threads": {
"type": "integer"
},
"queue": {
"type": "integer"
},
"rejected": {
"type": "long"
}
}
}
}
}
}
}
}
},
"index_recovery": {
"properties": {
},
"index_recovery": {
"enabled": false
}
}
},
"shards": {
"properties": {
"state_uuid": {
"type": "keyword"
"type": "object"
},
"shard": {
"properties": {
@ -679,6 +836,58 @@
"type": "keyword"
}
}
},
"job_stats": {
"properties": {
"job_id": {
"type": "keyword"
},
"state": {
"type": "keyword"
},
"data_counts": {
"properties": {
"input_bytes": {
"type": "long"
},
"processed_record_count": {
"type": "long"
},
"empty_bucket_count": {
"type": "long"
},
"sparse_bucket_count": {
"type": "long"
},
"bucket_count": {
"type": "long"
},
"earliest_record_timestamp": {
"type": "date"
},
"latest_record_timestamp": {
"type": "date"
}
}
},
"model_size_stats": {
"properties": {
"model_bytes": {
"type": "long"
},
"bucket_allocation_failures_count": {
"type": "long"
}
}
},
"node": {
"properties": {
"id": {
"type": "keyword"
}
}
}
}
}
}
}

30
plugin/src/main/resources/monitoring-kibana.json
View File

@ -1,13 +1,14 @@
{
"index_patterns": ".monitoring-kibana-${monitoring.template.version}-*",
"version": 6000002,
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapping.single_type": false
"index.codec": "best_compression"
},
"mappings": {
"_default_": {
"doc": {
"dynamic": false,
"properties": {
"cluster_uuid": {
"type": "keyword"
@ -16,6 +17,9 @@
"type": "date",
"format": "date_time"
},
"type": {
"type": "keyword"
},
"source_node": {
"properties": {
"uuid": {
@ -32,27 +36,9 @@
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
}
}
},
"kibana_stats": {
"properties": {
},
"kibana_stats": {
"properties": {
"kibana": {

153
plugin/src/main/resources/monitoring-logstash.json
View File

@ -1,13 +1,14 @@
{
"index_patterns": ".monitoring-logstash-${monitoring.template.version}-*",
"version": 6000002,
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapping.single_type": false
"index.codec": "best_compression"
},
"mappings": {
"_default_": {
"doc": {
"dynamic": false,
"properties": {
"cluster_uuid": {
"type": "keyword"
@ -16,6 +17,9 @@
"type": "date",
"format": "date_time"
},
"type": {
"type": "keyword"
},
"source_node": {
"properties": {
"uuid": {
@ -32,27 +36,9 @@
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
}
}
},
"logstash_stats": {
"properties": {
},
"logstash_stats": {
"type": "object",
"properties": {
@ -245,7 +231,9 @@
"events_count": {
"type": "long"
},
"type": {"type": "keyword"}
"type": {
"type": "keyword"
}
}
},
"pipelines": {
@ -275,35 +263,57 @@
"events_count": {
"type": "long"
},
"type": {"type": "keyword"}
"type": {
"type": "keyword"
}
}
},
"components": {
"type": "nested",
"properties": {
"id": {"type": "keyword"},
"id": {
"type": "keyword"
},
"long_stat": {
"type": "nested",
"properties": {
"name": {"type": "keyword"},
"value": {"type": "long"},
"metric_type": { "type": "keyword" }
"name": {
"type": "keyword"
},
"value": {
"type": "long"
},
"metric_type": {
"type": "keyword"
}
}
},
"double_stat": {
"type": "nested",
"properties": {
"name": {"type": "keyword"},
"value": {"type": "double"},
"metric_type": { "type": "keyword" }
"name": {
"type": "keyword"
},
"value": {
"type": "double"
},
"metric_type": {
"type": "keyword"
}
}
},
"string_stat": {
"type": "nested",
"properties": {
"name": {"type": "keyword"},
"value": {"type": "keyword"},
"metric_type": { "type": "keyword" }
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
},
"metric_type": {
"type": "keyword"
}
}
}
}
@ -327,57 +337,54 @@
"type": "integer"
}
}
}
}
},
"logstash_state": {
"properties": {
"timestamp": {
"type": "date"
},
"uuid": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"http_address": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"snapshot": {
"type": "boolean"
},
"status": {
"type": "keyword"
},
"pipeline": {
"logstash_state": {
"properties": {
"uuid": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"hash": {
"host": {
"type": "keyword"
},
"workers": {
"type": "short"
},
"batch_size": {
"type": "integer"
},
"format": {
"http_address": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"representation": {
"enabled": false
"snapshot": {
"type": "boolean"
},
"status": {
"type": "keyword"
},
"pipeline": {
"properties": {
"name": {
"type": "keyword"
},
"hash": {
"type": "keyword"
},
"workers": {
"type": "short"
},
"batch_size": {
"type": "integer"
},
"format": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"representation": {
"enabled": false
}
}
}
}
}

29
plugin/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json
View File

@ -2,12 +2,12 @@
"metadata": {
"name": "X-Pack Monitoring: Cluster Status (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-2",
"alert_index": ".monitoring-alerts-6",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "elasticsearch/indices",
"severity": 2100,
"type": "monitoring",
"version": "5.4.0",
"version_created": 6000002,
"watch": "${monitoring.watch.id}"
}
},
@ -24,7 +24,7 @@
"search": {
"request": {
"indices": [
".monitoring-es-2-*"
".monitoring-es-*"
],
"body": {
"size": 1,
@ -47,8 +47,19 @@
}
},
{
"term": {
"_type": "cluster_state"
"bool": {
"should": [
{
"term": {
"_type": "cluster_state"
}
},
{
"term": {
"type": "cluster_state"
}
}
]
}
}
]
@ -64,6 +75,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6",
".monitoring-alerts-2"
],
"body": {
@ -77,7 +89,10 @@
}
}
}
}
},
"sort": [
{ "timestamp": { "order": "desc" } }
]
}
}
}
@ -101,7 +116,7 @@
"actions": {
"trigger_alert": {
"index": {
"index": ".monitoring-alerts-2",
"index": ".monitoring-alerts-6",
"doc_type": "doc",
"doc_id": "${monitoring.watch.unique_id}"
}

30
plugin/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json
View File

@ -2,11 +2,12 @@
"metadata": {
"name": "X-Pack Monitoring: Elasticsearch Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-2",
"alert_index": ".monitoring-alerts-6",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "elasticsearch/nodes",
"severity": 1000,
"type": "monitoring",
"version_created": 6000002,
"watch": "${monitoring.watch.id}"
}
},
@ -23,7 +24,7 @@
"search": {
"request": {
"indices": [
".monitoring-es-2-*"
".monitoring-es-*"
],
"body": {
"size": 1,
@ -39,15 +40,26 @@
}
},
{
"term": {
"_type": "cluster_stats"
"bool": {
"should": [
{
"term": {
"_type": "cluster_stats"
}
},
{
"term": {
"type": "cluster_stats"
}
}
]
}
}
]
}
},
"sort": [
"timestamp"
{ "timestamp": { "order": "desc" } }
]
}
}
@ -59,6 +71,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6",
".monitoring-alerts-2"
],
"body": {
@ -72,7 +85,10 @@
}
}
}
}
},
"sort": [
{ "timestamp": { "order": "desc" } }
]
}
}
}
@ -94,7 +110,7 @@
"actions": {
"trigger_alert": {
"index": {
"index": ".monitoring-alerts-2",
"index": ".monitoring-alerts-6",
"doc_type": "doc",
"doc_id": "${monitoring.watch.unique_id}"
}

28
plugin/src/main/resources/monitoring/watches/kibana_version_mismatch.json
View File

@ -2,11 +2,12 @@
"metadata": {
"name": "X-Pack Monitoring: Kibana Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-2",
"alert_index": ".monitoring-alerts-6",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "kibana/instances",
"severity": 1000,
"type": "monitoring",
"version_created": 6000002,
"watch": "${monitoring.watch.id}"
}
},
@ -23,7 +24,7 @@
"search": {
"request": {
"indices": [
".monitoring-kibana-2-*"
".monitoring-kibana-*"
],
"body": {
"size": 0,
@ -43,8 +44,19 @@
}
},
{
"term": {
"_type": "kibana_stats"
"bool": {
"should": [
{
"term": {
"_type": "kibana_stats"
}
},
{
"term": {
"type": "kibana_stats"
}
}
]
}
}
]
@ -86,6 +98,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6",
".monitoring-alerts-2"
],
"body": {
@ -99,7 +112,10 @@
}
}
}
}
},
"sort": [
{ "timestamp": { "order": "desc" } }
]
}
}
}
@ -121,7 +137,7 @@
"actions": {
"trigger_alert": {
"index": {
"index": ".monitoring-alerts-2",
"index": ".monitoring-alerts-6",
"doc_type": "doc",
"doc_id": "${monitoring.watch.unique_id}"
}

28
plugin/src/main/resources/monitoring/watches/logstash_version_mismatch.json
View File

@ -2,11 +2,12 @@
"metadata": {
"name": "X-Pack Monitoring: Logstash Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-2",
"alert_index": ".monitoring-alerts-6",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "logstash/instances",
"severity": 1000,
"type": "monitoring",
"version_created": 6000002,
"watch": "${monitoring.watch.id}"
}
},
@ -23,7 +24,7 @@
"search": {
"request": {
"indices": [
".monitoring-logstash-2-*"
".monitoring-logstash-*"
],
"body": {
"size": 0,
@ -43,8 +44,19 @@
}
},
{
"term": {
"_type": "logstash_stats"
"bool": {
"should": [
{
"term": {
"_type": "logstash_stats"
}
},
{
"term": {
"type": "logstash_stats"
}
}
]
}
}
]
@ -86,6 +98,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6",
".monitoring-alerts-2"
],
"body": {
@ -99,7 +112,10 @@
}
}
}
}
},
"sort": [
{ "timestamp": { "order": "desc" } }
]
}
}
}
@ -121,7 +137,7 @@
"actions": {
"trigger_alert": {
"index": {
"index": ".monitoring-alerts-2",
"index": ".monitoring-alerts-6",
"doc_type": "doc",
"doc_id": "${monitoring.watch.unique_id}"
}

9
plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringSettingsIntegTests.java
View File

@ -31,6 +31,7 @@ public class MonitoringSettingsIntegTests extends MonitoringIntegTestCase {
private final String[] indices = randomStringArray();
private final TimeValue clusterStateTimeout = newRandomTimeValue();
private final TimeValue clusterStatsTimeout = newRandomTimeValue();
private final TimeValue jobStatsTimeout = newRandomTimeValue();
private final TimeValue recoveryTimeout = newRandomTimeValue();
private final Boolean recoveryActiveOnly = randomBoolean();
@ -58,19 +59,20 @@ public class MonitoringSettingsIntegTests extends MonitoringIntegTestCase {
.putArray(MonitoringSettings.INDICES.getKey(), indices)
.put(MonitoringSettings.CLUSTER_STATE_TIMEOUT.getKey(), clusterStateTimeout)
.put(MonitoringSettings.CLUSTER_STATS_TIMEOUT.getKey(), clusterStatsTimeout)
.put(MonitoringSettings.JOB_STATS_TIMEOUT.getKey(), jobStatsTimeout)
.put(MonitoringSettings.INDEX_RECOVERY_TIMEOUT.getKey(), recoveryTimeout)
.put(MonitoringSettings.INDEX_RECOVERY_ACTIVE_ONLY.getKey(), recoveryActiveOnly)
.build();
}
public void testMonitoringSettings() throws Exception {
logger.info("--> testing monitoring settings service initialization");
for (final MonitoringSettings monitoringSettings : internalCluster().getInstances(MonitoringSettings.class)) {
assertThat(monitoringSettings.indexStatsTimeout().millis(), equalTo(indexStatsTimeout.millis()));
assertThat(monitoringSettings.indicesStatsTimeout().millis(), equalTo(indicesStatsTimeout.millis()));
assertArrayEquals(monitoringSettings.indices(), indices);
assertThat(monitoringSettings.clusterStateTimeout().millis(), equalTo(clusterStateTimeout.millis()));
assertThat(monitoringSettings.clusterStatsTimeout().millis(), equalTo(clusterStatsTimeout.millis()));
assertThat(monitoringSettings.jobStatsTimeout().millis(), equalTo(jobStatsTimeout.millis()));
assertThat(monitoringSettings.recoveryTimeout().millis(), equalTo(recoveryTimeout.millis()));
assertThat(monitoringSettings.recoveryActiveOnly(), equalTo(recoveryActiveOnly));
}
@ -90,7 +92,8 @@ public class MonitoringSettingsIntegTests extends MonitoringIntegTestCase {
MonitoringSettings.INDICES_STATS_TIMEOUT,
MonitoringSettings.INDEX_RECOVERY_ACTIVE_ONLY,
MonitoringSettings.CLUSTER_STATE_TIMEOUT,
MonitoringSettings.CLUSTER_STATS_TIMEOUT};
MonitoringSettings.CLUSTER_STATS_TIMEOUT,
MonitoringSettings.JOB_STATS_TIMEOUT};
for (Setting<?> setting : monitoringSettings) {
if (setting.isDynamic()) {
Object updated = null;
@ -130,6 +133,8 @@ public class MonitoringSettingsIntegTests extends MonitoringIntegTestCase {
assertEquals(monitoringSettings1.indicesStatsTimeout(), setting.get(updatedSettings));
} else if (setting == MonitoringSettings.CLUSTER_STATS_TIMEOUT) {
assertEquals(monitoringSettings1.clusterStatsTimeout(), setting.get(updatedSettings));
} else if (setting == MonitoringSettings.JOB_STATS_TIMEOUT) {
assertEquals(monitoringSettings1.jobStatsTimeout(), setting.get(updatedSettings));
} else if (setting == MonitoringSettings.CLUSTER_STATE_TIMEOUT) {
assertEquals(monitoringSettings1.clusterStateTimeout(), setting.get(updatedSettings));
} else if (setting == MonitoringSettings.INDEX_RECOVERY_TIMEOUT) {

23
plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
View File

@ -112,7 +112,6 @@ public class MonitoringBulkRequestTests extends ESTestCase {
@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1353")
public void testAddMultipleDocs() throws Exception {
final int nbDocs = randomIntBetween(3, 20);
final MonitoringIndex[] indices = new MonitoringIndex[nbDocs];
final String[] types = new String[nbDocs];
final XContentType xContentType = XContentType.JSON;
int dataCount = 0;
@ -123,15 +122,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
for (i = 0; i < nbDocs; i++) {
builder.startObject().startObject("index");
if (rarely()) {
indices[i] = MonitoringIndex.DATA;
builder.field("_index", "_data");
dataCount++;
} else {
indices[i] = MonitoringIndex.TIMESTAMPED;
if (rarely()) {
builder.field("_index", "");
}
builder.field("_index", "");
}
if (randomBoolean()) {
types[i] = randomAlphaOfLength(5);
@ -155,15 +146,11 @@ public class MonitoringBulkRequestTests extends ESTestCase {
i = 0;
for (final MonitoringBulkDoc doc : request.getDocs()) {
// we actively ignore IndexRequests for the _data index; this provides BWC to older versions
if (indices[i] != MonitoringIndex.DATA) {
final String expectedType = types[i] != null ? types[i] : defaultType;
final String expectedType = types[i] != null ? types[i] : defaultType;
assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId));
assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion));
assertThat(doc.getIndex(), sameInstance(MonitoringIndex.TIMESTAMPED));
assertThat(doc.getType(), equalTo(expectedType));
}
assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId));
assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion));
assertThat(doc.getType(), equalTo(expectedType));
++i;
}

16
plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringIndexTests.java
View File

@ -17,13 +17,13 @@ import java.io.IOException;
public class MonitoringIndexTests extends ESTestCase {
public void testDataMatchesIndexName() {
assertTrue(MonitoringIndex.DATA.matchesIndexName("_data"));
assertFalse(MonitoringIndex.DATA.matchesIndexName("_DATA"));
assertFalse(MonitoringIndex.DATA.matchesIndexName("_dAtA"));
assertFalse(MonitoringIndex.DATA.matchesIndexName("_data "));
assertFalse(MonitoringIndex.DATA.matchesIndexName(" _data "));
assertFalse(MonitoringIndex.DATA.matchesIndexName(""));
assertFalse(MonitoringIndex.DATA.matchesIndexName(null));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName("_data"));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName("_DATA"));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName("_dAtA"));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName("_data "));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName(" _data "));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName(""));
assertFalse(MonitoringIndex.IGNORED_DATA.matchesIndexName(null));
}
public void testTimestampMatchesIndexName() {
@ -34,7 +34,7 @@ public class MonitoringIndexTests extends ESTestCase {
}
public void testFrom() {
assertSame(MonitoringIndex.DATA, MonitoringIndex.from("_data"));
assertSame(MonitoringIndex.IGNORED_DATA, MonitoringIndex.from("_data"));
assertSame(MonitoringIndex.TIMESTAMPED, MonitoringIndex.from(""));
assertSame(MonitoringIndex.TIMESTAMPED, MonitoringIndex.from(null));
}

33
plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java
View File

@ -22,8 +22,6 @@ import org.joda.time.DateTimeZone;
import java.util.Locale;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.VersionUtils.randomVersion;
import static org.elasticsearch.xpack.monitoring.action.MonitoringBulkDocTests.newRandomSourceNode;
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTestCase {
@ -78,13 +76,12 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
// Won't be deleted
createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1));
createDataIndex(now().minusDays(10), "0");
createDataIndex(now().minusDays(10), "1");
assertIndicesCount(4);
createDataIndex(now().minusDays(10));
assertIndicesCount(3);
CleanerService.Listener listener = getListener();
listener.onCleanUpIndices(days(0));
assertIndicesCount(3);
assertIndicesCount(2);
}
public void testIgnoreCurrentTimestampedIndex() throws Exception {
@ -185,31 +182,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
throw new IllegalStateException("unable to find listener");
}
private MonitoringDoc randomMonitoringDoc() {
String monitoringId = randomFrom(MonitoredSystem.values()).getSystem();
String monitoringVersion = randomVersion(random()).toString();
String type = randomFrom("type1", "type2", "type3");
String id = randomBoolean() ? randomAlphaOfLength(3) : null;
String clusterUUID = randomBoolean() ? randomAlphaOfLength(5) : null;
long timestamp = randomBoolean() ? randomNonNegativeLong() : 0L;
MonitoringDoc.Node sourceNode = randomBoolean() ? newRandomSourceNode() : null;
return new MonitoringDoc(monitoringId, monitoringVersion, type, id, clusterUUID,
timestamp, sourceNode);
}
/**
* Creates a monitoring data index in a given version.
* Creates a monitoring data index from an earlier version (from when we used to have them).
*/
protected void createDataIndex(DateTime creationDate) {
createDataIndex(creationDate, MonitoringTemplateUtils.TEMPLATE_VERSION);
}
/**
* Creates a monitoring data index in a given version.
*/
protected void createDataIndex(DateTime creationDate, String version) {
createIndex(new MockDataIndexNameResolver(version).index(randomMonitoringDoc()), creationDate);
createIndex(".monitoring-data-2", creationDate);
}
/**

34
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java
View File

@ -16,19 +16,14 @@ import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.isEmptyOrNullString;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
@ -79,7 +74,6 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
}
}
flush();
refresh();
for (int i = 0; i < nbIndices; i++) {
@ -124,13 +118,10 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
}
private void assertMonitoringDocs(Collection<MonitoringDoc> results, final int nbShards) {
assertThat("expecting 1 document for cluster state and 1 document per node", results, hasSize(1 + internalCluster().size()));
assertThat("expecting 1 document for cluster state", results, hasSize(1));
final ClusterState clusterState = securedClient().admin().cluster().prepareState().get().getState();
final String clusterUUID = clusterState.getMetaData().clusterUUID();
final String stateUUID = clusterState.stateUUID();
List<ClusterStateNodeMonitoringDoc> clusterStateNodes = new ArrayList<>();
for (MonitoringDoc doc : results) {
assertThat(doc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
@ -138,37 +129,16 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
assertThat(doc.getClusterUUID(), equalTo(clusterUUID));
assertThat(doc.getTimestamp(), greaterThan(0L));
assertThat(doc.getSourceNode(), notNullValue());
assertThat(doc, anyOf(instanceOf(ClusterStateMonitoringDoc.class), instanceOf(ClusterStateNodeMonitoringDoc.class)));
assertThat(doc, instanceOf(ClusterStateMonitoringDoc.class));
if (doc instanceof ClusterStateMonitoringDoc) {
ClusterStateMonitoringDoc clusterStateMonitoringDoc = (ClusterStateMonitoringDoc) doc;
assertThat(clusterStateMonitoringDoc.getClusterState().getRoutingTable().allShards(), hasSize(nbShards));
assertThat(clusterStateMonitoringDoc.getClusterState().getNodes().getSize(), equalTo(internalCluster().size()));
} else if (doc instanceof ClusterStateNodeMonitoringDoc) {
ClusterStateNodeMonitoringDoc clusterStateNodeMonitoringDoc = (ClusterStateNodeMonitoringDoc) doc;
assertThat(clusterStateNodeMonitoringDoc.getStateUUID(), equalTo(stateUUID));
assertThat(clusterStateNodeMonitoringDoc.getNodeId(), not(isEmptyOrNullString()));
clusterStateNodes.add(clusterStateNodeMonitoringDoc);
} else {
fail("unknown monitoring document type " + doc);
}
}
assertThat(clusterStateNodes, hasSize(internalCluster().size()));
for (final String nodeName : internalCluster().getNodeNames()) {
final String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
boolean found = false;
for (ClusterStateNodeMonitoringDoc doc : clusterStateNodes) {
if (nodeId.equals(doc.getNodeId())) {
found = true;
break;
}
}
assertTrue("Could not find node name [" + nodeName + "]", found);
}
}
}

32
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
import java.util.Collection;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
@ -15,30 +14,24 @@ import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.Collector;
import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.notNullValue;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
public void testClusterStatsCollector() throws Exception {
Collection<MonitoringDoc> results = newClusterStatsCollector().doCollect();
assertThat(results, hasSize(2));
assertThat(results, hasSize(1));
// Check cluster info document
MonitoringDoc monitoringDoc = results.stream().filter(o -> o instanceof ClusterInfoMonitoringDoc).findFirst().get();
assertNotNull(monitoringDoc);
assertThat(monitoringDoc, instanceOf(ClusterInfoMonitoringDoc.class));
// validate the document
ClusterStatsMonitoringDoc clusterInfoMonitoringDoc =
(ClusterStatsMonitoringDoc)results.stream().filter(o -> o instanceof ClusterStatsMonitoringDoc).findFirst().get();
ClusterInfoMonitoringDoc clusterInfoMonitoringDoc = (ClusterInfoMonitoringDoc) monitoringDoc;
assertThat(clusterInfoMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
assertThat(clusterInfoMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
assertThat(clusterInfoMonitoringDoc.getClusterUUID(),
@ -56,23 +49,6 @@ public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
assertNotNull(clusterInfoMonitoringDoc.getClusterStats());
assertThat(clusterInfoMonitoringDoc.getClusterStats().getNodesStats().getCounts().getTotal(),
equalTo(internalCluster().getNodeNames().length));
// Check cluster stats document
monitoringDoc = results.stream().filter(o -> o instanceof ClusterStatsMonitoringDoc).findFirst().get();
assertNotNull(monitoringDoc);
assertThat(monitoringDoc, instanceOf(ClusterStatsMonitoringDoc.class));
ClusterStatsMonitoringDoc clusterStatsMonitoringDoc = (ClusterStatsMonitoringDoc) monitoringDoc;
assertThat(clusterStatsMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
assertThat(clusterStatsMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
assertThat(clusterStatsMonitoringDoc.getClusterUUID(),
equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
assertThat(clusterStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
assertThat(clusterStatsMonitoringDoc.getSourceNode(), notNullValue());
assertNotNull(clusterStatsMonitoringDoc.getClusterStats());
assertThat(clusterStatsMonitoringDoc.getClusterStats().getNodesStats().getCounts().getTotal(),
equalTo(internalCluster().getNodeNames().length));
}
private ClusterStatsCollector newClusterStatsCollector() {

204
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java Normal file
View File

@ -0,0 +1,204 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.ml;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Request;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.client.MachineLearningClient;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.security.InternalClient;
import java.util.ArrayList;
import java.util.List;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Tests {@link JobStatsCollector}.
*/
public class JobStatsCollectorTests extends ESTestCase {
private final ClusterService clusterService = mock(ClusterService.class);
private final ClusterState clusterState = mock(ClusterState.class);
private final DiscoveryNodes nodes = mock(DiscoveryNodes.class);
private final MonitoringSettings monitoringSettings = mock(MonitoringSettings.class);
private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
private final InternalClient client = mock(InternalClient.class);
public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
final Settings settings = randomFrom(mlEnabledSettings(), mlDisabledSettings());
final boolean mlAllowed = randomBoolean();
// this controls the blockage
when(licenseState.isMonitoringAllowed()).thenReturn(false);
when(licenseState.isMachineLearningAllowed()).thenReturn(mlAllowed);
final JobStatsCollector collector = new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client);
assertThat(collector.shouldCollect(), is(false));
verify(licenseState).isMonitoringAllowed();
}
public void testShouldCollectReturnsFalseIfNotMaster() {
// regardless of ML being enabled
final Settings settings = randomFrom(mlEnabledSettings(), mlDisabledSettings());
when(licenseState.isMonitoringAllowed()).thenReturn(randomBoolean());
when(licenseState.isMachineLearningAllowed()).thenReturn(randomBoolean());
// this controls the blockage
whenLocalNodeElectedMaster(false);
final JobStatsCollector collector = new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client);
assertThat(collector.shouldCollect(), is(false));
verify(licenseState).isMonitoringAllowed();
}
public void testShouldCollectReturnsFalseIfMLIsDisabled() {
// this is controls the blockage
final Settings settings = mlDisabledSettings();
when(licenseState.isMonitoringAllowed()).thenReturn(randomBoolean());
when(licenseState.isMachineLearningAllowed()).thenReturn(randomBoolean());
whenLocalNodeElectedMaster(randomBoolean());
final JobStatsCollector collector = new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client);
assertThat(collector.shouldCollect(), is(false));
verify(licenseState).isMonitoringAllowed();
}
public void testShouldCollectReturnsFalseIfMLIsNotAllowed() {
final Settings settings = randomFrom(mlEnabledSettings(), mlDisabledSettings());
when(licenseState.isMonitoringAllowed()).thenReturn(randomBoolean());
// this is controls the blockage
when(licenseState.isMachineLearningAllowed()).thenReturn(false);
whenLocalNodeElectedMaster(randomBoolean());
final JobStatsCollector collector = new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client);
assertThat(collector.shouldCollect(), is(false));
verify(licenseState).isMonitoringAllowed();
}
public void testShouldCollectReturnsTrue() {
final Settings settings = mlEnabledSettings();
when(licenseState.isMonitoringAllowed()).thenReturn(true);
when(licenseState.isMachineLearningAllowed()).thenReturn(true);
whenLocalNodeElectedMaster(true);
final JobStatsCollector collector = new JobStatsCollector(settings, clusterService, monitoringSettings, licenseState, client);
assertThat(collector.shouldCollect(), is(true));
verify(licenseState).isMonitoringAllowed();
}
public void testDoCollect() throws Exception {
final TimeValue timeout = mock(TimeValue.class);
final MetaData metaData = mock(MetaData.class);
final String clusterUuid = randomAlphaOfLength(5);
final String nodeUuid = randomAlphaOfLength(5);
final DiscoveryNode localNode = localNode(nodeUuid);
final MachineLearningClient client = mock(MachineLearningClient.class);
when(monitoringSettings.jobStatsTimeout()).thenReturn(timeout);
when(clusterService.state()).thenReturn(clusterState);
when(clusterState.metaData()).thenReturn(metaData);
when(metaData.clusterUUID()).thenReturn(clusterUuid);
when(clusterService.localNode()).thenReturn(localNode);
final JobStatsCollector collector =
new JobStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
final List<JobStats> jobStats = mockJobStats();
@SuppressWarnings("unchecked")
final ActionFuture<Response> future = (ActionFuture<Response>)mock(ActionFuture.class);
final Response response = new Response(new QueryPage<>(jobStats, jobStats.size(), Job.RESULTS_FIELD));
when(client.getJobsStats(eq(new Request(Job.ALL)))).thenReturn(future);
when(future.actionGet(timeout)).thenReturn(response);
final List<MonitoringDoc> monitoringDocs = collector.doCollect();
assertThat(monitoringDocs, hasSize(jobStats.size()));
for (int i = 0; i < monitoringDocs.size(); ++i) {
final JobStatsMonitoringDoc jobStatsMonitoringDoc = (JobStatsMonitoringDoc)monitoringDocs.get(i);
final JobStats jobStat = jobStats.get(i);
assertThat(jobStatsMonitoringDoc.getType(), is(JobStatsMonitoringDoc.TYPE));
assertThat(jobStatsMonitoringDoc.getSourceNode(), notNullValue());
assertThat(jobStatsMonitoringDoc.getSourceNode().getUUID(), is(nodeUuid));
assertThat(jobStatsMonitoringDoc.getClusterUUID(), is(clusterUuid));
assertThat(jobStatsMonitoringDoc.getJobStats(), is(jobStat));
}
}
private List<JobStats> mockJobStats() {
final int jobs = randomIntBetween(1, 5);
final List<JobStats> jobStats = new ArrayList<>(jobs);
for (int i = 0; i < jobs; ++i) {
jobStats.add(mock(JobStats.class));
}
return jobStats;
}
private DiscoveryNode localNode(final String uuid) {
return new DiscoveryNode(uuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
}
private void whenLocalNodeElectedMaster(final boolean electedMaster) {
when(clusterService.state()).thenReturn(clusterState);
when(clusterState.nodes()).thenReturn(nodes);
when(nodes.isLocalNodeElectedMaster()).thenReturn(electedMaster);
}
private Settings mlEnabledSettings() {
// since it's the default, we want to ensure we test both with/without it
return randomBoolean() ? Settings.EMPTY : Settings.builder().put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true).build();
}
private Settings mlDisabledSettings() {
return Settings.builder().put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false).build();
}
}

51
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.collector.ml;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Mockito.mock;
/**
* Tests {@link JobStatsMonitoringDocTests}.
*/
public class JobStatsMonitoringDocTests extends ESTestCase {
private final String clusterUuid = randomAlphaOfLength(5);
private final long timestamp = System.currentTimeMillis();
private final String nodeUuid = randomAlphaOfLength(5);
private final DiscoveryNode node =
new DiscoveryNode(nodeUuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
private final JobStats jobStats = mock(JobStats.class);
private final JobStatsMonitoringDoc doc = new JobStatsMonitoringDoc(clusterUuid, timestamp, node, jobStats);
public void testConstructorJobStatsMustNotBeNull() {
expectThrows(NullPointerException.class,
() -> new JobStatsMonitoringDoc(clusterUuid, timestamp, node, null));
}
public void testConstructor() {
assertThat(doc.getMonitoringId(), is(MonitoredSystem.ES.getSystem()));
assertThat(doc.getMonitoringVersion(), is(Version.CURRENT.toString()));
assertThat(doc.getType(), is(JobStatsMonitoringDoc.TYPE));
assertThat(doc.getId(), nullValue());
assertThat(doc.getClusterUUID(), is(clusterUuid));
assertThat(doc.getTimestamp(), is(timestamp));
assertThat(doc.getSourceNode(), notNullValue());
assertThat(doc.getSourceNode().getUUID(), is(nodeUuid));
assertThat(doc.getJobStats(), is(jobStats));
}
}

1
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java
View File

@ -34,7 +34,6 @@ public class NodeStatsCollectorTests extends AbstractCollectorTestCase {
public void testNodeStatsCollector() throws Exception {
String[] nodes = internalCluster().getNodeNames();
for (String node : nodes) {
logger.info("--> collecting node stats on node [{}]", node);
Collection<MonitoringDoc> results = newNodeStatsCollector(node).doCollect();
assertThat(results, hasSize(1));

10
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java
View File

@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
@ -26,7 +25,6 @@ import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
import org.elasticsearch.xpack.security.InternalClient;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -212,14 +210,6 @@ public class ExportersTests extends ESTestCase {
assertThat(settings, hasEntry("_name1.foo", "bar"));
}
public void testEmptyPipeline() throws IOException {
String json = Exporter.emptyPipeline(XContentType.JSON).string();
// ensure the description starts with the API version
assertThat(json, containsString("\"description\":\"" + MonitoringTemplateUtils.TEMPLATE_VERSION + ":"));
assertThat(json, containsString("\"processors\":[]"));
}
public void testExporterBlocksOnClusterState() {
when(state.version()).thenReturn(ClusterState.UNKNOWN_VERSION);

58
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtilsTests.java
View File

@ -5,31 +5,83 @@
*/
package org.elasticsearch.xpack.monitoring.exporter;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.oldTemplateName;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.templateName;
import static org.elasticsearch.xpack.template.TemplateUtilsTests.assertTemplate;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
public class MonitoringTemplateUtilsTests extends ESTestCase {
public void testTemplateName() {
assertThat(templateName("abc"), equalTo(".monitoring-abc"));
assertThat(templateName("XYZ"), equalTo(".monitoring-XYZ"));
assertThat(templateName("es"), equalTo(".monitoring-es"));
assertThat(templateName("kibana"), equalTo(".monitoring-kibana"));
assertThat(templateName("logstash"), equalTo(".monitoring-logstash"));
assertThat(templateName("beats"), equalTo(".monitoring-beats"));
}
public void testOldTemplateName() {
assertThat(oldTemplateName("abc"), equalTo(".monitoring-abc-" + OLD_TEMPLATE_VERSION));
assertThat(oldTemplateName("XYZ"), equalTo(".monitoring-XYZ-" + OLD_TEMPLATE_VERSION));
assertThat(oldTemplateName("es"), equalTo(".monitoring-es-" + OLD_TEMPLATE_VERSION));
assertThat(oldTemplateName("kibana"), equalTo(".monitoring-kibana-" + OLD_TEMPLATE_VERSION));
assertThat(oldTemplateName("logstash"), equalTo(".monitoring-logstash-" + OLD_TEMPLATE_VERSION));
assertThat(oldTemplateName("beats"), equalTo(".monitoring-beats-" + OLD_TEMPLATE_VERSION));
}
public void testLoadTemplate() throws IOException {
String source = MonitoringTemplateUtils.loadTemplate("test");
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
assertTemplate(source, equalTo("{\n" +
" \"index_patterns\": \".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "\",\n" +
" \"index_patterns\": \".monitoring-data-" + TEMPLATE_VERSION + "\",\n" +
" \"mappings\": {\n" +
" \"type_1\": {\n" +
" \"doc\": {\n" +
" \"_meta\": {\n" +
" \"template.version\": \"" + MonitoringTemplateUtils.TEMPLATE_VERSION + "\"\n" +
" \"template.version\": \"" + TEMPLATE_VERSION + "\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}\n"));
}
public void testCreateEmptyTemplate() throws IOException {
final String id = randomFrom(OLD_TEMPLATE_IDS);
final String json = MonitoringTemplateUtils.createEmptyTemplate(id);
// ensure that the index is created with the proper ID
assertThat(json, containsString("\".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION + "*\""));
assertThat(json, containsString("\"version\":" + LAST_UPDATED_VERSION));
}
public void testPipelineName() {
assertThat(pipelineName("aBc123"), equalTo("xpack_monitoring_aBc123"));
assertThat(pipelineName(TEMPLATE_VERSION), equalTo("xpack_monitoring_" + TEMPLATE_VERSION));
assertThat(pipelineName(OLD_TEMPLATE_VERSION), equalTo("xpack_monitoring_" + OLD_TEMPLATE_VERSION));
}
public void testEmptyPipeline() throws IOException {
final String json = MonitoringTemplateUtils.emptyPipeline(XContentType.JSON).string();
// ensure the description contains the API version
assertThat(json, containsString("Monitoring API version " + MonitoringTemplateUtils.TEMPLATE_VERSION));
assertThat(json, containsString("\"processors\":[]"));
assertThat(json, containsString("\"version\":" + LAST_UPDATED_VERSION));
}
}

104
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java
View File

@ -9,6 +9,9 @@ import java.util.HashMap;
import org.apache.http.HttpEntity;
import org.apache.http.RequestLine;
import org.apache.http.StatusLine;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.Version;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
@ -187,19 +190,43 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
assertThat(parameters.isEmpty(), is(true));
}
protected void assertVersionParameters(final PublishableHttpResource resource) {
final Map<String, String> parameters = new HashMap<>(resource.getParameters());
if (masterTimeout != null) {
assertThat(parameters.remove("master_timeout"), is(masterTimeout.toString()));
}
assertThat(parameters.remove("filter_path"), is("*.version"));
assertThat(parameters.isEmpty(), is(true));
}
protected void doCheckWithStatusCode(final PublishableHttpResource resource, final String resourceBasePath, final String resourceName,
final RestStatus status,
final CheckResponse expected)
throws IOException {
doCheckWithStatusCode(resource, resourceBasePath, resourceName, status, GET_EXISTS, GET_DOES_NOT_EXIST, expected);
doCheckWithStatusCode(resource, resourceBasePath, resourceName, status, expected, null);
}
protected void doCheckWithStatusCode(final PublishableHttpResource resource, final String resourceBasePath, final String resourceName,
final RestStatus status, final CheckResponse expected, final HttpEntity entity)
throws IOException {
doCheckWithStatusCode(resource, resourceBasePath, resourceName, status, GET_EXISTS, GET_DOES_NOT_EXIST, expected, entity);
}
protected void doCheckWithStatusCode(final PublishableHttpResource resource, final String resourceBasePath, final String resourceName,
final RestStatus status, final Set<Integer> exists, final Set<Integer> doesNotExist,
final CheckResponse expected)
throws IOException {
doCheckWithStatusCode(resource, resourceBasePath, resourceName, status, exists, doesNotExist, expected, null);
}
protected void doCheckWithStatusCode(final PublishableHttpResource resource, final String resourceBasePath, final String resourceName,
final RestStatus status, final Set<Integer> exists, final Set<Integer> doesNotExist,
final CheckResponse expected, final HttpEntity entity)
throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final Response response = response("GET", endpoint, status);
final Response response = response("GET", endpoint, status, entity);
doCheckWithStatusCode(resource, getParameters(resource.getParameters(), exists, doesNotExist), endpoint, expected, response);
}
@ -329,4 +356,77 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
return parametersWithIgnore;
}
protected HttpEntity entityForResource(final CheckResponse expected, final String resourceName, final int minimumVersion) {
HttpEntity entity = null;
switch (expected) {
// the version check is what is expected to cause it to be replaced
case DOES_NOT_EXIST:
final int olderVersion = minimumVersion - randomIntBetween(1, 10000);
entity = randomFrom(
new StringEntity("{}", ContentType.APPLICATION_JSON),
new StringEntity("{\"" + resourceName + "\":{}}", ContentType.APPLICATION_JSON),
new StringEntity("{\"" + resourceName + "\":{\"version\":\"123\"}}", ContentType.APPLICATION_JSON),
new StringEntity("{\"" + resourceName + "\":{\"version\":" + olderVersion + "}}", ContentType.APPLICATION_JSON)
);
break;
// the version is there, and it's >= to what we expect
case EXISTS:
final int version = randomFrom(
Math.max(minimumVersion, Version.CURRENT.id),
minimumVersion + randomIntBetween(1, 100000)
);
entity = new StringEntity("{\"" + resourceName + "\":{\"version\":" + version + "}}", ContentType.APPLICATION_JSON);
break;
// malformed
case ERROR:
entity = randomFrom(
new StringEntity("{", ContentType.APPLICATION_JSON),
new StringEntity("{\"" + resourceName + "\":\"not an object\"}", ContentType.APPLICATION_JSON)
);
break;
default:
fail("Unhandled/unknown CheckResponse");
}
return entity;
}
protected HttpEntity entityForClusterAlert(final CheckResponse expected, final int minimumVersion) {
HttpEntity entity = null;
switch (expected) {
// the version check is what is expected to cause it to be replaced
case DOES_NOT_EXIST:
final int olderVersion = minimumVersion - randomIntBetween(1, 10000);
entity = randomFrom(
new StringEntity("{}", ContentType.APPLICATION_JSON),
new StringEntity("{\"metadata\":{}}", ContentType.APPLICATION_JSON),
new StringEntity("{\"metadata\":{\"xpack\":{\"version_created\":\"123\"}}}", ContentType.APPLICATION_JSON),
new StringEntity("{\"metadata\":{\"xpack\":{\"version_created\":" + olderVersion + "}}}}", ContentType.APPLICATION_JSON)
);
break;
// the version is there and it's exactly what we specify
case EXISTS:
entity = new StringEntity("{\"metadata\":{\"xpack\":{\"version_created\":" +
minimumVersion + "}}}", ContentType.APPLICATION_JSON);
break;
// malformed
case ERROR:
entity = randomFrom(
new StringEntity("{", ContentType.APPLICATION_JSON),
new StringEntity("{\"\"metadata\":\"not an object\"}", ContentType.APPLICATION_JSON),
new StringEntity("{\"\"metadata\":{\"xpack\":\"not an object\"}}", ContentType.APPLICATION_JSON)
);
break;
default:
fail("Unhandled/unknown CheckResponse");
}
return entity;
}
}

81
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResourceTests.java
View File

@ -12,8 +12,13 @@ import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.Version;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.mock;
@ -27,6 +32,7 @@ public class ClusterAlertHttpResourceTests extends AbstractPublishableHttpResour
private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
private final String watchId = randomFrom(ClusterAlertsUtil.WATCH_IDS);
private final String watchValue = "{\"totally-valid\":{}}";
private final int minimumVersion = randomFrom(ClusterAlertsUtil.LAST_UPDATED_VERSION, Version.CURRENT.id);
private final ClusterAlertHttpResource resource = new ClusterAlertHttpResource(owner, licenseState, () -> watchId, () -> watchValue);
@ -49,19 +55,40 @@ public class ClusterAlertHttpResourceTests extends AbstractPublishableHttpResour
public void testDoCheckGetWatchExists() throws IOException {
when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(true);
assertCheckExists(resource, "/_xpack/watcher/watch", watchId);
final HttpEntity entity = entityForClusterAlert(CheckResponse.EXISTS, minimumVersion);
doCheckWithStatusCode(resource, "/_xpack/watcher/watch", watchId, successfulCheckStatus(),
CheckResponse.EXISTS, entity);
}
public void testDoCheckGetWatchDoesNotExist() throws IOException {
when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(true);
assertCheckDoesNotExist(resource, "/_xpack/watcher/watch", watchId);
if (randomBoolean()) {
// it does not exist because it's literally not there
assertCheckDoesNotExist(resource, "/_xpack/watcher/watch", watchId);
} else {
// it does not exist because we need to replace it
final HttpEntity entity = entityForClusterAlert(CheckResponse.DOES_NOT_EXIST, minimumVersion);
doCheckWithStatusCode(resource, "/_xpack/watcher/watch", watchId, successfulCheckStatus(),
CheckResponse.DOES_NOT_EXIST, entity);
}
}
public void testDoCheckWithExceptionGetWatchError() throws IOException {
when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(true);
assertCheckWithException(resource, "/_xpack/watcher/watch", watchId);
if (randomBoolean()) {
// error because of a server error
assertCheckWithException(resource, "/_xpack/watcher/watch", watchId);
} else {
// error because of a malformed response
final HttpEntity entity = entityForClusterAlert(CheckResponse.ERROR, minimumVersion);
doCheckWithStatusCode(resource, "/_xpack/watcher/watch", watchId, successfulCheckStatus(),
CheckResponse.ERROR, entity);
}
}
public void testDoCheckAsDeleteWatchExists() throws IOException {
@ -88,10 +115,56 @@ public class ClusterAlertHttpResourceTests extends AbstractPublishableHttpResour
assertPublishWithException(resource, "/_xpack/watcher/watch", watchId, StringEntity.class);
}
public void testShouldReplaceClusterAlertRethrowsIOException() throws IOException {
final Response response = mock(Response.class);
final HttpEntity entity = mock(HttpEntity.class);
final XContent xContent = mock(XContent.class);
when(response.getEntity()).thenReturn(entity);
when(entity.getContent()).thenThrow(new IOException("TEST - expected"));
expectThrows(IOException.class, () -> resource.shouldReplaceClusterAlert(response, xContent, randomInt()));
}
public void testShouldReplaceClusterAlertThrowsExceptionForMalformedResponse() throws IOException {
final Response response = mock(Response.class);
final HttpEntity entity = entityForClusterAlert(CheckResponse.ERROR, randomInt());
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
expectThrows(RuntimeException.class, () -> resource.shouldReplaceClusterAlert(response, xContent, randomInt()));
}
public void testShouldReplaceClusterAlertReturnsTrueVersionIsNotExpected() throws IOException {
final int minimumVersion = randomInt();
final Response response = mock(Response.class);
final HttpEntity entity = entityForClusterAlert(CheckResponse.DOES_NOT_EXIST, minimumVersion);
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
assertThat(resource.shouldReplaceClusterAlert(response, xContent, minimumVersion), is(true));
}
public void testShouldReplaceCheckAlertChecksVersion() throws IOException {
final int minimumVersion = randomInt();
final int version = randomInt();
final boolean shouldReplace = version < minimumVersion;
final Response response = mock(Response.class);
final HttpEntity entity = entityForClusterAlert(CheckResponse.EXISTS, version);
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
assertThat(resource.shouldReplaceClusterAlert(response, xContent, minimumVersion), is(shouldReplace));
}
public void testParameters() {
final Map<String, String> parameters = new HashMap<>(resource.getParameters());
assertThat(parameters.remove("filter_path"), is("$NONE"));
assertThat(parameters.remove("filter_path"), is("metadata.xpack.version_created"));
assertThat(parameters.isEmpty(), is(true));
}

92
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/DataTypeMappingHttpResourceTests.java
View File

@ -1,92 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.exporter.http;
import org.apache.http.HttpEntity;
import org.elasticsearch.client.Response;
import org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse;
import org.apache.http.entity.StringEntity;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.DATA_INDEX;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Tests {@link DataTypeMappingHttpResource}.
*/
public class DataTypeMappingHttpResourceTests extends AbstractPublishableHttpResourceTestCase {
private final String typeName = "my_type";
private final DataTypeMappingHttpResource resource = new DataTypeMappingHttpResource(owner, masterTimeout, typeName);
public void testDoCheckTrueFor404() throws IOException {
// if the index is not there, then we don't need to manually add the type
doCheckWithStatusCode(resource, "/" + DATA_INDEX + "/_mapping", typeName, notFoundCheckStatus(), CheckResponse.EXISTS);
}
public void testDoCheckTrue() throws IOException {
final String endpoint = "/" + DATA_INDEX + "/_mapping/" + typeName;
// success does't mean it exists unless the mapping exists! it returns {} if the index exists, but the type does not
final Response response = response("GET", endpoint, successfulCheckStatus());
final HttpEntity responseEntity = mock(HttpEntity.class);
final long validMapping = randomIntBetween(3, Integer.MAX_VALUE);
when(response.getEntity()).thenReturn(responseEntity);
when(responseEntity.getContentLength()).thenReturn(validMapping);
doCheckWithStatusCode(resource, endpoint, CheckResponse.EXISTS, response);
verify(responseEntity).getContentLength();
}
public void testDoCheckFalse() throws IOException {
final String endpoint = "/" + DATA_INDEX + "/_mapping/" + typeName;
// success does't mean it exists unless the mapping exists! it returns {} if the index exists, but the type does not
final Response response = response("GET", endpoint, successfulCheckStatus());
final HttpEntity responseEntity = mock(HttpEntity.class);
final long invalidMapping = randomIntBetween(Integer.MIN_VALUE, 2);
when(response.getEntity()).thenReturn(responseEntity);
when(responseEntity.getContentLength()).thenReturn(invalidMapping);
doCheckWithStatusCode(resource, endpoint, CheckResponse.DOES_NOT_EXIST, response);
verify(responseEntity).getContentLength();
}
public void testDoCheckNullWithException() throws IOException {
assertCheckWithException(resource, "/" + DATA_INDEX + "/_mapping", typeName);
}
public void testDoPublishTrue() throws IOException {
assertPublishSucceeds(resource, "/" + DATA_INDEX + "/_mapping", typeName, StringEntity.class);
}
public void testDoPublishFalse() throws IOException {
assertPublishFails(resource, "/" + DATA_INDEX + "/_mapping", typeName, StringEntity.class);
}
public void testDoPublishFalseWithException() throws IOException {
assertPublishWithException(resource, "/" + DATA_INDEX + "/_mapping", typeName, StringEntity.class);
}
public void testParameters() {
final Map<String, String> parameters = resource.getParameters();
if (masterTimeout != null) {
assertThat(parameters.get("master_timeout"), is(masterTimeout.toString()));
}
assertThat(parameters.size(), is(masterTimeout == null ? 0 : 1));
}
}

540
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
View File

@ -5,12 +5,15 @@
*/
package org.elasticsearch.xpack.monitoring.exporter.http;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
@ -20,10 +23,13 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.env.Environment;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.http.MockRequest;
@ -33,13 +39,13 @@ import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc;
import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.Exporters;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
import org.elasticsearch.xpack.monitoring.resolver.bulk.MonitoringBulkTimestampedResolver;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.elasticsearch.xpack.ssl.SSLService;
import org.joda.time.format.DateTimeFormat;
import org.junit.After;
import org.junit.Before;
@ -52,13 +58,15 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.DATA_INDEX;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.FILTER_PATH_NONE;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.http.ClusterAlertHttpResource.CLUSTER_ALERT_VERSION_PARAMETERS;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.FILTER_PATH_RESOURCE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.http.WatcherExistsHttpResource.WATCHER_CHECK_PARAMETERS;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
@ -68,20 +76,21 @@ import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.startsWith;
@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
@ESIntegTestCase.ClusterScope(scope = Scope.TEST,
numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
public class HttpExporterIT extends MonitoringIntegTestCase {
private final boolean typeMappingsExistAlready = randomBoolean();
private final boolean templatesExistsAlready = randomBoolean();
private final boolean includeOldTemplates = randomBoolean();
private final boolean pipelineExistsAlready = randomBoolean();
private final boolean remoteClusterAllowsWatcher = randomBoolean();
private final boolean currentLicenseAllowsWatcher = true;
private final boolean watcherAlreadyExists = randomBoolean();
private final boolean bwcIndexesExist = randomBoolean();
private final boolean bwcAliasesExist = randomBoolean();
private final Environment environment = new Environment(Settings.builder().put("path.home", createTempDir()).build());
private MockWebServer webServer;
@ -92,7 +101,9 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
@After
public void stopWebServer() throws Exception {
webServer.close();
if (webServer != null) {
webServer.close();
}
}
@Override
@ -100,25 +111,37 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
return true;
}
public void testExport() throws Exception {
final Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer));
@Override
protected Settings nodeSettings(int nodeOrdinal) {
// we create and disable the exporter to avoid the cluster actually using it (thus speeding up tests)
// we make an exporter on demand per test
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.enabled", false)
.build();
}
internalCluster().startNode(builder);
public void testExport() throws Exception {
final Settings settings = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
.build();
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
final int nbDocs = randomIntBetween(1, 25);
export(newRandomMonitoringDocs(nbDocs));
export(settings, newRandomMonitoringDocs(nbDocs));
assertMonitorResources(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
assertBulk(webServer, nbDocs);
@ -134,27 +157,27 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
headers.put("X-Found-Cluster", new String[] { headerValue });
headers.put("Array-Check", array);
Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
Settings settings = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
.put("xpack.monitoring.exporters._http.headers.X-Cloud-Cluster", headerValue)
.put("xpack.monitoring.exporters._http.headers.X-Found-Cluster", headerValue)
.putArray("xpack.monitoring.exporters._http.headers.Array-Check", array);
internalCluster().startNode(builder);
.putArray("xpack.monitoring.exporters._http.headers.Array-Check", array)
.build();
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
final int nbDocs = randomIntBetween(1, 25);
export(newRandomMonitoringDocs(nbDocs));
export(settings, newRandomMonitoringDocs(nbDocs));
assertMonitorResources(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist,
headers, null);
@ -189,31 +212,33 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
basePath = "/" + basePath;
}
final Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
final Settings.Builder builder = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.put("xpack.monitoring.exporters._http.proxy.base_path", basePath + (randomBoolean() ? "/" : ""));
if (includeOldTemplates == false) {
builder.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", false);
}
if (useHeaders) {
builder.put("xpack.monitoring.exporters._http.headers.X-Cloud-Cluster", headerValue)
.put("xpack.monitoring.exporters._http.headers.X-Found-Cluster", headerValue)
.putArray("xpack.monitoring.exporters._http.headers.Array-Check", array);
}
internalCluster().startNode(builder);
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
enqueueResponse(200, "{\"errors\": false}");
final int nbDocs = randomIntBetween(1, 25);
export(newRandomMonitoringDocs(nbDocs));
export(builder.build(), newRandomMonitoringDocs(nbDocs));
assertMonitorResources(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist,
headers, basePath);
@ -221,38 +246,43 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
public void testHostChangeReChecksTemplate() throws Exception {
Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
Settings settings = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer));
internalCluster().startNode(builder);
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
.build();
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
enqueueResponse(200, "{\"errors\": false}");
export(Collections.singletonList(newRandomMonitoringDoc()));
export(settings, Collections.singletonList(newRandomMonitoringDoc()));
assertMonitorResources(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
assertBulk(webServer);
try (MockWebServer secondWebServer = createMockWebServer()) {
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Settings.builder().putArray("xpack.monitoring.exporters._http.host", getFormattedAddress(secondWebServer))));
String missingTemplate = null;
final Settings newSettings = Settings.builder()
.put(settings)
.putArray("xpack.monitoring.exporters._http.host", getFormattedAddress(secondWebServer))
.build();
enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT);
enqueueMappingTypeResponses(secondWebServer, !typeMappingsExistAlready);
// pretend that one of the templates is missing
for (Tuple<String, String> template : monitoringTemplates()) {
if (template.v1().contains(MonitoringBulkTimestampedResolver.Data.DATA)) {
enqueueResponse(secondWebServer, 200, "template [" + template.v1() + "] exists");
for (Tuple<String, String> template : monitoringTemplates(includeOldTemplates)) {
if (missingTemplate != null) {
enqueueResponse(secondWebServer, 200, "{\"" + template.v1() + "\":{\"version\":" + LAST_UPDATED_VERSION + "}}");
} else {
missingTemplate = template.v1();
enqueueResponse(secondWebServer, 404, "template [" + template.v1() + "] does not exist");
enqueueResponse(secondWebServer, 201, "template [" + template.v1() + "] created");
}
@ -264,22 +294,21 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
enqueueResponse(secondWebServer, 200, "{\"errors\": false}");
// second event
export(Collections.singletonList(newRandomMonitoringDoc()));
export(newSettings, Collections.singletonList(newRandomMonitoringDoc()));
assertMonitorVersion(secondWebServer);
assertMonitorMappingTypes(secondWebServer, !typeMappingsExistAlready, null, null);
for (Tuple<String, String> template : monitoringTemplates()) {
for (Tuple<String, String> template : monitoringTemplates(includeOldTemplates)) {
MockRequest recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getUri().getPath(), equalTo("/_template/" + template.v1()));
assertThat(recordedRequest.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(recordedRequest.getUri().getQuery(), equalTo(resourceVersionQueryString()));
if (template.v1().contains(MonitoringBulkTimestampedResolver.Data.DATA) == false) {
if (missingTemplate.equals(template.v1())) {
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getUri().getPath(), equalTo("/_template/" + template.v1()));
assertThat(recordedRequest.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(recordedRequest.getUri().getQuery(), equalTo(resourceVersionQueryString()));
assertThat(recordedRequest.getBody(), equalTo(template.v2()));
}
}
@ -292,18 +321,24 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
public void testUnsupportedClusterVersion() throws Exception {
Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
Settings settings = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer));
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.build();
// returning an unsupported cluster version
enqueueGetClusterVersionResponse(randomFrom(Version.fromString("0.18.0"), Version.fromString("1.0.0"), Version.fromString("1.4.0"),
Version.fromString("2.4.0")));
enqueueGetClusterVersionResponse(
randomFrom(Version.fromString("0.18.0"),
Version.fromString("1.0.0"),
Version.fromString("1.4.0"),
Version.fromString("2.4.0"),
Version.fromString("5.0.0"),
Version.fromString("5.4.0")));
String agentNode = internalCluster().startNode(builder);
// fire off what should be an unsuccessful request
assertNull(getExporter(agentNode).openBulk());
// ensure that the exporter is not able to be used
try (HttpExporter exporter = createHttpExporter(settings)) {
assertThat(exporter.isExporterReady(), is(false));
}
assertThat(webServer.requests(), hasSize(1));
@ -311,24 +346,24 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
public void testDynamicIndexFormatChange() throws Exception {
Settings.Builder builder = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), "-1")
final Settings settings = Settings.builder()
.put("xpack.monitoring.exporters._http.type", "http")
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer));
internalCluster().startNode(builder);
.put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
.build();
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
MonitoringDoc doc = newRandomMonitoringDoc();
export(Collections.singletonList(doc));
export(settings, Collections.singletonList(doc));
assertMonitorResources(webServer,
typeMappingsExistAlready, templatesExistsAlready, pipelineExistsAlready,
templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist);
MockRequest recordedRequest = assertBulk(webServer);
@ -343,22 +378,25 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
assertThat(index.get("_index"), equalTo(indexName));
String newTimeFormat = randomFrom("YY", "YYYY", "YYYY.MM", "YYYY-MM", "MM.YYYY", "MM");
assertAcked(client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().put("xpack.monitoring.exporters._http.index.name.time_format", newTimeFormat)));
final Settings newSettings = Settings.builder()
.put(settings)
.put("xpack.monitoring.exporters._http.index.name.time_format", newTimeFormat)
.build();
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueSetupResponses(webServer, true, true, true,
enqueueSetupResponses(webServer, true, includeOldTemplates, true,
true, true, true,
false, false);
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
doc = newRandomMonitoringDoc();
export(Collections.singletonList(doc));
export(newSettings, Collections.singletonList(doc));
String expectedMonitoringIndex = ".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-"
String expectedMonitoringIndex = ".monitoring-es-" + TEMPLATE_VERSION + "-"
+ DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.getTimestamp());
assertMonitorResources(webServer, true, true, true,
assertMonitorResources(webServer, true, includeOldTemplates, true,
true, true, true,
false, false);
recordedRequest = assertBulk(webServer);
@ -388,105 +426,73 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
private void assertMonitorResources(final MockWebServer webServer,
final boolean typeMappingsExistAlready,
final boolean templateAlreadyExists, final boolean pipelineAlreadyExists,
final boolean templateAlreadyExists, final boolean includeOldTemplates,
final boolean pipelineAlreadyExists,
final boolean remoteClusterAllowsWatcher, final boolean currentLicenseAllowsWatcher,
final boolean watcherAlreadyExists,
final boolean bwcIndexesExist, final boolean bwcAliasesExist) throws Exception {
assertMonitorResources(webServer, typeMappingsExistAlready, templateAlreadyExists, pipelineAlreadyExists,
assertMonitorResources(webServer, templateAlreadyExists, includeOldTemplates, pipelineAlreadyExists,
remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
bwcIndexesExist, bwcAliasesExist, null, null);
}
private void assertMonitorResources(final MockWebServer webServer,
final boolean typeMappingsExistAlready,
final boolean templateAlreadyExists, final boolean pipelineAlreadyExists,
final boolean templateAlreadyExists, final boolean includeOldTemplates,
final boolean pipelineAlreadyExists,
final boolean remoteClusterAllowsWatcher, final boolean currentLicenseAllowsWatcher,
final boolean watcherAlreadyExists,
boolean bwcIndexesExist, boolean bwcAliasesExist,
@Nullable final Map<String, String[]> customHeaders,
@Nullable final String basePath) throws Exception {
assertMonitorVersion(webServer, customHeaders, basePath);
assertMonitorMappingTypes(webServer, typeMappingsExistAlready, customHeaders, basePath);
assertMonitorTemplates(webServer, templateAlreadyExists, customHeaders, basePath);
assertMonitorTemplates(webServer, templateAlreadyExists, includeOldTemplates, customHeaders, basePath);
assertMonitorPipelines(webServer, pipelineAlreadyExists, customHeaders, basePath);
assertMonitorBackwardsCompatibilityAliases(webServer, bwcIndexesExist && false == bwcAliasesExist, customHeaders, basePath);
assertMonitorWatches(webServer, remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists,
customHeaders, basePath);
}
private void assertMonitorMappingTypes(final MockWebServer webServer,
final boolean alreadyExists,
@Nullable final Map<String, String[]> customHeaders,
@Nullable final String basePath) throws Exception {
final String pathPrefix = basePathToAssertablePrefix(basePath);
MockRequest request;
private void assertMonitorTemplates(final MockWebServer webServer,
final boolean alreadyExists,
final boolean includeOldTemplates,
@Nullable final Map<String, String[]> customHeaders,
@Nullable final String basePath) throws Exception {
final List<Tuple<String, String>> templates = monitoringTemplates(includeOldTemplates);
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("GET"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/" + DATA_INDEX + "/_mapping/" + type));
assertThat(request.getUri().getQuery(), nullValue());
assertHeaders(request, customHeaders);
if (alreadyExists == false) {
request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("PUT"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/" + DATA_INDEX + "/_mapping/" + type));
assertThat(request.getUri().getQuery(), nullValue());
assertThat(request.getBody(), equalTo("{\"enabled\":false}"));
assertHeaders(request, customHeaders);
}
}
assertMonitorVersionResource(webServer, alreadyExists, "/_template/", templates, customHeaders, basePath);
}
private void assertMonitorTemplates(final MockWebServer webServer,
private void assertMonitorPipelines(final MockWebServer webServer,
final boolean alreadyExists,
@Nullable final Map<String, String[]> customHeaders,
@Nullable final String basePath) throws Exception {
final String pathPrefix = basePathToAssertablePrefix(basePath);
MockRequest request;
for (Tuple<String, String> template : monitoringTemplates()) {
request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("GET"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_template/" + template.v1()));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertHeaders(request, customHeaders);
if (alreadyExists == false) {
request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("PUT"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_template/" + template.v1()));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(request.getBody(), equalTo(template.v2()));
assertHeaders(request, customHeaders);
}
}
assertMonitorVersionResource(webServer, alreadyExists, "/_ingest/pipeline/", monitoringPipelines(),
customHeaders, basePath);
}
private void assertMonitorPipelines(final MockWebServer webServer, final boolean alreadyExists,
@Nullable final Map<String, String[]> customHeaders, @Nullable final String basePath) throws Exception {
private void assertMonitorVersionResource(final MockWebServer webServer, final boolean alreadyExists,
final String resourcePrefix, final List<Tuple<String, String>> resources,
@Nullable final Map<String, String[]> customHeaders,
@Nullable final String basePath) throws Exception {
final String pathPrefix = basePathToAssertablePrefix(basePath);
MockRequest request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("GET"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_ingest/pipeline/" + Exporter.EXPORT_PIPELINE_NAME));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertHeaders(request, customHeaders);
for (Tuple<String, String> resource : resources) {
final MockRequest getRequest = webServer.takeRequest();
if (alreadyExists == false) {
request = webServer.takeRequest();
assertThat(getRequest.getMethod(), equalTo("GET"));
assertThat(getRequest.getUri().getPath(), equalTo(pathPrefix + resourcePrefix + resource.v1()));
assertThat(getRequest.getUri().getQuery(), equalTo(resourceVersionQueryString()));
assertHeaders(getRequest, customHeaders);
assertThat(request.getMethod(), equalTo("PUT"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_ingest/pipeline/" + Exporter.EXPORT_PIPELINE_NAME));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(request.getBody(), equalTo(Exporter.emptyPipeline(XContentType.JSON).string()));
assertHeaders(request, customHeaders);
if (alreadyExists == false) {
final MockRequest putRequest = webServer.takeRequest();
assertThat(putRequest.getMethod(), equalTo("PUT"));
assertThat(putRequest.getUri().getPath(), equalTo(pathPrefix + resourcePrefix + resource.v1()));
assertThat(putRequest.getUri().getQuery(), equalTo(resourceVersionQueryString()));
assertThat(putRequest.getBody(), equalTo(resource.v2()));
assertHeaders(putRequest, customHeaders);
}
}
}
@ -514,7 +520,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
if (currentLicenseAllowsWatcher) {
assertThat(request.getMethod(), equalTo("GET"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
assertHeaders(request, customHeaders);
if (alreadyExists == false) {
@ -522,7 +528,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
assertThat(request.getMethod(), equalTo("PUT"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
assertThat(request.getBody(), equalTo(watch.v2()));
assertHeaders(request, customHeaders);
}
@ -530,7 +536,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
} else {
assertThat(request.getMethod(), equalTo("DELETE"));
assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
assertThat(request.getUri().getQuery(), equalTo(resourceQueryString()));
assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
assertHeaders(request, customHeaders);
}
}
@ -571,8 +577,6 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
@Nullable final Map<String, String[]> customHeaders, @Nullable final String basePath)
throws Exception {
final String pathPrefix = basePathToAssertablePrefix(basePath);
// the bulk request is fired off asynchronously so we might need to take a while, until we can get the request from the webserver
assertBusy(() -> assertThat("Waiting for further requests in web server", webServer.hasMoreRequests(), is(true)));
final MockRequest request = webServer.takeRequest();
assertThat(request.getMethod(), equalTo("POST"));
@ -603,21 +607,49 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
}
private void export(Collection<MonitoringDoc> docs) throws Exception {
Exporters exporters = internalCluster().getInstance(Exporters.class);
assertThat(exporters, notNullValue());
private HttpExporter createHttpExporter(final Settings settings) throws Exception {
final Exporter.Config config =
new Exporter.Config("_http", "http",
settings, settings.getAsSettings("xpack.monitoring.exporters._http"),
clusterService(), new XPackLicenseState());
// Wait for exporting bulks to be ready to export
assertBusy(() -> assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION)));
assertBusy(() -> exporters.forEach(exporter -> assertThat(exporter.openBulk(), notNullValue())));
PlainActionFuture<Void> future = new PlainActionFuture<>();
exporters.export(docs, future);
future.get();
return new HttpExporter(config, new SSLService(settings, environment), new ThreadContext(settings));
}
private HttpExporter getExporter(String nodeName) {
Exporters exporters = internalCluster().getInstance(Exporters.class, nodeName);
return (HttpExporter) exporters.iterator().next();
private void export(final Settings settings, final Collection<MonitoringDoc> docs) throws Exception {
// wait until the cluster is ready (this is done at the "Exporters" level)
assertBusy(() -> assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION)));
try (HttpExporter exporter = createHttpExporter(settings)) {
// the readiness check happens synchronously, so we don't need to busy-wait for it
assertThat("Exporter is not ready", exporter.isExporterReady(), is(true));
final HttpExportBulk bulk = exporter.openBulk();
assertThat("Bulk should never be null after the exporter is ready", bulk, notNullValue());
final CountDownLatch awaitResponseAndClose = new CountDownLatch(2);
final ActionListener<Void> listener = new ActionListener<Void>() {
@Override
public void onResponse(Void response) {
awaitResponseAndClose.countDown();
}
@Override
public void onFailure(Exception e) {
fail(e.getMessage());
awaitResponseAndClose.countDown();
}
};
bulk.doAdd(docs);
bulk.doFlush(listener);
bulk.doClose(listener);
// block until the bulk responds
awaitResponseAndClose.await(15, TimeUnit.SECONDS);
}
}
private MonitoringDoc newRandomMonitoringDoc() {
@ -652,8 +684,12 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
return basePath;
}
private String resourceQueryString() {
return "filter_path=" + FILTER_PATH_NONE;
private String resourceClusterAlertQueryString() {
return "filter_path=" + CLUSTER_ALERT_VERSION_PARAMETERS.get("filter_path");
}
private String resourceVersionQueryString() {
return "filter_path=" + FILTER_PATH_RESOURCE_VERSION;
}
private String watcherCheckQueryString() {
@ -661,7 +697,9 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
private String bulkQueryString() {
return "pipeline=" + Exporter.EXPORT_PIPELINE_NAME + "&filter_path=" + "errors,items.*.error";
final String pipelineName = MonitoringTemplateUtils.pipelineName(TEMPLATE_VERSION);
return "pipeline=" + pipelineName + "&filter_path=" + "errors,items.*.error";
}
private void enqueueGetClusterVersionResponse(Version v) throws IOException {
@ -674,63 +712,38 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
.field("number", v.toString()).endObject().endObject().bytes().utf8ToString()));
}
private void enqueueSetupResponses(MockWebServer webServer,
boolean typeMappingsAlreadyExist,
boolean templatesAlreadyExists, boolean pipelineAlreadyExists,
boolean remoteClusterAllowsWatcher, boolean currentLicenseAllowsWatcher,
boolean watcherAlreadyExists,
boolean bwcIndexesExist, boolean bwcAliasesExist) throws IOException {
enqueueMappingTypeResponses(webServer, typeMappingsAlreadyExist);
enqueueTemplateResponses(webServer, templatesAlreadyExists);
private void enqueueSetupResponses(final MockWebServer webServer,
final boolean templatesAlreadyExists, final boolean includeOldTemplates,
final boolean pipelineAlreadyExists,
final boolean remoteClusterAllowsWatcher, final boolean currentLicenseAllowsWatcher,
final boolean watcherAlreadyExists,
final boolean bwcIndexesExist, final boolean bwcAliasesExist) throws IOException {
enqueueTemplateResponses(webServer, templatesAlreadyExists, includeOldTemplates);
enqueuePipelineResponses(webServer, pipelineAlreadyExists);
enqueueBackwardsCompatibilityAliasResponse(webServer, bwcIndexesExist, bwcAliasesExist);
enqueueWatcherResponses(webServer, remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists);
}
private void enqueueMappingTypeResponses(final MockWebServer webServer, final boolean alreadyExists) throws IOException {
private void enqueueTemplateResponses(final MockWebServer webServer,
final boolean alreadyExists, final boolean includeOldTemplates)
throws IOException {
if (alreadyExists) {
enqueueMappingTypeResponsesExistsAlreadyOrWillBeCreated(webServer);
enqueueTemplateResponsesExistsAlready(webServer, includeOldTemplates);
} else {
enqueueMappingTypeResponsesDoesNotExistYet(webServer);
enqueueTemplateResponsesDoesNotExistYet(webServer, includeOldTemplates);
}
}
private void enqueueMappingTypeResponsesDoesNotExistYet(final MockWebServer webServer) throws IOException {
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
enqueueResponse(webServer, 200, "{}");
enqueueResponse(webServer, 200, "type [" + type + "] created");
}
private void enqueueTemplateResponsesDoesNotExistYet(final MockWebServer webServer,
final boolean includeOldTemplates)
throws IOException {
enqueueVersionedResourceResponsesDoesNotExistYet(monitoringTemplateNames(includeOldTemplates), webServer);
}
private void enqueueMappingTypeResponsesExistsAlreadyOrWillBeCreated(final MockWebServer webServer) throws IOException {
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
if (randomBoolean()) {
enqueueResponse(webServer, 200, "{\".monitoring-data-2\":{\"" + type + "\":{\"enabled\":false}}}");
} else {
enqueueResponse(webServer, 404, "index does not exist; template will create it");
}
}
}
private void enqueueTemplateResponses(final MockWebServer webServer, final boolean alreadyExists) throws IOException {
if (alreadyExists) {
enqueueTemplateResponsesExistsAlready(webServer);
} else {
enqueueTemplateResponsesDoesNotExistYet(webServer);
}
}
private void enqueueTemplateResponsesDoesNotExistYet(final MockWebServer webServer) throws IOException {
for (String template : monitoringTemplateNames()) {
enqueueResponse(webServer, 404, "template [" + template + "] does not exist");
enqueueResponse(webServer, 201, "template [" + template + "] created");
}
}
private void enqueueTemplateResponsesExistsAlready(final MockWebServer webServer) throws IOException {
for (String template : monitoringTemplateNames()) {
enqueueResponse(webServer, 200, "template [" + template + "] exists");
}
private void enqueueTemplateResponsesExistsAlready(final MockWebServer webServer,
final boolean includeOldTemplates)
throws IOException {
enqueueVersionedResourceResponsesExistsAlready(monitoringTemplateNames(includeOldTemplates), webServer);
}
private void enqueuePipelineResponses(final MockWebServer webServer, final boolean alreadyExists) throws IOException {
@ -742,12 +755,44 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
private void enqueuePipelineResponsesDoesNotExistYet(final MockWebServer webServer) throws IOException {
enqueueResponse(webServer, 404, "pipeline [" + Exporter.EXPORT_PIPELINE_NAME + "] does not exist");
enqueueResponse(webServer, 201, "pipeline [" + Exporter.EXPORT_PIPELINE_NAME + "] created");
enqueueVersionedResourceResponsesDoesNotExistYet(monitoringPipelineNames(), webServer);
}
private void enqueuePipelineResponsesExistsAlready(final MockWebServer webServer) throws IOException {
enqueueResponse(webServer, 200, "pipeline [" + Exporter.EXPORT_PIPELINE_NAME + "] exists");
enqueueVersionedResourceResponsesExistsAlready(monitoringPipelineNames(), webServer);
}
private void enqueueVersionedResourceResponsesDoesNotExistYet(final List<String> names, final MockWebServer webServer)
throws IOException {
for (String resource : names) {
if (randomBoolean()) {
enqueueResponse(webServer, 404, "[" + resource + "] does not exist");
} else if (randomBoolean()) {
final int version = LAST_UPDATED_VERSION - randomIntBetween(1, 1000000);
// it DOES exist, but it's an older version
enqueueResponse(webServer, 200, "{\"" + resource + "\":{\"version\":" + version + "}}");
} else {
// no version specified
enqueueResponse(webServer, 200, "{\"" + resource + "\":{}}");
}
enqueueResponse(webServer, 201, "[" + resource + "] created");
}
}
private void enqueueVersionedResourceResponsesExistsAlready(final List<String> names, final MockWebServer webServer)
throws IOException {
for (String resource : names) {
if (randomBoolean()) {
final int newerVersion = randomFrom(Version.CURRENT.id, LAST_UPDATED_VERSION) + randomIntBetween(1, 1000000);
// it's a NEWER resource (template / pipeline)
enqueueResponse(webServer, 200, "{\"" + resource + "\":{\"version\":" + newerVersion + "}}");
} else {
// we already put it
enqueueResponse(webServer, 200, "{\"" + resource + "\":{\"version\":" + LAST_UPDATED_VERSION + "}}");
}
}
}
private void enqueueWatcherResponses(final MockWebServer webServer,
@ -761,13 +806,13 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
// if we have an active license that's not Basic, then we should add watches
if (currentLicenseAllowsWatcher) {
if (alreadyExists) {
enqueuePutWatchResponsesExistsAlready(webServer);
enqueueClusterAlertResponsesExistsAlready(webServer);
} else {
enqueuePutWatchResponsesDoesNotExistYet(webServer);
enqueueClusterAlertResponsesDoesNotExistYet(webServer);
}
// otherwise we need to delete them from the remote cluster
} else {
enqueueDeleteWatchResponses(webServer);
enqueueDeleteClusterAlertResponses(webServer);
}
} else {
// X-Pack exists but Watcher just cannot be used
@ -786,21 +831,43 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
}
}
private void enqueuePutWatchResponsesDoesNotExistYet(final MockWebServer webServer) throws IOException {
for (String watchId : monitoringWatchIds()) {
enqueueResponse(webServer, 404, "watch [" + watchId + "] does not exist");
enqueueResponse(webServer, 201, "watch [" + watchId + "] created");
private void enqueueClusterAlertResponsesDoesNotExistYet(final MockWebServer webServer)
throws IOException {
for (final String watchId : monitoringWatchIds()) {
if (randomBoolean()) {
enqueueResponse(webServer, 404, "watch [" + watchId + "] does not exist");
} else if (randomBoolean()) {
final int version = LAST_UPDATED_VERSION - randomIntBetween(1, 1000000);
// it DOES exist, but it's an older version
enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + version + "}}}");
} else {
// no version specified
enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{}}}");
}
enqueueResponse(webServer, 201, "[" + watchId + "] created");
}
}
private void enqueuePutWatchResponsesExistsAlready(final MockWebServer webServer) throws IOException {
for (String watchId : monitoringWatchIds()) {
enqueueResponse(webServer, 200, "watch [" + watchId + "] exists");
private void enqueueClusterAlertResponsesExistsAlready(final MockWebServer webServer) throws IOException {
final int count = monitoringWatchIds().size();
for (int i = 0; i < count; ++i) {
final int existsVersion;
if (randomBoolean()) {
// it's a NEWER cluster alert
existsVersion = randomFrom(Version.CURRENT.id, ClusterAlertsUtil.LAST_UPDATED_VERSION) + randomIntBetween(1, 1000000);
} else {
// we already put it
existsVersion = ClusterAlertsUtil.LAST_UPDATED_VERSION;
}
enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + existsVersion + "}}}");
}
}
private void enqueueDeleteWatchResponses(final MockWebServer webServer) throws IOException {
for (String watchId : monitoringWatchIds()) {
private void enqueueDeleteClusterAlertResponses(final MockWebServer webServer) throws IOException {
for (final String watchId : monitoringWatchIds()) {
if (randomBoolean()) {
enqueueResponse(webServer, 404, "watch [" + watchId + "] did not exist");
} else {
@ -867,4 +934,37 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
server.start();
return server;
}
private List<Tuple<String, String>> monitoringTemplates(final boolean includeOldTemplates) {
return includeOldTemplates ? monitoringTemplatesWithOldTemplates() : monitoringTemplates();
}
// this can be removed in 7.0
private List<Tuple<String, String>> monitoringTemplatesWithOldTemplates() {
final List<Tuple<String, String>> expectedTemplates = monitoringTemplates();
expectedTemplates.addAll(
Arrays.stream(MonitoringTemplateUtils.OLD_TEMPLATE_IDS)
.map(id -> new Tuple<>(MonitoringTemplateUtils.oldTemplateName(id), MonitoringTemplateUtils.createEmptyTemplate(id)))
.collect(Collectors.toList()));
return expectedTemplates;
}
private List<String> monitoringTemplateNames(final boolean includeOldTemplates) {
return includeOldTemplates ? monitoringTemplateNamesWithOldTemplates() : monitoringTemplateNames();
}
// this can be removed in 7.0
protected List<String> monitoringTemplateNamesWithOldTemplates() {
final List<String> expectedTemplateNames = monitoringTemplateNames();
expectedTemplateNames.addAll(
Arrays.stream(MonitoringTemplateUtils.OLD_TEMPLATE_IDS)
.map(MonitoringTemplateUtils::oldTemplateName)
.collect(Collectors.toList()));
return expectedTemplateNames;
}
}

457
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
View File

@ -20,15 +20,27 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.DATA_INDEX;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter.TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.DOES_NOT_EXIST;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.EXISTS;
import static org.hamcrest.Matchers.hasSize;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyMapOf;
import static org.mockito.Matchers.eq;
@ -49,21 +61,53 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
private final boolean remoteClusterHasWatcher = randomBoolean();
private final boolean validLicense = randomBoolean();
private final boolean createOldTemplates = randomBoolean();
/**
* kibana, logstash, beats
*/
private final int EXPECTED_TYPES = MonitoringTemplateUtils.NEW_DATA_TYPES.length;
private final int EXPECTED_TEMPLATES = 6;
private final int EXPECTED_TEMPLATES = 5 + (createOldTemplates ? OLD_TEMPLATE_IDS.length : 0);
private final int EXPECTED_PIPELINES = PIPELINE_IDS.length;
private final int EXPECTED_WATCHES = 4;
private final RestClient client = mock(RestClient.class);
private final Response versionResponse = mock(Response.class);
private final ResolversRegistry registry = new ResolversRegistry(Settings.EMPTY);
private final List<String> templateNames = new ArrayList<>(EXPECTED_TEMPLATES);
private final List<String> pipelineNames = new ArrayList<>(EXPECTED_PIPELINES);
private final List<String> watchNames = new ArrayList<>(EXPECTED_WATCHES);
private final Settings exporterSettings = Settings.builder().put(TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING, createOldTemplates).build();
private final MultiHttpResource resources =
HttpExporter.createResources(
new Exporter.Config("_http", "http", Settings.EMPTY, Settings.EMPTY, clusterService, licenseState),
new ResolversRegistry(Settings.EMPTY));
new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState), registry);
@Before
public void setupResources() {
templateNames.addAll(Arrays.stream(TEMPLATE_IDS).map(MonitoringTemplateUtils::templateName).collect(Collectors.toList()));
// TODO: when resolvers are removed, all templates managed by this loop should be included in the TEMPLATE_IDS above
for (final MonitoringIndexNameResolver resolver : registry) {
final String templateName = resolver.templateName();
if (templateNames.contains(templateName) == false) {
templateNames.add(templateName);
}
}
if (createOldTemplates) {
templateNames.addAll(
Arrays.stream(OLD_TEMPLATE_IDS).map(MonitoringTemplateUtils::oldTemplateName).collect(Collectors.toList()));
}
pipelineNames.addAll(Arrays.stream(PIPELINE_IDS).map(MonitoringTemplateUtils::pipelineName).collect(Collectors.toList()));
watchNames.addAll(Arrays.stream(ClusterAlertsUtil.WATCH_IDS).map(id -> "my_cluster_uuid_" + id).collect(Collectors.toList()));
assertThat("Not all templates are supplied", templateNames, hasSize(EXPECTED_TEMPLATES));
assertThat("Not all pipelines are supplied", pipelineNames, hasSize(EXPECTED_PIPELINES));
assertThat("Not all watches are supplied", watchNames, hasSize(EXPECTED_WATCHES));
}
public void testInvalidVersionBlocks() throws IOException {
final HttpEntity entity = new StringEntity("{\"version\":{\"number\":\"unknown\"}}", ContentType.APPLICATION_JSON);
@ -80,7 +124,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
verifyNoMoreInteractions(client);
}
public void testTypeMappingCheckBlocksAfterSuccessfulVersion() throws IOException {
public void testTemplateCheckBlocksAfterSuccessfulVersion() throws IOException {
final Exception exception = failureGetException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
@ -88,109 +132,6 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
whenValidVersionResponse();
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
final boolean successfulFirst = randomBoolean();
// -2 from one success + a necessary failure after it!
final int extraPasses = Math.max(randomIntBetween(0, EXPECTED_TYPES - 2), 0);
final int successful = randomIntBetween(0, extraPasses);
final int unsuccessful = extraPasses - successful;
final Response first = successfulFirst ? successfulGetTypeMappingResponse() : unsuccessfulGetTypeMappingResponse();
final List<Response> otherResponses = getTypeMappingResponses(successful, unsuccessful);
// last check fails implies that N - 2 publishes succeeded!
when(client.performRequest(eq("GET"), startsWith("/" + DATA_INDEX + "/_mapping/"), anyMapOf(String.class, String.class)))
.thenReturn(first, otherResponses.toArray(new Response[otherResponses.size()]))
.thenThrow(exception);
whenSuccessfulPutTypeMappings(otherResponses.size() + 1);
expectedGets += 1 + successful + unsuccessful;
expectedPuts = (successfulFirst ? 0 : 1) + unsuccessful;
} else {
when(client.performRequest(eq("GET"), startsWith("/" + DATA_INDEX + "/_mapping/"), anyMapOf(String.class, String.class)))
.thenThrow(exception);
}
assertTrue(resources.isDirty());
assertFalse(resources.checkAndPublish(client));
// ensure it didn't magically become not-dirty
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(expectedGets);
verifyPutTypeMappings(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testTypeMappingPublishBlocksAfterSuccessfulVersion() throws IOException {
final Exception exception = failurePutException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 1;
whenValidVersionResponse();
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
final Response firstSuccess = successfulPutResponse();
// -2 from one success + a necessary failure after it!
final int extraPasses = randomIntBetween(0, EXPECTED_TYPES - 2);
final int successful = randomIntBetween(0, extraPasses);
final int unsuccessful = extraPasses - successful;
final List<Response> otherResponses = successfulPutResponses(unsuccessful);
// first one passes for sure, so we need an extra "unsuccessful" GET
whenGetTypeMappingResponse(successful, unsuccessful + 2);
// previous publishes must have succeeded
when(client.performRequest(eq("PUT"),
startsWith("/" + DATA_INDEX + "/_mapping/"),
anyMapOf(String.class, String.class),
any(HttpEntity.class)))
.thenReturn(firstSuccess, otherResponses.toArray(new Response[otherResponses.size()]))
.thenThrow(exception);
// GETs required for each PUT attempt (first is guaranteed "unsuccessful")
expectedGets += successful + unsuccessful + 1;
// unsuccessful are PUT attempts + the guaranteed successful PUT (first)
expectedPuts += unsuccessful + 1;
} else {
// fail the check so that it has to attempt the PUT
whenGetTypeMappingResponse(0, 1);
when(client.performRequest(eq("PUT"),
startsWith("/" + DATA_INDEX + "/_mapping/"),
anyMapOf(String.class, String.class),
any(HttpEntity.class)))
.thenThrow(exception);
}
assertTrue(resources.isDirty());
assertFalse(resources.checkAndPublish(client));
// ensure it didn't magically become not-dirty
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(expectedGets);
verifyPutTypeMappings(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testTemplateCheckBlocksAfterSuccessfulTypeMapping() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final Exception exception = failureGetException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 0;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
final boolean successfulFirst = randomBoolean();
@ -198,10 +139,17 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
final int extraPasses = randomIntBetween(0, EXPECTED_TEMPLATES - 2);
final int successful = randomIntBetween(0, extraPasses);
final int unsuccessful = extraPasses - successful;
final String templateName = templateNames.get(0);
final Response first = successfulFirst ? successfulGetResponse() : unsuccessfulGetResponse();
final Response first;
final List<Response> otherResponses = getResponses(successful, unsuccessful);
if (successfulFirst) {
first = successfulGetResourceResponse("/_template/", templateName);
} else {
first = unsuccessfulGetResourceResponse("/_template/", templateName);
}
final List<Response> otherResponses = getTemplateResponses(1, successful, unsuccessful);
// last check fails implies that N - 2 publishes succeeded!
when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
@ -222,24 +170,18 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(expectedGets);
verifyPutTemplates(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testTemplatePublishBlocksAfterSuccessfulTypeMapping() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
public void testTemplatePublishBlocksAfterSuccessfulVersion() throws IOException {
final Exception exception = failurePutException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 1;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
@ -277,29 +219,50 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(expectedGets);
verifyPutTemplates(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testPipelineCheckBlocksAfterSuccessfulTemplates() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final Exception exception = failureGetException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 0;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(EXPECTED_TEMPLATES);
// we only expect a single pipeline for now
when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
.thenThrow(exception);
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
final boolean successfulFirst = randomBoolean();
final String pipelineName = pipelineNames.get(0);
final Response first;
if (successfulFirst) {
first = successfulGetResourceResponse("/_ingest/pipeline/", pipelineName);
} else {
first = unsuccessfulGetResourceResponse("/_ingest/pipeline/", pipelineName);
}
// last check fails
when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
.thenReturn(first)
.thenThrow(exception);
if (successfulFirst == false) {
whenSuccessfulPutPipelines(1);
}
expectedGets = EXPECTED_PIPELINES;
expectedPuts = successfulFirst ? 0 : 1;
} else {
when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
.thenThrow(exception);
}
assertTrue(resources.isDirty());
assertFalse(resources.checkAndPublish(client));
@ -307,36 +270,54 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyPutPipelines(0);
verifyGetPipelines(expectedGets);
verifyPutPipelines(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testPipelinePublishBlocksAfterSuccessfulTemplates() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final Exception exception = failurePutException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 1;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(EXPECTED_TEMPLATES);
// pipeline can't be there
whenGetPipelines(0, 1);
// we only expect a single pipeline for now
when(client.performRequest(eq("PUT"),
startsWith("/_ingest/pipeline/"),
anyMapOf(String.class, String.class),
any(HttpEntity.class)))
.thenThrow(exception);
// failure in the middle of various templates being checked/published; suggests a node dropped
if (firstSucceeds) {
final Response firstSuccess = successfulPutResponse();
// We only have two pipelines for now, so the both GETs need to be "unsuccessful" for until we have a third
whenGetPipelines(0, 2);
// previous publishes must have succeeded
when(client.performRequest(eq("PUT"),
startsWith("/_ingest/pipeline/"),
anyMapOf(String.class, String.class),
any(HttpEntity.class)))
.thenReturn(firstSuccess)
.thenThrow(exception);
// GETs required for each PUT attempt (first is guaranteed "unsuccessful")
expectedGets += 1;
// unsuccessful are PUT attempts
expectedPuts += 1;
} else {
// fail the check so that it has to attempt the PUT
whenGetPipelines(0, 1);
when(client.performRequest(eq("PUT"),
startsWith("/_ingest/pipeline/"),
anyMapOf(String.class, String.class),
any(HttpEntity.class)))
.thenThrow(exception);
}
assertTrue(resources.isDirty());
assertFalse(resources.checkAndPublish(client));
@ -344,31 +325,25 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyPutPipelines(1);
verifyGetPipelines(expectedGets);
verifyPutPipelines(expectedPuts);
verifyNoMoreInteractions(client);
}
public void testWatcherCheckBlocksAfterSuccessfulPipelines() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final int successfulGetPipelines = randomIntBetween(0, 1);
final int unsuccessfulGetPipelines = 1 - successfulGetPipelines;
final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES);
final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines;
final Exception exception = failureGetException();
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(unsuccessfulGetTemplates);
whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines);
whenSuccessfulPutPipelines(1);
whenSuccessfulPutPipelines(unsuccessfulGetPipelines);
whenSuccessfulBackwardsCompatibilityAliases();
// there's only one check
@ -380,11 +355,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyGetPipelines(EXPECTED_PIPELINES);
verifyPutPipelines(unsuccessfulGetPipelines);
verifyBackwardsCompatibilityAliases();
verifyWatcherCheck();
@ -392,24 +365,20 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
}
public void testWatchCheckBlocksAfterSuccessfulWatcherCheck() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final int successfulGetPipelines = randomIntBetween(0, 1);
final int unsuccessfulGetPipelines = 1 - successfulGetPipelines;
final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES);
final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines;
final Exception exception = validLicense ? failureGetException() : failureDeleteException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 0;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(unsuccessfulGetTemplates);
whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines);
whenSuccessfulPutPipelines(1);
whenSuccessfulPutPipelines(unsuccessfulGetPipelines);
whenSuccessfulBackwardsCompatibilityAliases();
whenWatcherCanBeUsed(validLicense);
@ -423,8 +392,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
final int successful = randomIntBetween(0, extraPasses);
final int unsuccessful = extraPasses - successful;
final Response first = successfulFirst ? successfulGetResponse() : unsuccessfulGetResponse();
final List<Response> otherResponses = getResponses(successful, unsuccessful);
final String watchId = watchNames.get(0);
final Response first = successfulFirst ? successfulGetWatchResponse(watchId) : unsuccessfulGetWatchResponse(watchId);
final List<Response> otherResponses = getWatcherResponses(1, successful, unsuccessful);
// last check fails implies that N - 2 publishes succeeded!
when(client.performRequest(eq("GET"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
@ -462,11 +432,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyGetPipelines(EXPECTED_PIPELINES);
verifyPutPipelines(unsuccessfulGetPipelines);
verifyBackwardsCompatibilityAliases();
verifyWatcherCheck();
@ -480,24 +448,20 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
}
public void testWatchPublishBlocksAfterSuccessfulWatcherCheck() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final int successfulGetPipelines = randomIntBetween(0, 1);
final int unsuccessfulGetPipelines = 1 - successfulGetPipelines;
final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES);
final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines;
final Exception exception = failurePutException();
final boolean firstSucceeds = randomBoolean();
int expectedGets = 1;
int expectedPuts = 1;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(unsuccessfulGetTemplates);
whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines);
whenSuccessfulPutPipelines(1);
whenSuccessfulPutPipelines(unsuccessfulGetPipelines);
whenSuccessfulBackwardsCompatibilityAliases();
// license needs to be valid, otherwise we'll do DELETEs, which are tested earlier
whenWatcherCanBeUsed(true);
@ -544,11 +508,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertTrue(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyGetPipelines(EXPECTED_PIPELINES);
verifyPutPipelines(unsuccessfulGetPipelines);
verifyBackwardsCompatibilityAliases();
verifyWatcherCheck();
@ -558,22 +520,18 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
}
public void testSuccessfulChecksOnElectedMasterNode() throws IOException {
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final int successfulGetPipelines = randomIntBetween(0, 1);
final int unsuccessfulGetPipelines = 1 - successfulGetPipelines;
final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES);
final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines;
final int successfulGetWatches = randomIntBetween(0, EXPECTED_WATCHES);
final int unsuccessfulGetWatches = EXPECTED_WATCHES - successfulGetWatches;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(unsuccessfulGetTemplates);
whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines);
whenSuccessfulPutPipelines(1);
whenSuccessfulPutPipelines(unsuccessfulGetPipelines);
if (remoteClusterHasWatcher) {
whenWatcherCanBeUsed(validLicense);
if (validLicense) {
@ -594,11 +552,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertFalse(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyGetPipelines(EXPECTED_PIPELINES);
verifyPutPipelines(unsuccessfulGetPipelines);
verifyWatcherCheck();
if (remoteClusterHasWatcher) {
@ -622,19 +578,15 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
final MultiHttpResource resources =
HttpExporter.createResources(
new Exporter.Config("_http", "http", Settings.EMPTY, Settings.EMPTY, clusterService, licenseState),
new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState),
new ResolversRegistry(Settings.EMPTY));
final int successfulGetTypeMappings = randomIntBetween(0, EXPECTED_TYPES);
final int unsuccessfulGetTypeMappings = EXPECTED_TYPES - successfulGetTypeMappings;
final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
final int successfulGetPipelines = randomIntBetween(0, 1);
final int unsuccessfulGetPipelines = 1 - successfulGetPipelines;
final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines;
whenValidVersionResponse();
whenGetTypeMappingResponse(successfulGetTypeMappings, unsuccessfulGetTypeMappings);
whenSuccessfulPutTypeMappings(EXPECTED_TYPES);
whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates);
whenSuccessfulPutTemplates(unsuccessfulGetTemplates);
whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines);
@ -648,11 +600,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
assertFalse(resources.isDirty());
verifyVersionCheck();
verifyGetTypeMappings(EXPECTED_TYPES);
verifyPutTypeMappings(unsuccessfulGetTypeMappings);
verifyGetTemplates(EXPECTED_TEMPLATES);
verifyPutTemplates(unsuccessfulGetTemplates);
verifyGetPipelines(1);
verifyGetPipelines(EXPECTED_PIPELINES);
verifyPutPipelines(unsuccessfulGetPipelines);
verifyBackwardsCompatibilityAliases();
verifyNoMoreInteractions(client);
@ -676,65 +626,73 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
return randomFrom(new IOException("expected"), new RuntimeException("expected"), responseException);
}
private Response successfulGetResponse() {
return response("GET", "/_get_something", successfulCheckStatus());
}
private Response unsuccessfulGetResponse() {
return response("GET", "/_get_something", notFoundCheckStatus());
}
private Response successfulGetTypeMappingResponse() {
final Response response;
private Response successfulGetWatchResponse(final String watchId) {
final HttpEntity goodEntity = entityForClusterAlert(EXISTS, LAST_UPDATED_VERSION);
return response("GET", "/_xpack/watcher/watch/" + watchId, successfulCheckStatus(), goodEntity);
}
private Response unsuccessfulGetWatchResponse(final String watchId) {
if (randomBoolean()) {
// it returned 200, but we also need it to contain _something_ in the JSON {...}
final HttpEntity entity = new StringEntity("{\"" + DATA_INDEX + "\":{}}", ContentType.APPLICATION_JSON);
final HttpEntity badEntity = entityForClusterAlert(DOES_NOT_EXIST, LAST_UPDATED_VERSION);
response = successfulGetResponse();
when(response.getEntity()).thenReturn(entity);
} else {
// simulates the index does not exist
response = unsuccessfulGetResponse();
return response("GET", "/_xpack/watcher/watch/" + watchId, successfulCheckStatus(), badEntity);
}
return response;
return unsuccessfulGetResponse();
}
private Response unsuccessfulGetTypeMappingResponse() {
// "unsuccessful" for type mappings is a response code 200, but the response is literally "{}"
final Response response = successfulGetResponse();
final HttpEntity entity = new StringEntity("{}", ContentType.APPLICATION_JSON);
private Response successfulGetResourceResponse(final String resourcePath, final String resourceName) {
final HttpEntity goodEntity = entityForResource(EXISTS, resourceName, LAST_UPDATED_VERSION);
when(response.getEntity()).thenReturn(entity);
return response;
return response("GET", resourcePath + resourceName, successfulCheckStatus(), goodEntity);
}
private List<Response> getTypeMappingResponses(final int successful, final int unsuccessful) {
private Response unsuccessfulGetResourceResponse(final String resourcePath, final String resourceName) {
if (randomBoolean()) {
final HttpEntity badEntity = entityForResource(DOES_NOT_EXIST, resourceName, LAST_UPDATED_VERSION);
return response("GET", resourcePath + resourceName, successfulCheckStatus(), badEntity);
}
return unsuccessfulGetResponse();
}
private List<Response> getResourceResponses(final String resourcePath, final List<String> resourceNames,
final int skip, final int successful, final int unsuccessful) {
final List<Response> responses = new ArrayList<>(successful + unsuccessful);
for (int i = 0; i < successful; ++i) {
responses.add(successfulGetTypeMappingResponse());
responses.add(successfulGetResourceResponse(resourcePath, resourceNames.get(i + skip)));
}
for (int i = 0; i < unsuccessful; ++i) {
responses.add(unsuccessfulGetTypeMappingResponse());
responses.add(unsuccessfulGetResourceResponse(resourcePath, resourceNames.get(i + successful + skip)));
}
return responses;
}
private List<Response> getResponses(final int successful, final int unsuccessful) {
private List<Response> getTemplateResponses(final int skip, final int successful, final int unsuccessful) {
return getResourceResponses("/_template/", templateNames, skip, successful, unsuccessful);
}
private List<Response> getPipelineResponses(final int skip, final int successful, final int unsuccessful) {
return getResourceResponses("/_ingest/pipeline/", pipelineNames, skip, successful, unsuccessful);
}
private List<Response> getWatcherResponses(final int skip, final int successful, final int unsuccessful) {
final List<Response> responses = new ArrayList<>(successful + unsuccessful);
for (int i = 0; i < successful; ++i) {
responses.add(successfulGetResponse());
responses.add(successfulGetWatchResponse(watchNames.get(i + skip)));
}
for (int i = 0; i < unsuccessful; ++i) {
responses.add(unsuccessfulGetResponse());
responses.add(unsuccessfulGetWatchResponse(watchNames.get(i + successful + skip)));
}
return responses;
@ -783,37 +741,8 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
when(client.performRequest(eq("GET"), eq("/"), anyMapOf(String.class, String.class))).thenReturn(versionResponse);
}
private void whenGetTypeMappingResponse(final int successful, final int unsuccessful) throws IOException {
final List<Response> gets = getTypeMappingResponses(successful, unsuccessful);
if (gets.size() == 1) {
when(client.performRequest(eq("GET"), startsWith("/" + DATA_INDEX + "/_mapping"), anyMapOf(String.class, String.class)))
.thenReturn(gets.get(0));
} else {
when(client.performRequest(eq("GET"), startsWith("/" + DATA_INDEX + "/_mapping"), anyMapOf(String.class, String.class)))
.thenReturn(gets.get(0), gets.subList(1, gets.size()).toArray(new Response[gets.size() - 1]));
}
}
private void whenSuccessfulPutTypeMappings(final int successful) throws IOException {
final List<Response> successfulPuts = successfulPutResponses(successful);
// empty is possible if they all exist
if (successful == 1) {
when(client.performRequest(eq("PUT"),
startsWith("/" + DATA_INDEX + "/_mapping"),
anyMapOf(String.class, String.class), any(HttpEntity.class)))
.thenReturn(successfulPuts.get(0));
} else if (successful > 1) {
when(client.performRequest(eq("PUT"),
startsWith("/" + DATA_INDEX + "/_mapping"),
anyMapOf(String.class, String.class), any(HttpEntity.class)))
.thenReturn(successfulPuts.get(0), successfulPuts.subList(1, successful).toArray(new Response[successful - 1]));
}
}
private void whenGetTemplates(final int successful, final int unsuccessful) throws IOException {
final List<Response> gets = getResponses(successful, unsuccessful);
final List<Response> gets = getTemplateResponses(0, successful, unsuccessful);
if (gets.size() == 1) {
when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
@ -838,7 +767,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
}
private void whenGetPipelines(final int successful, final int unsuccessful) throws IOException {
final List<Response> gets = getResponses(successful, unsuccessful);
final List<Response> gets = getPipelineResponses(0, successful, unsuccessful);
if (gets.size() == 1) {
when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
@ -903,7 +832,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
}
private void whenGetWatches(final int successful, final int unsuccessful) throws IOException {
final List<Response> gets = getResponses(successful, unsuccessful);
final List<Response> gets = getWatcherResponses(0, successful, unsuccessful);
if (gets.size() == 1) {
when(client.performRequest(eq("GET"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
@ -970,18 +899,6 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
verify(client).performRequest(eq("GET"), eq("/"), anyMapOf(String.class, String.class));
}
private void verifyGetTypeMappings(final int called) throws IOException {
verify(client, times(called))
.performRequest(eq("GET"), startsWith("/" + DATA_INDEX + "/_mapping"), anyMapOf(String.class, String.class));
}
private void verifyPutTypeMappings(final int called) throws IOException {
verify(client, times(called)).performRequest(eq("PUT"), // method
startsWith("/" + DATA_INDEX + "/_mapping"), // endpoint
anyMapOf(String.class, String.class), // parameters (e.g., timeout)
any(HttpEntity.class)); // raw template
}
private void verifyGetTemplates(final int called) throws IOException {
verify(client, times(called)).performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class));
}

22
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java
View File

@ -22,7 +22,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.Exporter.Config;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
@ -38,6 +37,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;
@ -288,6 +289,7 @@ public class HttpExporterTests extends ESTestCase {
public void testCreateResources() {
final boolean useIngest = randomBoolean();
final boolean clusterAlertManagement = randomBoolean();
final boolean createOldTemplates = randomBoolean();
final TimeValue templateTimeout = randomFrom(TimeValue.timeValueSeconds(30), null);
final TimeValue pipelineTimeout = randomFrom(TimeValue.timeValueSeconds(30), null);
final TimeValue aliasTimeout = randomFrom(TimeValue.timeValueSeconds(30), null);
@ -303,6 +305,10 @@ public class HttpExporterTests extends ESTestCase {
builder.put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", false);
}
if (createOldTemplates == false) {
builder.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", false);
}
if (templateTimeout != null) {
builder.put("xpack.monitoring.exporters._http.index.template.master_timeout", templateTimeout.getStringRep());
}
@ -322,10 +328,6 @@ public class HttpExporterTests extends ESTestCase {
final List<HttpResource> resources = multiResource.getResources();
final int version = (int)resources.stream().filter((resource) -> resource instanceof VersionHttpResource).count();
final List<DataTypeMappingHttpResource> typeMappings =
resources.stream().filter((resource) -> resource instanceof DataTypeMappingHttpResource)
.map(DataTypeMappingHttpResource.class::cast)
.collect(Collectors.toList());
final List<TemplateHttpResource> templates =
resources.stream().filter((resource) -> resource instanceof TemplateHttpResource)
.map(TemplateHttpResource.class::cast)
@ -354,11 +356,10 @@ public class HttpExporterTests extends ESTestCase {
// expected number of resources
assertThat(multiResource.getResources().size(),
equalTo(version + typeMappings.size() + templates.size() + pipelines.size() + watcherCheck.size() + bwc.size()));
equalTo(version + templates.size() + pipelines.size() + watcherCheck.size() + bwc.size()));
assertThat(version, equalTo(1));
assertThat(typeMappings, hasSize(MonitoringTemplateUtils.NEW_DATA_TYPES.length));
assertThat(templates, hasSize(6));
assertThat(pipelines, hasSize(useIngest ? 1 : 0));
assertThat(templates, hasSize(createOldTemplates ? 5 + OLD_TEMPLATE_IDS.length : 5));
assertThat(pipelines, hasSize(useIngest ? PIPELINE_IDS.length : 0));
assertThat(watcherCheck, hasSize(clusterAlertManagement ? 1 : 0));
assertThat(watches, hasSize(clusterAlertManagement ? ClusterAlertsUtil.WATCH_IDS.length : 0));
assertThat(bwc, hasSize(1));
@ -402,7 +403,8 @@ public class HttpExporterTests extends ESTestCase {
}
if (useIngest) {
assertThat(parameters.remove("pipeline"), equalTo(Exporter.EXPORT_PIPELINE_NAME));
assertThat(parameters.remove("pipeline"),
equalTo(MonitoringTemplateUtils.pipelineName(MonitoringTemplateUtils.TEMPLATE_VERSION)));
}
// should have removed everything

40
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResourceTests.java
View File

@ -8,10 +8,16 @@ package org.elasticsearch.xpack.monitoring.exporter.http;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.elasticsearch.Version;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import java.io.IOException;
import java.io.InputStream;
import java.util.function.Supplier;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.DOES_NOT_EXIST;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.ERROR;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.EXISTS;
import static org.hamcrest.Matchers.is;
/**
@ -22,6 +28,7 @@ public class PipelineHttpResourceTests extends AbstractPublishableHttpResourceTe
private final String pipelineName = ".my_pipeline";
private final byte[] pipelineBytes = new byte[] { randomByte(), randomByte(), randomByte() };
private final Supplier<byte[]> pipeline = () -> pipelineBytes;
private final int minimumVersion = randomFrom(MonitoringTemplateUtils.LAST_UPDATED_VERSION, Version.CURRENT.id);
private final PipelineHttpResource resource = new PipelineHttpResource(owner, masterTimeout, pipelineName, pipeline);
@ -41,16 +48,35 @@ public class PipelineHttpResourceTests extends AbstractPublishableHttpResourceTe
assertThat(byteStream.available(), is(0));
}
public void testDoCheckTrue() throws IOException {
assertCheckExists(resource, "/_ingest/pipeline", pipelineName);
public void testDoCheckExists() throws IOException {
final HttpEntity entity = entityForResource(EXISTS, pipelineName, minimumVersion);
doCheckWithStatusCode(resource, "/_ingest/pipeline", pipelineName, successfulCheckStatus(), EXISTS, entity);
}
public void testDoCheckFalse() throws IOException {
assertCheckDoesNotExist(resource, "/_ingest/pipeline", pipelineName);
public void testDoCheckDoesNotExist() throws IOException {
if (randomBoolean()) {
// it does not exist because it's literally not there
assertCheckDoesNotExist(resource, "/_ingest/pipeline", pipelineName);
} else {
// it does not exist because we need to replace it
final HttpEntity entity = entityForResource(DOES_NOT_EXIST, pipelineName, minimumVersion);
doCheckWithStatusCode(resource, "/_ingest/pipeline", pipelineName,
successfulCheckStatus(), DOES_NOT_EXIST, entity);
}
}
public void testDoCheckNullWithException() throws IOException {
assertCheckWithException(resource, "/_ingest/pipeline", pipelineName);
public void testDoCheckError() throws IOException {
if (randomBoolean()) {
// error because of a server error
assertCheckWithException(resource, "/_ingest/pipeline", pipelineName);
} else {
// error because of a malformed response
final HttpEntity entity = entityForResource(ERROR, pipelineName, minimumVersion);
doCheckWithStatusCode(resource, "/_ingest/pipeline", pipelineName, successfulCheckStatus(), ERROR, entity);
}
}
public void testDoPublishTrue() throws IOException {
@ -66,7 +92,7 @@ public class PipelineHttpResourceTests extends AbstractPublishableHttpResourceTe
}
public void testParameters() {
assertParameters(resource);
assertVersionParameters(resource);
}
}

153
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResourceTests.java
View File

@ -7,12 +7,16 @@ package org.elasticsearch.xpack.monitoring.exporter.http;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.SuppressLoggerChecks;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse;
@ -69,6 +73,66 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
verifyNoMoreInteractions(client, logger);
}
public void testVersionCheckForResourceExists() throws IOException {
assertVersionCheckForResource(successfulCheckStatus(), CheckResponse.EXISTS, randomInt(), "{} [{}] found on the [{}] {}");
}
public void testVersionCheckForResourceDoesNotExist() throws IOException {
if (randomBoolean()) {
// it literally does not exist
assertVersionCheckForResource(notFoundCheckStatus(), CheckResponse.DOES_NOT_EXIST,
randomInt(), "{} [{}] does not exist on the [{}] {}");
} else {
// it DOES exist, but the version needs to be replaced
assertVersionCheckForResource(successfulCheckStatus(), CheckResponse.DOES_NOT_EXIST,
randomInt(), "{} [{}] found on the [{}] {}");
}
}
public void testVersionCheckForResourceUnexpectedResponse() throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final RestStatus failedStatus = failedCheckStatus();
final Response response = response("GET", endpoint, failedStatus);
final XContent xContent = mock(XContent.class);
final int minimumVersion = randomInt();
when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
assertThat(resource.versionCheckForResource(client, logger,
resourceBasePath, resourceName, resourceType, owner, ownerType,
xContent, minimumVersion),
is(CheckResponse.ERROR));
verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), any(ResponseException.class));
verifyNoMoreInteractions(client, logger);
}
public void testVersionCheckForResourceMalformedResponse() throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final RestStatus okStatus = successfulCheckStatus();
final int minimumVersion = randomInt();
final HttpEntity entity = entityForResource(CheckResponse.ERROR, resourceName, minimumVersion);
final Response response = response("GET", endpoint, okStatus, entity);
final XContent xContent = mock(XContent.class);
when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
assertThat(resource.versionCheckForResource(client, logger,
resourceBasePath, resourceName, resourceType, owner, ownerType,
xContent, minimumVersion),
is(CheckResponse.ERROR));
verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
verify(logger).debug("{} [{}] found on the [{}] {}", resourceType, resourceName, owner, ownerType);
verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), any(ResponseException.class));
verifyNoMoreInteractions(client, logger);
}
public void testCheckForResourceErrors() throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final RestStatus failedStatus = failedCheckStatus();
@ -160,7 +224,55 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
assertThat(resource.doCheckAndPublish(client), is(exists == CheckResponse.EXISTS || publish));
}
@SuppressLoggerChecks(reason = "mock usage")
public void testShouldReplaceResourceRethrowsIOException() throws IOException {
final Response response = mock(Response.class);
final HttpEntity entity = mock(HttpEntity.class);
final XContent xContent = mock(XContent.class);
when(response.getEntity()).thenReturn(entity);
when(entity.getContent()).thenThrow(new IOException("TEST - expected"));
expectThrows(IOException.class, () -> resource.shouldReplaceResource(response, xContent, resourceName, randomInt()));
}
public void testShouldReplaceResourceThrowsExceptionForMalformedResponse() throws IOException {
final Response response = mock(Response.class);
final HttpEntity entity = entityForResource(CheckResponse.ERROR, resourceName, randomInt());
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
expectThrows(RuntimeException.class, () -> resource.shouldReplaceResource(response, xContent, resourceName, randomInt()));
}
public void testShouldReplaceResourceReturnsTrueVersionIsNotExpected() throws IOException {
final int minimumVersion = randomInt();
final Response response = mock(Response.class);
final HttpEntity entity = entityForResource(CheckResponse.DOES_NOT_EXIST, resourceName, minimumVersion);
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
assertThat(resource.shouldReplaceResource(response, xContent, resourceName, minimumVersion), is(true));
}
public void testShouldReplaceResourceChecksVersion() throws IOException {
final int minimumVersion = randomInt();
final int version = randomInt();
final boolean shouldReplace = version < minimumVersion;
final Response response = mock(Response.class);
// { "resourceName": { "version": randomLong } }
final HttpEntity entity =
new StringEntity("{\"" + resourceName + "\":{\"version\":" + version + "}}", ContentType.APPLICATION_JSON);
final XContent xContent = XContentType.JSON.xContent();
when(response.getEntity()).thenReturn(entity);
assertThat(resource.shouldReplaceResource(response, xContent, resourceName, minimumVersion), is(shouldReplace));
}
@SuppressLoggerChecks(reason = "mock logger used")
private void assertCheckForResource(final RestStatus status, final CheckResponse expected, final String debugLogMessage)
throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
@ -187,6 +299,45 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
verifyNoMoreInteractions(client, response, logger);
}
@SuppressLoggerChecks(reason = "mock logger used")
private void assertVersionCheckForResource(final RestStatus status, final CheckResponse expected,
final int minimumVersion,
final String debugLogMessage)
throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final boolean shouldReplace = status == RestStatus.OK && expected == CheckResponse.DOES_NOT_EXIST;
final HttpEntity entity = status == RestStatus.OK ? entityForResource(expected, resourceName, minimumVersion) : null;
final Response response = response("GET", endpoint, status, entity);
final XContent xContent = XContentType.JSON.xContent();
when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
assertThat(resource.versionCheckForResource(client, logger,
resourceBasePath, resourceName, resourceType, owner, ownerType,
xContent, minimumVersion),
is(expected));
verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
if (shouldReplace || expected == CheckResponse.EXISTS) {
verify(response).getStatusLine();
verify(response).getEntity();
} else if (expected == CheckResponse.DOES_NOT_EXIST) {
verify(response).getStatusLine();
} else {
verify(response).getStatusLine();
verify(response).getRequestLine();
verify(response).getHost();
verify(response).getEntity();
}
verify(logger).debug(debugLogMessage, resourceType, resourceName, owner, ownerType);
verifyNoMoreInteractions(client, response, logger);
}
private void assertPutResource(final RestStatus status, final boolean expected) throws IOException {
final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
final Response response = response("PUT", endpoint, status);

38
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResourceTests.java
View File

@ -8,11 +8,16 @@ package org.elasticsearch.xpack.monitoring.exporter.http;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.Version;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import java.io.IOException;
import java.io.InputStream;
import java.util.function.Supplier;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.DOES_NOT_EXIST;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.ERROR;
import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.EXISTS;
import static org.hamcrest.Matchers.is;
/**
@ -23,6 +28,7 @@ public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTe
private final String templateName = ".my_template";
private final String templateValue = "{\"template\":\".xyz-*\",\"mappings\":{}}";
private final Supplier<String> template = () -> templateValue;
private final int minimumVersion = randomFrom(MonitoringTemplateUtils.LAST_UPDATED_VERSION, Version.CURRENT.id);
private final TemplateHttpResource resource = new TemplateHttpResource(owner, masterTimeout, templateName, template);
@ -43,16 +49,34 @@ public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTe
assertThat(byteStream.available(), is(0));
}
public void testDoCheckTrue() throws IOException {
assertCheckExists(resource, "/_template", templateName);
public void testDoCheckExists() throws IOException {
final HttpEntity entity = entityForResource(EXISTS, templateName, minimumVersion);
doCheckWithStatusCode(resource, "/_template", templateName, successfulCheckStatus(), EXISTS, entity);
}
public void testDoCheckFalse() throws IOException {
assertCheckDoesNotExist(resource, "/_template", templateName);
public void testDoCheckDoesNotExist() throws IOException {
if (randomBoolean()) {
// it does not exist because it's literally not there
assertCheckDoesNotExist(resource, "/_template", templateName);
} else {
// it does not exist because we need to replace it
final HttpEntity entity = entityForResource(DOES_NOT_EXIST, templateName, minimumVersion);
doCheckWithStatusCode(resource, "/_template", templateName, successfulCheckStatus(), DOES_NOT_EXIST, entity);
}
}
public void testDoCheckNullWithException() throws IOException {
assertCheckWithException(resource, "/_template", templateName);
public void testDoCheckError() throws IOException {
if (randomBoolean()) {
// error because of a server error
assertCheckWithException(resource, "/_template", templateName);
} else {
// error because of a malformed response
final HttpEntity entity = entityForResource(ERROR, templateName, minimumVersion);
doCheckWithStatusCode(resource, "/_template", templateName, successfulCheckStatus(), ERROR, entity);
}
}
public void testDoPublishTrue() throws IOException {
@ -68,7 +92,7 @@ public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTe
}
public void testParameters() {
assertParameters(resource);
assertVersionParameters(resource);
}
}

97
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.exporter.local;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.elasticsearch.xpack.security.InternalClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
/**
* {@code LocalExporterIntegTestCase} offers a basis for integration tests for the {@link LocalExporter}.
*/
public abstract class LocalExporterIntegTestCase extends MonitoringIntegTestCase {
protected final String exporterName = "_local";
private static ThreadPool THREADPOOL;
private static Boolean ENABLE_WATCHER;
@BeforeClass
public static void setupThreadPool() {
THREADPOOL = new TestThreadPool(LocalExporterIntegTestCase.class.getName());
}
@AfterClass
public static void cleanUpStatic() throws Exception {
ENABLE_WATCHER = null;
if (THREADPOOL != null) {
terminate(THREADPOOL);
}
}
@Override
protected boolean enableWatcher() {
if (ENABLE_WATCHER == null) {
ENABLE_WATCHER = randomBoolean();
}
return ENABLE_WATCHER;
}
protected Settings localExporterSettings() {
return Settings.builder()
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters." + exporterName + ".type", LocalExporter.TYPE)
.put("xpack.monitoring.exporters." + exporterName + ".enabled", false)
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
.build();
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(localExporterSettings())
.build();
}
/**
* Create a new {@link LocalExporter}. Expected usage:
* <pre><code>
* final Settings settings = Settings.builder().put("xpack.monitoring.exporters._local.type", "local").build();
* try (LocalExporter exporter = createLocalExporter("_local", settings)) {
* // ...
* }
* </code></pre>
*
* @return Never {@code null}.
*/
protected LocalExporter createLocalExporter() {
final Settings settings = localExporterSettings();
final XPackLicenseState licenseState = new XPackLicenseState();
final Exporter.Config config =
new Exporter.Config(exporterName, "local",
settings, settings.getAsSettings("xpack.monitoring.exporters." + exporterName),
clusterService(), licenseState);
final CleanerService cleanerService =
new CleanerService(settings, clusterService().getClusterSettings(), THREADPOOL, licenseState);
return new LocalExporter(config, new InternalClient(settings, THREADPOOL, client()), cleanerService);
}
}

263
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java → plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
View File

@ -5,11 +5,9 @@
*/
package org.elasticsearch.xpack.monitoring.exporter.local;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.ingest.GetPipelineResponse;
@ -18,26 +16,24 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.XPackClient;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkRequestBuilder;
import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchRequest;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchResponse;
@ -46,10 +42,10 @@ import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.junit.AfterClass;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -64,96 +60,24 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.xpack.monitoring.MonitoredSystem.BEATS;
import static org.elasticsearch.xpack.monitoring.MonitoredSystem.KIBANA;
import static org.elasticsearch.xpack.monitoring.MonitoredSystem.LOGSTASH;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.DATA_INDEX;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
public class LocalExporterTests extends MonitoringIntegTestCase {
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE,
numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
private SetOnce<String> indexTimeFormat = new SetOnce<>();
private static Boolean ENABLE_WATCHER;
@AfterClass
public static void cleanUpStatic() {
ENABLE_WATCHER = null;
}
@Override
protected boolean enableWatcher() {
if (ENABLE_WATCHER == null) {
ENABLE_WATCHER = randomBoolean();
}
return ENABLE_WATCHER;
}
@Override
protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
String customTimeFormat = null;
if (randomBoolean()) {
customTimeFormat = randomFrom("YY", "YYYY", "YYYY.MM", "YYYY-MM", "MM.YYYY", "MM");
}
indexTimeFormat.set(customTimeFormat);
return super.buildTestCluster(scope, seed);
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("xpack.monitoring.exporters._local.type", LocalExporter.TYPE)
.put("xpack.monitoring.exporters._local.enabled", false)
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
.put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
.build();
}
private final String indexTimeFormat = randomFrom("YY", "YYYY", "YYYY.MM", "YYYY-MM", "MM.YYYY", "MM", null);
private void stopMonitoring() throws Exception {
// Now disabling the monitoring service, so that no more collection are started
assertAcked(client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().putNull(MonitoringSettings.INTERVAL.getKey())));
// Exporters are still enabled, allowing on-going collections to be exported without errors.
// This assertion loop waits for in flight exports to terminate. It checks that the latest
// node_stats document collected for each node is at least 10 seconds old, corresponding to
// 2 or 3 elapsed collection intervals.
final int elapsedInSeconds = 10;
final DateTime startTime = DateTime.now(DateTimeZone.UTC);
assertBusy(() -> {
IndicesExistsResponse indicesExistsResponse = client().admin().indices().prepareExists(".monitoring-*").get();
if (indicesExistsResponse.isExists()) {
ensureYellow(".monitoring-*");
refresh(".monitoring-es-*");
SearchResponse response = client().prepareSearch(".monitoring-es-*").setTypes("node_stats").setSize(0)
.addAggregation(terms("agg_nodes_ids").field("node_stats.node_id")
.subAggregation(max("agg_last_time_collected").field("timestamp")))
.get();
Terms aggregation = response.getAggregations().get("agg_nodes_ids");
for (String nodeName : internalCluster().getNodeNames()) {
String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
Terms.Bucket bucket = aggregation.getBucketByKey(nodeId);
assertTrue("No bucket found for node id [" + nodeId + "]", bucket != null);
assertTrue(bucket.getDocCount() >= 1L);
Max subAggregation = bucket.getAggregations().get("agg_last_time_collected");
DateTime lastCollection = new DateTime(Math.round(subAggregation.getValue()), DateTimeZone.UTC);
assertTrue(lastCollection.plusSeconds(elapsedInSeconds).isBefore(DateTime.now(DateTimeZone.UTC)));
}
} else {
assertTrue(DateTime.now(DateTimeZone.UTC).isAfter(startTime.plusSeconds(elapsedInSeconds)));
}
}, 30L, TimeUnit.SECONDS);
// We can now disable the exporters and reset the settings.
assertAcked(client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder()
.putNull("xpack.monitoring.exporters._local.enabled")
.putNull("xpack.monitoring.exporters._local.index.name.time_format")));
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Settings.builder().putNull(MonitoringSettings.INTERVAL.getKey())
.putNull("xpack.monitoring.exporters._local.enabled")
.putNull("xpack.monitoring.exporters._local.index.name.time_format")));
}
public void testExport() throws Exception {
@ -180,10 +104,8 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
Settings.Builder exporterSettings = Settings.builder()
.put("xpack.monitoring.exporters._local.enabled", true);
String timeFormat = indexTimeFormat.get();
if (timeFormat != null) {
exporterSettings.put("xpack.monitoring.exporters._local.index.name.time_format",
timeFormat);
if (indexTimeFormat != null) {
exporterSettings.put("xpack.monitoring.exporters._local.index.name.time_format", indexTimeFormat);
}
// local exporter is now enabled
@ -211,50 +133,54 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
});
checkMonitoringTemplates();
checkMonitoringPipeline();
checkMonitoringPipelines();
checkMonitoringAliases();
checkMonitoringMappings();
checkMonitoringDocs();
}
// monitoring service is started
exporterSettings = Settings.builder()
.put(MonitoringSettings.INTERVAL.getKey(), 3L, TimeUnit.SECONDS);
assertAcked(client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(exporterSettings));
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
final int numNodes = internalCluster().getNodeNames().length;
assertBusy(() -> {
assertThat(client().admin().indices().prepareExists(".monitoring-*").get().isExists(), is(true));
ensureYellow(".monitoring-*");
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("cluster_state")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "cluster_state"))
.get().getHits().getTotalHits(), greaterThan(0L));
assertEquals(0L, client().prepareSearch(".monitoring-es-*").setTypes("node")
.get().getHits().getTotalHits() % numNodes);
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("cluster_stats")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "cluster_stats"))
.get().getHits().getTotalHits(), greaterThan(0L));
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("index_recovery")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "index_recovery"))
.get().getHits().getTotalHits(), greaterThan(0L));
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("index_stats")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "index_stats"))
.get().getHits().getTotalHits(), greaterThan(0L));
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("indices_stats")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "indices_stats"))
.get().getHits().getTotalHits(), greaterThan(0L));
assertThat(client().prepareSearch(".monitoring-es-*").setTypes("shards")
.get().getHits().getTotalHits(), greaterThan(0L));
assertThat(client().prepareSearch(".monitoring-data-2").setTypes("cluster_info")
assertThat(client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "shards"))
.get().getHits().getTotalHits(), greaterThan(0L));
SearchResponse response = client().prepareSearch(".monitoring-es-*")
.setTypes("node_stats")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "node_stats"))
.addAggregation(terms("agg_nodes_ids").field("node_stats.node_id"))
.get();
@ -272,16 +198,47 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
}, 30L, TimeUnit.SECONDS);
checkMonitoringTemplates();
checkMonitoringPipeline();
checkMonitoringPipelines();
checkMonitoringAliases();
checkMonitoringMappings();
checkMonitoringWatches();
checkMonitoringDocs();
logger.info("All checks passed.");
} finally {
stopMonitoring();
}
// This assertion loop waits for in flight exports to terminate. It checks that the latest
// node_stats document collected for each node is at least 10 seconds old, corresponding to
// 2 or 3 elapsed collection intervals.
final int elapsedInSeconds = 10;
final DateTime startTime = DateTime.now(DateTimeZone.UTC);
assertBusy(() -> {
IndicesExistsResponse indicesExistsResponse = client().admin().indices().prepareExists(".monitoring-*").get();
if (indicesExistsResponse.isExists()) {
ensureYellow(".monitoring-*");
refresh(".monitoring-es-*");
SearchResponse response = client().prepareSearch(".monitoring-es-*")
.setSize(0)
.setQuery(QueryBuilders.termQuery("type", "node_stats"))
.addAggregation(terms("agg_nodes_ids").field("node_stats.node_id")
.subAggregation(max("agg_last_time_collected").field("timestamp")))
.get();
Terms aggregation = response.getAggregations().get("agg_nodes_ids");
for (String nodeName : internalCluster().getNodeNames()) {
String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
Terms.Bucket bucket = aggregation.getBucketByKey(nodeId);
assertTrue("No bucket found for node id [" + nodeId + "]", bucket != null);
assertTrue(bucket.getDocCount() >= 1L);
Max subAggregation = bucket.getAggregations().get("agg_last_time_collected");
DateTime lastCollection = new DateTime(Math.round(subAggregation.getValue()), DateTimeZone.UTC);
assertTrue(lastCollection.plusSeconds(elapsedInSeconds).isBefore(DateTime.now(DateTimeZone.UTC)));
}
} else {
assertTrue(DateTime.now(DateTimeZone.UTC).isAfter(startTime.plusSeconds(elapsedInSeconds)));
}
}, 30L, TimeUnit.SECONDS);
}
/**
@ -289,10 +246,9 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
*/
private void checkMonitoringTemplates() {
final Set<String> templates = new HashSet<>();
templates.add(".monitoring-data-2");
templates.add(".monitoring-alerts-2");
templates.add(".monitoring-alerts");
for (MonitoredSystem system : MonitoredSystem.values()) {
templates.add(String.join("-", ".monitoring", system.getSystem(), "2"));
templates.add(String.join("-", ".monitoring", system.getSystem()));
}
GetIndexTemplatesResponse response =
@ -303,11 +259,18 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
}
/**
* Checks that the monitoring ingest pipeline have been created by the local exporter
* Checks that the monitoring ingest pipelines have been created by the local exporter
*/
private void checkMonitoringPipeline() {
GetPipelineResponse response =
client().admin().cluster().prepareGetPipeline(Exporter.EXPORT_PIPELINE_NAME).get();
private void checkMonitoringPipelines() {
final Set<String> expectedPipelines =
Arrays.stream(PIPELINE_IDS).map(MonitoringTemplateUtils::pipelineName).collect(Collectors.toSet());
final GetPipelineResponse response = client().admin().cluster().prepareGetPipeline("xpack_monitoring_*").get();
// actual pipelines
final Set<String> pipelines = response.pipelines().stream().map(PipelineConfiguration::getId).collect(Collectors.toSet());
assertEquals("Missing expected pipelines", expectedPipelines, pipelines);
assertTrue("monitoring ingest pipeline not found", response.isFound());
}
@ -323,26 +286,11 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
assertEquals("marvel index should have at least 1 alias: " + index, 1, aliases.size());
String indexDate = index.substring(".marvel-es-1-".length());
String expectedAlias = ".monitoring-es-" + TEMPLATE_VERSION + "-" + indexDate + "-alias";
String expectedAlias = ".monitoring-es-" + OLD_TEMPLATE_VERSION + "-" + indexDate + "-alias";
assertEquals(expectedAlias, aliases.get(0).getAlias());
}
}
/**
* Checks that the local exporter correctly updated the mappings of an existing data index.
*/
private void checkMonitoringMappings() {
IndicesExistsResponse exists = client().admin().indices().prepareExists(DATA_INDEX).get();
if (exists.isExists()) {
GetMappingsResponse response =
client().admin().indices().prepareGetMappings(DATA_INDEX).get();
for (String mapping : MonitoringTemplateUtils.NEW_DATA_TYPES) {
assertTrue("mapping [" + mapping + "] should exist in data index",
response.getMappings().get(DATA_INDEX).containsKey(mapping));
}
}
}
/**
* Checks that the local exporter correctly creates Watches.
*/
@ -368,7 +316,7 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
ClusterStateResponse response = client().admin().cluster().prepareState().get();
String customTimeFormat = response.getState().getMetaData().transientSettings()
.get("xpack.monitoring.exporters._local.index.name.time_format");
assertEquals(indexTimeFormat.get(), customTimeFormat);
assertEquals(indexTimeFormat, customTimeFormat);
if (customTimeFormat == null) {
customTimeFormat = "YYYY.MM.dd";
}
@ -380,44 +328,35 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
assertThat(searchResponse.getHits().getTotalHits(), greaterThan(0L));
for (SearchHit hit : searchResponse.getHits().getHits()) {
Map<String, Object> source = hit.getSourceAsMap();
final Map<String, Object> source = hit.getSourceAsMap();
assertTrue(source != null && source.isEmpty() == false);
String clusterUUID = (String) source.get("cluster_uuid");
assertTrue("document is missing cluster_uuid field", Strings.hasText(clusterUUID));
final String timestamp = (String) source.get("timestamp");
final String type = (String) source.get("type");
String timestamp = (String) source.get("timestamp");
assertTrue("document is missing cluster_uuid field", Strings.hasText((String) source.get("cluster_uuid")));
assertTrue("document is missing timestamp field", Strings.hasText(timestamp));
String type = hit.getType();
assertTrue(Strings.hasText(type));
assertTrue("document is missing type field", Strings.hasText(type));
assertEquals("document _type is 'doc'", "doc", hit.getType());
@SuppressWarnings("unchecked")
Map<String, Object> docSource = (Map<String, Object>) source.get("doc");
boolean isData;
MonitoredSystem expectedSystem;
if (docSource == null) {
// This is a document indexed by the Monitoring service
expectedSystem = MonitoredSystem.ES;
isData = "cluster_info".equals(type);
} else {
// This is a document indexed through the Monitoring Bulk API
expectedSystem = MonitoredSystem.fromSystem((String) docSource.get("expected_system"));
isData = (Boolean) docSource.getOrDefault("is_data", false);
}
Set<String> expectedIndex = new HashSet<>();
if (isData) {
expectedIndex.add(".monitoring-data-2");
} else {
String dateTime = dateFormatter.print(dateParser.parseDateTime(timestamp));
expectedIndex.add(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-" + dateTime);
if ("node".equals(type)) {
expectedIndex.add(".monitoring-data-2");
}
}
String dateTime = dateFormatter.print(dateParser.parseDateTime(timestamp));
expectedIndex.add(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-" + dateTime);
assertTrue("Expected " + expectedIndex + " but got " + hit.getIndex(), expectedIndex.contains(hit.getIndex()));
@SuppressWarnings("unchecked")
@ -431,17 +370,14 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
private static MonitoringBulkDoc createMonitoringBulkDoc(String id) throws IOException {
String monitoringId = randomFrom(BEATS, KIBANA, LOGSTASH).getSystem();
String monitoringVersion = TEMPLATE_VERSION;
MonitoringIndex index = randomFrom(MonitoringIndex.values());
XContentType xContentType = randomFrom(XContentType.values());
BytesReference source;
try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
builder.startObject();
{
builder.field("expected_system", monitoringId);
if (index == MonitoringIndex.DATA) {
builder.field("is_data", true);
}
final int nbFields = randomIntBetween(1, 3);
for (int i = 0; i < nbFields; i++) {
builder.field("field_" + i, i);
@ -451,6 +387,7 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
source = builder.bytes();
}
return new MonitoringBulkDoc(monitoringId, monitoringVersion, index, "doc", id, source, xContentType);
return new MonitoringBulkDoc(monitoringId, monitoringVersion, MonitoringIndex.TIMESTAMPED, "doc", id, source, xContentType);
}
}

297
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java Normal file
View File

@ -0,0 +1,297 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.exporter.local;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.common.text.TextTemplate;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
import org.elasticsearch.xpack.watcher.client.WatchSourceBuilders;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.NeverCondition;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchResponse;
import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchRequest;
import org.elasticsearch.xpack.watcher.trigger.manual.ManualTrigger;
import org.elasticsearch.xpack.watcher.watch.Payload;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.ClusterScope(scope = TEST,
numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase {
private final MonitoredSystem system = randomFrom(MonitoredSystem.values());
private final MockTimestampedIndexNameResolver resolver =
new MockTimestampedIndexNameResolver(system, Settings.EMPTY, MonitoringTemplateUtils.TEMPLATE_VERSION);
public void testCreateWhenResourcesNeedToBeAddedOrUpdated() throws Exception {
// sometimes they need to be added; sometimes they need to be replaced
if (randomBoolean()) {
putResources(oldVersion());
}
assertResourcesExist();
}
public void testCreateWhenResourcesShouldNotBeReplaced() throws Exception {
putResources(newEnoughVersion());
assertResourcesExist();
// these were "newer" or at least the same version, so they shouldn't be replaced
assertTemplateNotUpdated();
assertPipelinesNotUpdated();
assertWatchesNotUpdated();
}
private void createResources() throws Exception {
// wait until the cluster is ready (this is done at the "Exporters" level)
// this is not a busy assertion because it's checked earlier
assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION));
try (LocalExporter exporter = createLocalExporter()) {
assertBusy(() -> assertThat(exporter.isExporterReady(), is(true)));
}
}
/**
* Generates a basic template that loosely represents a monitoring template.
*/
private static BytesReference generateTemplateSource(final String name, final Integer version) throws IOException {
final XContentBuilder builder = jsonBuilder().startObject();
// this would totally break Monitoring UI, but the idea is that it's different from a real template and
// the version controls that; it also won't break indexing (just searching) so this test can use it blindly
builder
.field("index_patterns", name)
.startObject("settings")
.field("index.number_of_shards", 1)
.field("index.number_of_replicas", 0)
.endObject()
.startObject("mappings")
.startObject("doc")
.startObject("_meta")
.field("test", true)
.endObject()
.field("enabled", false)
.endObject()
.endObject();
if (version != null) {
builder.field("version", version);
}
return builder.endObject().bytes();
}
private void putResources(final Integer version) throws Exception {
waitNoPendingTasksOnAll();
putTemplate(version);
putPipelines(version);
putWatches(version);
}
private void putTemplate(final Integer version) throws Exception {
final BytesReference source = generateTemplateSource(resolver.templateName(), version);
assertAcked(client().admin().indices().preparePutTemplate(resolver.templateName()).setSource(source, XContentType.JSON).get());
}
private void putPipelines(final Integer version) throws Exception {
for (final String pipelineId : MonitoringTemplateUtils.PIPELINE_IDS) {
putPipeline(MonitoringTemplateUtils.pipelineName(pipelineId), version);
}
}
private void putPipeline(final String pipelineName, final Integer version) throws Exception {
assertAcked(client().admin().cluster().preparePutPipeline(pipelineName, replaceablePipeline(version), XContentType.JSON).get());
}
private void putWatches(final Integer version) throws Exception {
if (enableWatcher()) {
// wait until the cluster is ready so that we can get the cluster's UUID
assertBusy(() -> assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION)));
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
putWatch(ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId), version);
}
}
}
private void putWatch(final String watchName, final Integer version) throws Exception {
final WatcherClient watcherClient = new WatcherClient(client());
assertThat(watcherClient.putWatch(new PutWatchRequest(watchName, replaceableWatch(version))).get().isCreated(), is(true));
}
/**
* Create a Watch with nothing in it that will never fire its action, with a metadata field named after {@linkplain #getTestName()}.
*
* @param version Version to add to the watch, if any
* @return Never {@code null}.
*/
private WatchSourceBuilder replaceableWatch(final Integer version) {
final WatchSourceBuilder builder = WatchSourceBuilders.watchBuilder();
final Map<String, Object> metadata = new HashMap<>();
// add a marker so that we know this test made the watch
metadata.put(getTestName(), true);
if (version != null) {
metadata.put("xpack", MapBuilder.<String, Object>newMapBuilder().put("version_created", version).map());
}
builder.metadata(metadata);
builder.trigger(new ManualTrigger());
builder.input(new SimpleInput(Payload.EMPTY));
builder.condition(NeverCondition.INSTANCE);
builder.addAction("_action", LoggingAction.builder(new TextTemplate("never runs")));
return builder;
}
/**
* Create a pipeline with nothing in it whose description is literally "test".
*
* @param version Version to add to the pipeline, if any
* @return Never {@code null}.
*/
private BytesReference replaceablePipeline(final Integer version) {
try {
final XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
builder.startObject();
{
builder.startArray("processors").endArray();
// something we can quickly check to ensure we have/have not replaced it
builder.field("description", getTestName());
// sometimes give it a version that should be overwritten (and sometimes don't give it a version at all)
if (version != null) {
builder.field("version", version);
}
}
return builder.endObject().bytes();
} catch (final IOException e) {
throw new RuntimeException("Failed to create pipeline", e);
}
}
private Integer oldVersion() {
final int minimumVersion = Math.min(ClusterAlertsUtil.LAST_UPDATED_VERSION, MonitoringTemplateUtils.LAST_UPDATED_VERSION);
// randomly supply an older version, or no version at all
return randomBoolean() ? minimumVersion - randomIntBetween(1, 100000) : null;
}
private int newEnoughVersion() {
final int maximumVersion = Math.max(ClusterAlertsUtil.LAST_UPDATED_VERSION, MonitoringTemplateUtils.LAST_UPDATED_VERSION);
// randomly supply a newer version or the expected version
return randomFrom(maximumVersion + randomIntBetween(1, 100000), maximumVersion);
}
private void assertTemplatesExist() {
for (String templateName : monitoringTemplateNames()) {
assertTemplateInstalled(templateName);
}
}
private void assertPipelinesExist() {
for (PipelineConfiguration pipeline : client().admin().cluster().prepareGetPipeline("xpack_monitoring_*").get().pipelines()) {
final Object description = pipeline.getConfigAsMap().get("description");
// this just ensures that it's set; not who set it
assertThat(description, notNullValue());
}
}
private void assertWatchesExist() {
if (enableWatcher()) {
final WatcherClient watcherClient = new WatcherClient(client());
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
final String watchName = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
final GetWatchResponse response = watcherClient.prepareGetWatch(watchName).get();
// this just ensures that it's set; not who set it
assertThat(response.isFound(), is(true));
}
}
}
private void assertResourcesExist() throws Exception {
createResources();
waitNoPendingTasksOnAll();
assertBusy(() -> {
assertTemplatesExist();
assertPipelinesExist();
assertWatchesExist();
});
}
private void assertTemplateNotUpdated() {
final String name = resolver.templateName();
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
final String docMapping = template.getMappings().get("doc").toString();
assertThat(docMapping, notNullValue());
assertThat(docMapping, containsString("test"));
}
}
private void assertPipelinesNotUpdated() {
for (PipelineConfiguration pipeline : client().admin().cluster().prepareGetPipeline("xpack_monitoring_*").get().pipelines()) {
final Object description = pipeline.getConfigAsMap().get("description");
assertThat(description, equalTo(getTestName()));
}
}
private void assertWatchesNotUpdated() throws Exception {
if (enableWatcher()) {
final WatcherClient watcherClient = new WatcherClient(client());
for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
final String watchName = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
final GetWatchResponse response = watcherClient.prepareGetWatch(watchName).get();
// ensure that the watch still contains a value associated with the test rather than the real watch
assertThat(response.getSource().getValue("metadata." + getTestName()), notNullValue());
}
}
}
}

263
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTemplateTests.java
View File

@ -1,263 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.exporter.local;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.Collector;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsCollector;
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
import org.elasticsearch.xpack.monitoring.exporter.Exporters;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.elasticsearch.xpack.security.InternalClient;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public class LocalExporterTemplateTests extends MonitoringIntegTestCase {
private final Settings localExporter = Settings.builder().put("type", LocalExporter.TYPE).build();
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters._exporter.type", LocalExporter.TYPE);
return settings.build();
}
@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1410")
public void testCreateWhenExistingTemplatesAreOld() throws Exception {
internalCluster().startNode();
// put an old variant of the monitoring-data-# index so that types need to be added
final CreateIndexRequest request = new CreateIndexRequest(MonitoringTemplateUtils.DATA_INDEX);
request.settings(Settings.builder().put("index.mapper.dynamic", false).build());
// notably absent are: kibana, logstash, and beats
request.mapping("cluster_info", "{\"enabled\": false}", XContentType.JSON);
request.mapping("node", "{\"enabled\": false}", XContentType.JSON);
request.mapping("fake", "{\"enabled\": false}", XContentType.JSON);
client().admin().indices().create(request).actionGet();
putTemplate(indexTemplateName());
putTemplate(dataTemplateName());
putPipeline(Exporter.EXPORT_PIPELINE_NAME);
doExporting();
logger.debug("--> existing templates are old");
assertTemplateExists(dataTemplateName());
assertTemplateExists(indexTemplateName());
logger.debug("--> existing templates are old: new templates should be created");
for (String template : monitoringTemplateNames()) {
assertTemplateExists(template);
}
assertPipelineExists(Exporter.EXPORT_PIPELINE_NAME);
doExporting();
logger.debug("--> indices should have been created");
awaitIndexExists(currentDataIndexName());
assertIndicesExists(currentTimestampedIndexName());
// ensure that it added mapping types to monitoring-data-2, without throwing away the index
assertMapping(MonitoringTemplateUtils.DATA_INDEX, "fake");
for (final String type : MonitoringTemplateUtils.NEW_DATA_TYPES) {
assertMapping(MonitoringTemplateUtils.DATA_INDEX, type);
}
}
private void assertMapping(final String index, final String type) throws Exception {
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
assertThat(mappings, notNullValue());
MappingMetaData mappingMetaData = mappings.get(type);
assertThat(mappingMetaData, notNullValue());
}
public void testCreateWhenExistingTemplateAreUpToDate() throws Exception {
internalCluster().startNode();
putTemplate(indexTemplateName());
putTemplate(dataTemplateName());
putPipeline(Exporter.EXPORT_PIPELINE_NAME);
doExporting();
logger.debug("--> existing templates are up to date");
for (String template : monitoringTemplateNames()) {
assertTemplateExists(template);
}
assertPipelineExists(Exporter.EXPORT_PIPELINE_NAME);
logger.debug("--> existing templates has the same version: they should not be changed");
assertTemplateNotUpdated(indexTemplateName());
assertTemplateNotUpdated(dataTemplateName());
assertPipelineNotUpdated(Exporter.EXPORT_PIPELINE_NAME);
doExporting();
logger.debug("--> indices should have been created");
awaitIndexExists(currentDataIndexName());
awaitIndexExists(currentTimestampedIndexName());
}
protected void doExporting() throws Exception {
// TODO: these should be unit tests, not using guice (copied from now-deleted AbstractExporterTemplateTestCase)
final String node = randomFrom(internalCluster().getNodeNames());
ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node);
XPackLicenseState licenseState = internalCluster().getInstance(XPackLicenseState.class, node);
LicenseService licenseService = internalCluster().getInstance(LicenseService.class, node);
InternalClient client = internalCluster().getInstance(InternalClient.class, node);
Collector collector = new ClusterStatsCollector(clusterService.getSettings(), clusterService,
new MonitoringSettings(clusterService.getSettings(), clusterService.getClusterSettings()),
licenseState, client, licenseService);
Exporters exporters = internalCluster().getInstance(Exporters.class, node);
assertNotNull(exporters);
Exporter exporter = exporters.getExporter("_exporter");
// Wait for exporting bulks to be ready to export
Runnable busy = () -> assertThat(exporter.openBulk(), notNullValue());
assertBusy(busy);
PlainActionFuture<Void> future = new PlainActionFuture<>();
exporters.export(collector.collect(), future);
future.get();
}
private String dataTemplateName() {
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(MonitoringTemplateUtils.TEMPLATE_VERSION);
return resolver.templateName();
}
private String indexTemplateName() {
MockTimestampedIndexNameResolver resolver =
new MockTimestampedIndexNameResolver(MonitoredSystem.ES, localExporter, MonitoringTemplateUtils.TEMPLATE_VERSION);
return resolver.templateName();
}
private String currentDataIndexName() {
return MonitoringTemplateUtils.DATA_INDEX;
}
private String currentTimestampedIndexName() {
MonitoringDoc doc = new MonitoringDoc(MonitoredSystem.ES.getSystem(), Version.CURRENT
.toString(), null, null, null, System.currentTimeMillis(),
(MonitoringDoc.Node) null);
MockTimestampedIndexNameResolver resolver =
new MockTimestampedIndexNameResolver(MonitoredSystem.ES, localExporter, MonitoringTemplateUtils.TEMPLATE_VERSION);
return resolver.index(doc);
}
/** Generates a basic template **/
private BytesReference generateTemplateSource(String name) throws IOException {
return jsonBuilder().startObject()
.field("template", name)
.startObject("settings")
.field("index.number_of_shards", 1)
.field("index.number_of_replicas", 1)
.field("index.mapping.single_type", false)
.endObject()
.startObject("mappings")
.startObject("_default_")
.field("date_detection", false)
.startObject("properties")
.startObject("cluster_uuid")
.field("type", "keyword")
.endObject()
.startObject("timestamp")
.field("type", "date")
.field("format", "date_time")
.endObject()
.endObject()
.endObject()
.startObject("cluster_info")
.field("enabled", false)
.endObject()
.startObject("cluster_stats")
.startObject("properties")
.startObject("cluster_stats")
.field("type", "object")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject().bytes();
}
private void putTemplate(String name) throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name), XContentType.JSON).get());
}
private void putPipeline(String name) throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().cluster().preparePutPipeline(name, Exporter.emptyPipeline(XContentType.JSON).bytes(),
XContentType.JSON).get());
}
private void assertTemplateExists(String name) throws Exception {
waitNoPendingTasksOnAll();
waitForMonitoringTemplate(name);
}
private void assertPipelineExists(String name) throws Exception {
waitNoPendingTasksOnAll();
assertPipelineInstalled(name);
}
private void assertPipelineInstalled(String name) throws Exception {
assertBusy(() -> {
boolean found = false;
for (PipelineConfiguration pipeline : client().admin().cluster().prepareGetPipeline(name).get().pipelines()) {
if (Regex.simpleMatch(name, pipeline.getId())) {
found = true;
}
}
assertTrue("failed to find a pipeline matching [" + name + "]", found);
}, 60, TimeUnit.SECONDS);
}
private void assertTemplateNotUpdated(String name) throws Exception {
waitNoPendingTasksOnAll();
assertTemplateExists(name);
}
private void assertPipelineNotUpdated(String name) throws Exception {
waitNoPendingTasksOnAll();
assertPipelineExists(name);
}
}

52
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/DataResolverTests.java
View File

@ -1,52 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import java.io.IOException;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
public class DataResolverTests extends MonitoringIndexNameResolverTestCase {
@Override
protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
return newDataResolver();
}
@Override
protected MonitoringDoc newMonitoringDoc() {
MonitoringDoc doc = new MonitoringDoc(randomMonitoringId(), randomAlphaOfLength(2),
null, null, randomAlphaOfLength(5), Math.abs(randomLong()),
new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT));
return doc;
}
@Override
protected boolean checkFilters() {
return false;
}
public void testDataResolver() {
assertThat(newDataResolver().index(newMonitoringDoc()), equalTo(".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
}
private MonitoringIndexNameResolver.Data<MonitoringDoc> newDataResolver() {
return new MonitoringIndexNameResolver.Data<MonitoringDoc>() {
@Override
protected void buildXContent(MonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
}
};
}
}

11
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/MonitoringIndexNameResolverTestCase.java
View File

@ -24,6 +24,7 @@ import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameRes
import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.CLUSTER_UUID;
import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.SOURCE_NODE;
import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.TIMESTAMP;
import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.TYPE;
import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.PREFIX;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.anyOf;
@ -117,11 +118,6 @@ public abstract class MonitoringIndexNameResolverTestCase<M extends MonitoringDo
assertThat(resolver.index(newMonitoringDoc()),
startsWith(PREFIX + DELIMITER + timestamped.getId() + DELIMITER + TEMPLATE_VERSION + DELIMITER));
}
if (resolver instanceof MonitoringIndexNameResolver.Data) {
assertThat(resolver.index(newMonitoringDoc()),
equalTo(PREFIX + DELIMITER + MonitoringIndexNameResolver.Data.DATA + DELIMITER + TEMPLATE_VERSION));
}
}
protected void assertSource(BytesReference source, Set<String> fields, XContentType xContentType) {
@ -131,9 +127,12 @@ public abstract class MonitoringIndexNameResolverTestCase<M extends MonitoringDo
String[] commons = new String[]{
CLUSTER_UUID,
TIMESTAMP,
TYPE,
SOURCE_NODE,
};
assertThat("source must contains default fields", sourceFields.keySet(), hasItems(commons));
assertThat("source must contain default fields", sourceFields.keySet(), hasItems(commons));
assertThat("type is set improperly", sourceFields.get(TYPE), equalTo(newMonitoringDoc().getType()));
if (fields != null && fields.isEmpty() == false) {
for (String field : fields) {

56
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/bulk/MonitoringBulkDataResolverTests.java
View File

@ -1,56 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.bulk;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDocTests;
import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
import static org.hamcrest.Matchers.equalTo;
/**
* Tests {@link MonitoringBulkDataResolver}.
*/
public class MonitoringBulkDataResolverTests extends MonitoringIndexNameResolverTestCase<MonitoringBulkDoc, MonitoringBulkDataResolver> {
private final String id = randomBoolean() ? randomAlphaOfLength(35) : null;
@Override
protected MonitoringBulkDoc newMonitoringDoc() {
MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(),
MonitoringTemplateUtils.TEMPLATE_VERSION, MonitoringIndex.DATA, "kibana", id,
randomAlphaOfLength(5), Math.abs(randomLong()),
MonitoringBulkDocTests.newRandomSourceNode(),
new BytesArray("{\"field1\" : \"value1\"}"), XContentType.JSON);
return doc;
}
@Override
protected boolean checkFilters() {
return false;
}
public void testMonitoringBulkResolver() throws Exception {
MonitoringBulkDoc doc = newMonitoringDoc();
MonitoringBulkDataResolver resolver = newResolver(doc);
assertThat(resolver.index(doc), equalTo(".monitoring-data-2"));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"kibana",
"kibana.field1"), XContentType.JSON);
}
}

3
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/bulk/MonitoringBulkTimestampedResolverTests.java
View File

@ -44,12 +44,13 @@ public class MonitoringBulkTimestampedResolverTests
MonitoringBulkDoc doc = newMonitoringDoc();
MonitoringBulkTimestampedResolver resolver = newResolver(doc);
assertThat(resolver.index(doc), equalTo(".monitoring-kibana-2-2015.07.22"));
assertThat(resolver.index(doc), equalTo(".monitoring-kibana-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"kibana_stats",
"kibana_stats.field1"), XContentType.JSON);

87
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoResolverTests.java
View File

@ -1,87 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
import java.util.Collections;
import java.util.UUID;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
public class ClusterInfoResolverTests extends MonitoringIndexNameResolverTestCase<ClusterInfoMonitoringDoc, ClusterInfoResolver> {
@Override
protected ClusterInfoMonitoringDoc newMonitoringDoc() {
try {
License.Builder licenseBuilder = License.builder()
.uid(UUID.randomUUID().toString())
.type("trial")
.issuer("elasticsearch")
.issuedTo("customer")
.maxNodes(5)
.issueDate(1437580442979L)
.expiryDate(1437580442979L + TimeValue.timeValueHours(2).getMillis());
ClusterInfoMonitoringDoc doc = new ClusterInfoMonitoringDoc(randomMonitoringId(),
randomAlphaOfLength(2), randomAlphaOfLength(5),
Math.abs(randomLong()),
new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
randomAlphaOfLength(5),
randomFrom(Version.V_5_0_0, Version.CURRENT).toString(),
licenseBuilder.build(),
Collections.singletonList(new MonitoringFeatureSet.Usage(randomBoolean(), randomBoolean(), emptyMap())),
new ClusterStatsResponse(
Math.abs(randomLong()),
ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY),
Collections.emptyList(),
Collections.emptyList())
);
return doc;
} catch (Exception e) {
throw new IllegalStateException("Failed to generated random ClusterInfoMonitoringDoc", e);
}
}
@Override
protected boolean checkFilters() {
return false;
}
public void testClusterInfoResolver() throws Exception {
ClusterInfoMonitoringDoc doc = newMonitoringDoc();
ClusterInfoResolver resolver = newResolver();
assertThat(resolver.index(doc), equalTo(".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"cluster_name",
"version",
"license",
"cluster_stats",
"stack_stats.xpack"), XContentType.JSON);
}
}

154
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoTests.java
View File

@ -1,154 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.license.License;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.junit.After;
import org.junit.Before;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.isEmptyOrNullString;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(scope = TEST)
public class ClusterInfoTests extends MonitoringIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.build();
}
@Before
public void init() throws Exception {
updateMonitoringInterval(3L, TimeUnit.SECONDS);
}
@After
public void cleanup() throws Exception {
disableMonitoringInterval();
wipeMonitoringIndices();
}
public void testClusterInfo() throws Exception {
ensureGreen();
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
// waiting for the monitoring data index to be created (it should have been created by the ClusterInfoCollector
String dataIndex = ".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION;
awaitIndexExists(dataIndex);
// waiting for cluster info collector to collect data
awaitMonitoringDocsCountOnPrimary(equalTo(1L), ClusterInfoMonitoringDoc.TYPE);
// retrieving cluster info document
GetResponse response = client().prepareGet(dataIndex, ClusterInfoMonitoringDoc.TYPE, clusterUUID).setPreference("_primary").get();
assertTrue("cluster_info document does not exist in data index", response.isExists());
assertThat(response.getIndex(), equalTo(dataIndex));
assertThat(response.getType(), equalTo(ClusterInfoMonitoringDoc.TYPE));
assertThat(response.getId(), equalTo(clusterUUID));
Map<String, Object> source = response.getSource();
assertThat(source.get(MonitoringIndexNameResolver.Fields.CLUSTER_UUID), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.TIMESTAMP), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.SOURCE_NODE), notNullValue());
assertThat(source.get("cluster_name"), equalTo(cluster().getClusterName()));
assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
Object licenseObj = source.get("license");
assertThat(licenseObj, instanceOf(Map.class));
Map license = (Map) licenseObj;
assertThat(license, instanceOf(Map.class));
String uid = (String) license.get("uid");
assertThat(uid, not(isEmptyOrNullString()));
String type = (String) license.get("type");
assertThat(type, not(isEmptyOrNullString()));
String status = (String) license.get(License.Fields.STATUS);
assertThat(status, not(isEmptyOrNullString()));
Long expiryDate = (Long) license.get(License.Fields.EXPIRY_DATE_IN_MILLIS);
assertThat(expiryDate, greaterThan(0L));
// We basically recompute the hash here
String hkey = (String) license.get("hkey");
String recalculated = ClusterInfoResolver.hash(status, uid, type, String.valueOf(expiryDate), clusterUUID);
assertThat(hkey, equalTo(recalculated));
assertThat((String) license.get(License.Fields.ISSUER), not(isEmptyOrNullString()));
assertThat((String) license.get(License.Fields.ISSUED_TO), not(isEmptyOrNullString()));
assertThat((Long) license.get(License.Fields.ISSUE_DATE_IN_MILLIS), greaterThan(0L));
assertThat((Integer) license.get(License.Fields.MAX_NODES), greaterThan(0));
Object clusterStats = source.get("cluster_stats");
assertNotNull(clusterStats);
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
Object stackStats = source.get("stack_stats");
assertNotNull(stackStats);
assertThat(stackStats, instanceOf(Map.class));
assertThat(((Map) stackStats).size(), equalTo(1));
Object xpack = ((Map)stackStats).get("xpack");
assertNotNull(xpack);
assertThat(xpack, instanceOf(Map.class));
// it must have at least monitoring, but others may be hidden
assertThat(((Map) xpack).size(), greaterThanOrEqualTo(1));
Object monitoring = ((Map)xpack).get("monitoring");
assertNotNull(monitoring);
// we don't make any assumptions about what's in it, only that it's there
assertThat(monitoring, instanceOf(Map.class));
waitForMonitoringTemplates();
// check that the cluster_info is not indexed
flush();
refresh();
assertHitCount(client().prepareSearch().setSize(0)
.setIndices(dataIndex)
.setTypes(ClusterInfoMonitoringDoc.TYPE)
.setPreference("_primary")
.setQuery(
QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.ACTIVE.label()))
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.INVALID.label()))
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.EXPIRED.label()))
.should(QueryBuilders.matchQuery("cluster_name", cluster().getClusterName()))
.minimumShouldMatch(1)
).get(), 0L);
}
}

54
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateNodeResolverTests.java
View File

@ -1,54 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
import java.util.UUID;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
public class ClusterStateNodeResolverTests extends
MonitoringIndexNameResolverTestCase<ClusterStateNodeMonitoringDoc, ClusterStateNodeResolver> {
@Override
protected ClusterStateNodeMonitoringDoc newMonitoringDoc() {
ClusterStateNodeMonitoringDoc doc = new ClusterStateNodeMonitoringDoc(randomMonitoringId(),
randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
UUID.randomUUID().toString(), randomAlphaOfLength(5));
return doc;
}
@Override
protected boolean checkFilters() {
return false;
}
public void testClusterStateNodeResolver() throws Exception {
ClusterStateNodeMonitoringDoc doc = newMonitoringDoc();
ClusterStateNodeResolver resolver = newResolver();
assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"state_uuid",
"node.id"), XContentType.JSON);
}
}

2
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateResolverTests.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
public class ClusterStateResolverTests extends MonitoringIndexNameResolverTestCase<ClusterStateMonitoringDoc, ClusterStateResolver> {
@ -52,6 +51,7 @@ public class ClusterStateResolverTests extends MonitoringIndexNameResolverTestCa
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"cluster_state"), XContentType.JSON);
}

165
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java
View File

@ -5,38 +5,45 @@
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateCollector;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.elasticsearch.xpack.security.InternalClient;
import org.junit.After;
import org.junit.Before;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
//test is just too slow, please fix it to not be sleep-based
@LuceneTestCase.BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ClusterScope(scope = Scope.TEST)
public class ClusterStateTests extends MonitoringIntegTestCase {
private int randomInt = randomInt();
private ThreadPool threadPool = null;
@Before
public void setupThreadPool() {
threadPool = new TestThreadPool(getTestName());
}
@After
public void removeThreadPool() throws InterruptedException {
if (threadPool != null) {
terminate(threadPool);
}
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -48,105 +55,37 @@ public class ClusterStateTests extends MonitoringIntegTestCase {
.build();
}
@Before
public void init() throws Exception {
updateMonitoringInterval(3L, TimeUnit.SECONDS);
waitForMonitoringIndices();
}
@After
public void cleanup() throws Exception {
disableMonitoringInterval();
wipeMonitoringIndices();
}
public void testClusterState() throws Exception {
logger.debug("--> waiting for documents to be collected");
awaitMonitoringDocsCountOnPrimary(greaterThan(0L), ClusterStateResolver.TYPE);
final String masterNodeName = internalCluster().getMasterName();
final MonitoringSettings monitoringSettings = new MonitoringSettings(Settings.EMPTY, clusterService().getClusterSettings());
final InternalClient client = new InternalClient(Settings.EMPTY, threadPool, internalCluster().client(masterNodeName));
final ClusterStateCollector collector =
new ClusterStateCollector(Settings.EMPTY,
internalCluster().clusterService(masterNodeName),
monitoringSettings, new XPackLicenseState(), client);
logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateResolver.TYPE).setPreference("_primary").get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
final Collection<MonitoringDoc> monitoringDocs = collector.collect();
logger.debug("--> checking that every document contains the expected fields");
Set<String> filters = ClusterStateResolver.FILTERS;
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.getSourceAsMap();
// just one cluster state
assertThat(monitoringDocs, hasSize(1));
for (String filter : filters) {
assertContains(filter, fields);
}
// get the cluster state document that we fetched
final ClusterStateMonitoringDoc clusterStateDoc = (ClusterStateMonitoringDoc)monitoringDocs.iterator().next();
assertThat(clusterStateDoc.getClusterState(), notNullValue());
assertThat(clusterStateDoc.getStatus(), notNullValue());
// turn the monitoring doc into JSON
final ClusterStateResolver resolver = new ClusterStateResolver(MonitoredSystem.ES, Settings.EMPTY);
final BytesReference jsonBytes = resolver.source(clusterStateDoc, XContentType.JSON);
// parse the JSON to figure out what we just did
final Map<String, Object> fields = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, jsonBytes).map();
// ensure we did what we wanted
for (final String filter : ClusterStateResolver.FILTERS) {
assertContains(filter, fields);
}
logger.debug("--> cluster state successfully collected");
}
/**
* This test should fail if the mapping for the 'nodes' attribute
* in the 'cluster_state' document is NOT set to 'enable: false'
*/
public void testNoNodesIndexing() throws Exception {
logger.debug("--> waiting for documents to be collected");
awaitMonitoringDocsCountOnPrimary(greaterThan(0L), ClusterStateResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateResolver.TYPE).setPreference("_primary").get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
DiscoveryNodes nodes = client().admin().cluster().prepareState().clear().setNodes(true).get().getState().nodes();
logger.debug("--> ensure that the 'nodes' attributes of the cluster state document is not indexed");
assertHitCount(client().prepareSearch().setSize(0).setTypes(ClusterStateResolver.TYPE).setPreference("_primary")
.setQuery(matchQuery("cluster_state.nodes." + nodes.getMasterNodeId() + ".name",
nodes.getMasterNode().getName())).get(), 0L);
}
public void testClusterStateNodes() throws Exception {
final long nbNodes = internalCluster().size();
MonitoringIndexNameResolver.Timestamped timestampedResolver =
new MockTimestampedIndexNameResolver(MonitoredSystem.ES, Settings.EMPTY, MonitoringTemplateUtils.TEMPLATE_VERSION);
assertNotNull(timestampedResolver);
String timestampedIndex = timestampedResolver.indexPattern();
awaitIndexExists(timestampedIndex);
logger.debug("--> waiting for documents to be collected");
awaitMonitoringDocsCountOnPrimary(greaterThanOrEqualTo(nbNodes), ClusterStateNodeMonitoringDoc.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateNodeMonitoringDoc.TYPE);
SearchResponse response = client().prepareSearch(timestampedIndex).setTypes(ClusterStateNodeMonitoringDoc.TYPE)
.setPreference("_primary").get();
assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
logger.debug("--> checking that every document contains the expected fields");
String[] filters = {
MonitoringIndexNameResolver.Fields.CLUSTER_UUID,
MonitoringIndexNameResolver.Fields.TIMESTAMP,
MonitoringIndexNameResolver.Fields.SOURCE_NODE,
ClusterStateNodeResolver.Fields.STATE_UUID,
ClusterStateNodeResolver.Fields.NODE,
ClusterStateNodeResolver.Fields.NODE + "."
+ ClusterStateNodeResolver.Fields.ID,
};
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.getSourceAsMap();
for (String filter : filters) {
assertContains(filter, fields);
}
}
logger.debug("--> check that node attributes are indexed");
assertThat(client().prepareSearch().setSize(0)
.setIndices(timestampedIndex)
.setTypes(ClusterStateNodeMonitoringDoc.TYPE)
.setPreference("_primary")
.setQuery(QueryBuilders.matchQuery(MonitoringIndexNameResolver.Fields.SOURCE_NODE + ".attributes.custom", randomInt))
.get().getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> cluster state nodes successfully collected");
}
}

163
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsResolverTests.java
View File

@ -6,146 +6,79 @@
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RecoverySource;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.http.HttpInfo;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.cache.query.QueryCacheStats;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.indices.NodeIndicesStats;
import org.elasticsearch.ingest.IngestInfo;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.os.DummyOsInfo;
import org.elasticsearch.monitor.process.ProcessInfo;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.threadpool.ThreadPoolInfo;
import org.elasticsearch.transport.TransportInfo;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.startsWith;
public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCase<ClusterStatsMonitoringDoc, ClusterStatsResolver> {
@Override
protected ClusterStatsMonitoringDoc newMonitoringDoc() {
ClusterStatsMonitoringDoc doc = new ClusterStatsMonitoringDoc(randomMonitoringId(),
randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
randomClusterStats());
return doc;
try {
License.Builder licenseBuilder = License.builder()
.uid(UUID.randomUUID().toString())
.type("trial")
.issuer("elasticsearch")
.issuedTo("customer")
.maxNodes(5)
.issueDate(1437580442979L)
.expiryDate(1437580442979L + TimeValue.timeValueHours(2).getMillis());
return new ClusterStatsMonitoringDoc(randomMonitoringId(),
randomAlphaOfLength(2), randomAlphaOfLength(5),
Math.abs(randomLong()),
new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
randomAlphaOfLength(5),
randomFrom(Version.V_5_0_0, Version.CURRENT).toString(),
licenseBuilder.build(),
Collections.singletonList(new MonitoringFeatureSet.Usage(randomBoolean(), randomBoolean(), emptyMap())),
new ClusterStatsResponse(
Math.abs(randomLong()),
ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY),
Collections.emptyList(),
Collections.emptyList())
);
} catch (Exception e) {
throw new IllegalStateException("Failed to generated random ClusterStatsMonitoringDoc", e);
}
}
public void testClusterStatsResolver() throws Exception {
@Override
protected boolean checkFilters() {
return false;
}
public void testClusterInfoResolver() throws Exception {
ClusterStatsMonitoringDoc doc = newMonitoringDoc();
ClusterStatsResolver resolver = newResolver();
assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
assertThat(resolver.index(doc), startsWith(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"cluster_stats"), XContentType.JSON);
}
/**
* @return a testing {@link ClusterStatsResponse} used to resolve a monitoring document.
*/
private ClusterStatsResponse randomClusterStats() {
List<ClusterStatsNodeResponse> responses = Collections.singletonList(
new ClusterStatsNodeResponse(new DiscoveryNode("node_0", buildNewFakeTransportAddress(),
emptyMap(), emptySet(), Version.CURRENT),
ClusterHealthStatus.GREEN, randomNodeInfo(), randomNodeStats(), randomShardStats())
);
return new ClusterStatsResponse(
Math.abs(randomLong()),
ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY),
responses,
Collections.emptyList());
}
/**
* @return a random {@link NodeInfo} used to resolve a monitoring document.
*/
private NodeInfo randomNodeInfo() {
BoundTransportAddress transportAddress = new BoundTransportAddress(new TransportAddress[]{buildNewFakeTransportAddress()},
buildNewFakeTransportAddress());
return new NodeInfo(Version.CURRENT, org.elasticsearch.Build.CURRENT,
new DiscoveryNode("node_0", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), Settings.EMPTY,
DummyOsInfo.INSTANCE, new ProcessInfo(randomInt(), randomBoolean(), randomNonNegativeLong()), JvmInfo.jvmInfo(),
new ThreadPoolInfo(Collections.singletonList(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5))),
new TransportInfo(transportAddress, Collections.emptyMap()), new HttpInfo(transportAddress, randomLong()),
new PluginsAndModules(Collections.emptyList(), Collections.emptyList()),
new IngestInfo(Collections.emptyList()), new ByteSizeValue(randomIntBetween(1, 1024)));
}
/**
* @return a random {@link NodeStats} used to resolve a monitoring document.
*/
private NodeStats randomNodeStats() {
Index index = new Index("test", UUID.randomUUID().toString());
FsInfo.Path[] pathInfo = new FsInfo.Path[]{
new FsInfo.Path("/test", "/dev/sda", 10, -8, 0),
};
Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
statsByShard.put(index, Collections.singletonList(new IndexShardStats(new ShardId(index, 0), randomShardStats())));
return new NodeStats(new DiscoveryNode("node_0", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0,
new NodeIndicesStats(new CommonStats(), statsByShard), null, null, null, null,
new FsInfo(0, null, pathInfo), null, null, null, null, null, null);
}
/**
* @return a random ShardStats[] used to resolve a monitoring document.
*/
private ShardStats[] randomShardStats() {
Index index = new Index("test", UUID.randomUUID().toString());
Path shardPath = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), false,
RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
CommonStats shardCommonStats = new CommonStats();
shardCommonStats.fieldData = new FieldDataStats();
shardCommonStats.queryCache = new QueryCacheStats();
return new ShardStats[]{
new ShardStats(
shardRouting,
new ShardPath(false, shardPath, shardPath, new ShardId(index, 0)),
shardCommonStats,
null,
null)};
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"cluster_name",
"version",
"license",
"cluster_stats",
"stack_stats.xpack"), XContentType.JSON);
}
}

131
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsTests.java
View File

@ -5,24 +5,35 @@
*/
package org.elasticsearch.xpack.monitoring.resolver.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
import org.elasticsearch.Version;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.license.License;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
import org.junit.After;
import org.junit.Before;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.isEmptyOrNullString;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
@ClusterScope(scope = TEST)
public class ClusterStatsTests extends MonitoringIntegTestCase {
@Override
@ -30,10 +41,14 @@ public class ClusterStatsTests extends MonitoringIntegTestCase {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MonitoringSettings.INTERVAL.getKey(), "-1")
.put("xpack.monitoring.exporters.default_local.type", "local")
.build();
}
@Before
public void init() throws Exception {
updateMonitoringInterval(3L, TimeUnit.SECONDS);
}
@After
public void cleanup() throws Exception {
disableMonitoringInterval();
@ -41,42 +56,84 @@ public class ClusterStatsTests extends MonitoringIntegTestCase {
}
public void testClusterStats() throws Exception {
logger.debug("--> creating some indices so that every data nodes will at least a shard");
ClusterStatsNodes.Counts counts = client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts();
assertThat(counts.getTotal(), greaterThan(0));
String indexNameBase = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
int indicesCount = randomIntBetween(1, 5);
String[] indices = new String[indicesCount];
for (int i = 0; i < indicesCount; i++) {
indices[i] = indexNameBase + "-" + i;
index(indices[i], "foo", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
}
flush();
refresh();
ensureGreen();
// ok.. we'll start collecting now...
updateMonitoringInterval(3L, TimeUnit.SECONDS);
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
awaitMonitoringDocsCountOnPrimary(greaterThan(0L), ClusterStatsResolver.TYPE);
// waiting for cluster stats collector to collect data
awaitMonitoringDocsCountOnPrimary(equalTo(1L), ClusterStatsMonitoringDoc.TYPE);
assertBusy(new Runnable() {
@Override
public void run() {
logger.debug("--> checking that every document contains the expected fields");
SearchResponse response = client().prepareSearch().setTypes(ClusterStatsResolver.TYPE).setPreference("_primary").get();
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.getSourceAsMap();
refresh();
for (String filter : ClusterStatsResolver.FILTERS) {
assertContains(filter, fields);
}
}
}
});
// retrieving cluster stats document
final SearchResponse response =
client().prepareSearch(".monitoring-es-*")
.setQuery(QueryBuilders.termQuery("type", ClusterStatsMonitoringDoc.TYPE))
.addSort("timestamp", SortOrder.DESC)
.setSize(1)
.setPreference("_primary").get();
assertTrue("cluster_stats document does not exist", response.getHits().getTotalHits() != 0);
logger.debug("--> cluster stats successfully collected");
final SearchHit hit = response.getHits().getAt(0);
assertTrue("_source is disabled", hit.hasSource());
Map<String, Object> source = hit.getSourceAsMap();
assertThat(source.get(MonitoringIndexNameResolver.Fields.CLUSTER_UUID), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.TIMESTAMP), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.SOURCE_NODE), notNullValue());
assertThat(source.get("cluster_name"), equalTo(cluster().getClusterName()));
assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
Object licenseObj = source.get("license");
assertThat(licenseObj, instanceOf(Map.class));
Map license = (Map) licenseObj;
assertThat(license, instanceOf(Map.class));
String uid = (String) license.get("uid");
assertThat(uid, not(isEmptyOrNullString()));
String type = (String) license.get("type");
assertThat(type, not(isEmptyOrNullString()));
String status = (String) license.get(License.Fields.STATUS);
assertThat(status, not(isEmptyOrNullString()));
Long expiryDate = (Long) license.get(License.Fields.EXPIRY_DATE_IN_MILLIS);
assertThat(expiryDate, greaterThan(0L));
// We basically recompute the hash here
String hkey = (String) license.get("hkey");
String recalculated = ClusterStatsResolver.hash(status, uid, type, String.valueOf(expiryDate), clusterUUID);
assertThat(hkey, equalTo(recalculated));
assertThat((String) license.get(License.Fields.ISSUER), not(isEmptyOrNullString()));
assertThat((String) license.get(License.Fields.ISSUED_TO), not(isEmptyOrNullString()));
assertThat((Long) license.get(License.Fields.ISSUE_DATE_IN_MILLIS), greaterThan(0L));
assertThat((Integer) license.get(License.Fields.MAX_NODES), greaterThan(0));
Object clusterStats = source.get("cluster_stats");
assertNotNull(clusterStats);
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
Object stackStats = source.get("stack_stats");
assertNotNull(stackStats);
assertThat(stackStats, instanceOf(Map.class));
assertThat(((Map) stackStats).size(), equalTo(1));
Object xpack = ((Map)stackStats).get("xpack");
assertNotNull(xpack);
assertThat(xpack, instanceOf(Map.class));
// it must have at least monitoring, but others may be hidden
assertThat(((Map) xpack).size(), greaterThanOrEqualTo(1));
Object monitoring = ((Map)xpack).get("monitoring");
assertNotNull(monitoring);
// we don't make any assumptions about what's in it, only that it's there
assertThat(monitoring, instanceOf(Map.class));
}
}

1
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexRecoveryResolverTests.java
View File

@ -61,6 +61,7 @@ public class IndexRecoveryResolverTests extends MonitoringIndexNameResolverTestC
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"index_recovery"), XContentType.JSON);
}

49
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexRecoveryTests.java
View File

@ -9,6 +9,9 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
@ -19,7 +22,6 @@ import org.junit.After;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.greaterThan;
@ -29,6 +31,7 @@ import static org.hamcrest.Matchers.is;
public class IndexRecoveryTests extends MonitoringIntegTestCase {
private static final String INDEX_PREFIX = "test-index-recovery-";
private final TermQueryBuilder indexRecoveryType = QueryBuilders.termQuery("type", IndexRecoveryResolver.TYPE);
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -47,12 +50,10 @@ public class IndexRecoveryTests extends MonitoringIntegTestCase {
}
public void testIndexRecovery() throws Exception {
logger.debug("--> creating some indices so that index recovery collector reports data");
for (int i = 0; i < randomIntBetween(1, 10); i++) {
client().prepareIndex(INDEX_PREFIX + i, "foo").setSource("field1", "value1").get();
}
logger.debug("--> wait for index recovery collector to collect data");
assertBusy(new Runnable() {
@Override
public void run() {
@ -72,18 +73,20 @@ public class IndexRecoveryTests extends MonitoringIntegTestCase {
String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
logger.debug("--> searching for monitoring documents of type [{}]", IndexRecoveryResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(IndexRecoveryResolver.TYPE).setPreference("_primary").get();
SearchResponse response =
client().prepareSearch()
.setQuery(indexRecoveryType)
.setPreference("_primary")
.get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");
String[] filters = {
MonitoringIndexNameResolver.Fields.CLUSTER_UUID,
MonitoringIndexNameResolver.Fields.TIMESTAMP,
MonitoringIndexNameResolver.Fields.TYPE,
MonitoringIndexNameResolver.Fields.SOURCE_NODE,
IndexRecoveryResolver.Fields.INDEX_RECOVERY,
IndexRecoveryResolver.Fields.INDEX_RECOVERY + "."
+ IndexRecoveryResolver.Fields.SHARDS,
IndexRecoveryResolver.Fields.INDEX_RECOVERY + "." + IndexRecoveryResolver.Fields.SHARDS,
};
for (SearchHit searchHit : response.getHits().getHits()) {
@ -93,20 +96,9 @@ public class IndexRecoveryTests extends MonitoringIntegTestCase {
}
}
flush();
refresh();
logger.debug("--> checking that cluster_uuid field is correctly indexed");
response = client().prepareSearch().setTypes(IndexRecoveryResolver.TYPE).setPreference("_primary").setSize(0)
.setQuery(existsQuery("cluster_uuid")).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that timestamp field is correctly indexed");
response = client().prepareSearch().setTypes(IndexRecoveryResolver.TYPE).setPreference("_primary").setSize(0)
.setQuery(existsQuery("timestamp")).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that other fields are not indexed");
// ensure these fields are not indexed (searchable)
String[] fields = {
"index_recovery.shards.primary",
"index_recovery.shards.id",
@ -117,11 +109,22 @@ public class IndexRecoveryTests extends MonitoringIntegTestCase {
};
for (String field : fields) {
response = client().prepareSearch().setTypes(IndexRecoveryResolver.TYPE).setPreference("_primary").setSize(0)
.setQuery(existsQuery(field)).get();
response =
client().prepareSearch()
.setQuery(checkForFieldQuery(field))
.setPreference("_primary")
.setSize(0)
.get();
assertHitCount(response, 0L);
}
}
logger.debug("--> index recovery successfully collected");
private BoolQueryBuilder checkForFieldQuery(final String field) {
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must().add(indexRecoveryType);
boolQuery.must().add(QueryBuilders.existsQuery(field));
return boolQuery;
}
}

1
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexStatsResolverTests.java
View File

@ -60,6 +60,7 @@ public class IndexStatsResolverTests extends MonitoringIndexNameResolverTestCase
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"index_stats"), XContentType.JSON);
}

2
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsResolverTests.java
View File

@ -61,7 +61,7 @@ public class IndicesStatsResolverTests extends MonitoringIndexNameResolverTestCa
assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet("cluster_uuid", "timestamp", "source_node", "indices_stats"), XContentType.JSON);
Sets.newHashSet("cluster_uuid", "timestamp", "type", "source_node", "indices_stats"), XContentType.JSON);
}
/**

7
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsTests.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.monitoring.resolver.indices;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -74,7 +75,11 @@ public class IndicesStatsTests extends MonitoringIntegTestCase {
awaitMonitoringDocsCountOnPrimary(greaterThan(0L), IndicesStatsResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", IndicesStatsResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(IndicesStatsResolver.TYPE).setPreference("_primary").get();
SearchResponse response =
client().prepareSearch()
.setQuery(QueryBuilders.termQuery("type", IndicesStatsResolver.TYPE))
.setPreference("_primary")
.get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");

56
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/ml/JobStatsResolverTests.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.resolver.ml;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
import static org.hamcrest.Matchers.startsWith;
public class JobStatsResolverTests extends MonitoringIndexNameResolverTestCase<JobStatsMonitoringDoc, JobStatsResolver> {
@Override
protected JobStatsMonitoringDoc newMonitoringDoc() {
final JobStats jobStats =
new JobStats("fake-job1", new DataCounts("fake-job1"),
null, JobState.OPENED, null, null, null);
return new JobStatsMonitoringDoc(randomAlphaOfLength(5),
Math.abs(randomLong()),
new DiscoveryNode("id", buildNewFakeTransportAddress(), Version.CURRENT),
jobStats);
}
@Override
protected boolean checkFilters() {
return false;
}
public void testClusterInfoResolver() throws Exception {
JobStatsMonitoringDoc doc = newMonitoringDoc();
JobStatsResolver resolver = newResolver();
assertThat(resolver.index(doc), startsWith(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"job_stats"),
XContentType.JSON);
}
}

10
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/MultiNodesStatsTests.java
View File

@ -7,6 +7,8 @@ package org.elasticsearch.xpack.monitoring.resolver.node;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
@ -46,24 +48,20 @@ public class MultiNodesStatsTests extends MonitoringIntegTestCase {
int nodes = 0;
int n = randomIntBetween(1, 2);
logger.debug("--> starting {} master only nodes", n);
internalCluster().startMasterOnlyNodes(n);
nodes += n;
n = randomIntBetween(2, 3);
logger.debug("--> starting {} data only nodes", n);
internalCluster().startDataOnlyNodes(n);
nodes += n;
n = randomIntBetween(1, 2);
logger.debug("--> starting {} client only nodes", n);
internalCluster().startNodes(n,
Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), false)
.put(Node.NODE_INGEST_SETTING.getKey(), false).build());
nodes += n;
n = randomIntBetween(1, 2);
logger.debug("--> starting {} extra nodes", n);
// starting one by one to allow moving , for example, from a 2 node cluster to a 4 one while updating min_master_nodes
for (int i=0;i<n;i++) {
internalCluster().startNode();
@ -71,7 +69,6 @@ public class MultiNodesStatsTests extends MonitoringIntegTestCase {
nodes += n;
final int nbNodes = nodes;
logger.debug("--> waiting for {} nodes to be available", nbNodes);
assertBusy(() -> {
assertThat(cluster().size(), equalTo(nbNodes));
assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nbNodes)).get());
@ -80,14 +77,13 @@ public class MultiNodesStatsTests extends MonitoringIntegTestCase {
updateMonitoringInterval(3L, TimeUnit.SECONDS);
waitForMonitoringIndices();
logger.debug("--> checking that every node correctly reported its own node stats");
assertBusy(() -> {
String indices = MONITORING_INDICES_PREFIX + "*";
flush(indices);
refresh();
SearchResponse response = client().prepareSearch(indices)
.setTypes(NodeStatsResolver.TYPE)
.setQuery(QueryBuilders.termQuery("type", NodeStatsResolver.TYPE))
.setSize(0)
.addAggregation(AggregationBuilders.terms("nodes_ids").field("node_stats.node_id"))
.get();

2
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsResolverTests.java
View File

@ -53,7 +53,6 @@ import java.util.UUID;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<NodeStatsMonitoringDoc, NodeStatsResolver> {
@ -110,6 +109,7 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"node_stats"), XContentType.JSON);
}

7
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsTests.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.monitoring.resolver.node;
import org.apache.lucene.util.Constants;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -56,7 +57,11 @@ public class NodeStatsTests extends MonitoringIntegTestCase {
awaitMonitoringDocsCountOnPrimary(greaterThan(0L), NodeStatsResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(NodeStatsResolver.TYPE).setPreference("_primary").get();
SearchResponse response =
client().prepareSearch()
.setQuery(QueryBuilders.termQuery("type", NodeStatsResolver.TYPE))
.setPreference("_primary")
.get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
for (SearchHit searchHit : response.getHits().getHits()) {

1
plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/shards/ShardsResolverTests.java
View File

@ -53,6 +53,7 @@ public class ShardsResolverTests extends MonitoringIndexNameResolverTestCase<Sha
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"type",
"source_node",
"state_uuid",
"shard.state",

93
plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring.test;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.plugins.IngestPlugin;
import org.elasticsearch.plugins.Plugin;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Mock representation of the Ingest Common plugin for the subset of processors that are needed for the pipelines in Monitoring's exporters.
*/
public class MockIngestPlugin extends Plugin implements IngestPlugin {
@Override
public Map<String, Processor.Factory> getProcessors(final Processor.Parameters parameters) {
final Map<String, String[]> processorFields = MapBuilder.<String, String[]>newMapBuilder()
.put("gsub", new String[] { "field", "pattern", "replacement" })
.put("rename", new String[] { "field", "target_field" })
.put("set", new String[] { "field", "value" })
.put("script", new String[] { "inline" })
.map();
return processorFields.entrySet()
.stream()
.map(MockProcessorFactory::new)
.collect(Collectors.toMap(factory -> factory.type, factory -> factory));
}
static class MockProcessorFactory implements Processor.Factory {
private final String type;
private final String[] fields;
MockProcessorFactory(final Map.Entry<String, String[]> factory) {
this(factory.getKey(), factory.getValue());
}
MockProcessorFactory(final String type, final String[] fields) {
this.type = type;
this.fields = fields;
}
@Override
public Processor create(Map<String, Processor.Factory> processorFactories,
String tag,
Map<String, Object> config) throws Exception {
// read fields so the processor succeeds
for (final String field : fields) {
ConfigurationUtils.readObject(type, tag, config, field);
}
return new MockProcessor(type, tag);
}
}
static class MockProcessor implements Processor {
private final String type;
private final String tag;
MockProcessor(final String type, final String tag) {
this.type = type;
this.tag = tag;
}
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
// mock processor does nothing
}
@Override
public String getType() {
return type;
}
@Override
public String getTag() {
return tag;
}
}
}

102
plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
View File

@ -22,8 +22,10 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.TestCluster;
@ -81,12 +83,8 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
public static final String MONITORING_INDICES_PREFIX = MonitoringIndexNameResolver.PREFIX + MonitoringIndexNameResolver.DELIMITER;
/**
* Enables individual tests to control the behavior.
* <p>
* Control this by overriding {@link #enableSecurity()}, which defaults to enabling it randomly.
* Per test run this is enabled or disabled.
*/
// TODO: what is going on here?
// SCARY: This needs to be static or lots of tests randomly fail, but it's not used statically!
protected static Boolean securityEnabled;
/**
* Enables individual tests to control the behavior.
@ -104,7 +102,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
@Override
protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
if (securityEnabled == null) {
securityEnabled = enableSecurity();
securityEnabled = randomBoolean();
}
return super.buildTestCluster(scope, seed);
@ -148,6 +146,11 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
.build();
}
@Override
protected boolean addMockTransportService() {
return securityEnabled == false;
}
@Override
protected Collection<Class<? extends Plugin>> getMockPlugins() {
Set<Class<? extends Plugin>> plugins = new HashSet<>(super.getMockPlugins());
@ -158,7 +161,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(XPackPlugin.class, MockPainlessScriptEngine.TestPlugin.class);
return Arrays.asList(XPackPlugin.class, MockPainlessScriptEngine.TestPlugin.class, MockIngestPlugin.class);
}
@Override
@ -183,7 +186,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
@Override
protected Set<String> excludeTemplates() {
return monitoringTemplateNames();
return new HashSet<>(monitoringTemplateNames());
}
@Before
@ -201,13 +204,6 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
super.tearDown();
}
/**
* Override and return {@code false} to force running without Security.
*/
protected boolean enableSecurity() {
return randomBoolean();
}
/**
* Override and return {@code false} to force running without Watcher.
*/
@ -257,7 +253,8 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
protected void assertMonitoringDocsCountOnPrimary(Matcher<Long> matcher, String... types) {
flushAndRefresh(MONITORING_INDICES_PREFIX + "*");
long count = client().prepareSearch(MONITORING_INDICES_PREFIX + "*").setSize(0).setTypes(types)
long count = client().prepareSearch(MONITORING_INDICES_PREFIX + "*").setSize(0)
.setQuery(QueryBuilders.termsQuery("type", types))
.setPreference("_primary").get().getHits().getTotalHits();
logger.trace("--> searched for [{}] documents on primary, found [{}]", Strings.arrayToCommaDelimitedString(types), count);
assertThat(count, matcher);
@ -277,17 +274,47 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
return templates;
}
protected Set<String> monitoringTemplateNames() {
final Set<String> templates = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
protected List<String> monitoringTemplateNames() {
final List<String> templateNames = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
.map(MonitoringTemplateUtils::templateName)
.collect(Collectors.toSet());
.collect(Collectors.toList());
// TODO: remove this when we remove resolvers
templates.addAll(StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
.map(MonitoringIndexNameResolver::templateName)
.collect(Collectors.toSet()));
final ResolversRegistry registry = new ResolversRegistry(Settings.EMPTY);
return templates;
for (final MonitoringIndexNameResolver resolver : registry) {
final String templateName = resolver.templateName();
if (templateNames.contains(templateName) == false) {
templateNames.add(templateName);
}
}
return templateNames;
}
private Tuple<String, String> monitoringPipeline(final String pipelineId) {
try {
final XContentType json = XContentType.JSON;
return new Tuple<>(MonitoringTemplateUtils.pipelineName(pipelineId),
MonitoringTemplateUtils.loadPipeline(pipelineId, json).string());
} catch (final IOException e) {
// destroy whatever test is running
throw new AssertionError("Unable to use pipeline as JSON string [" + pipelineId + "]", e);
}
}
protected List<Tuple<String, String>> monitoringPipelines() {
return Arrays.stream(MonitoringTemplateUtils.PIPELINE_IDS)
.map(this::monitoringPipeline)
.collect(Collectors.toList());
}
protected List<String> monitoringPipelineNames() {
return Arrays.stream(MonitoringTemplateUtils.PIPELINE_IDS)
.map(MonitoringTemplateUtils::pipelineName)
.collect(Collectors.toList());
}
protected List<Tuple<String, String>> monitoringWatches() {
@ -315,19 +342,6 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
assertTrue("failed to find a template matching [" + name + "]", found);
}
protected void waitForMonitoringTemplate(String name) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
assertTemplateInstalled(name);
}
}, 30, TimeUnit.SECONDS);
}
protected void waitForMonitoringTemplates() throws Exception {
assertBusy(() -> monitoringTemplateNames().forEach(this::assertTemplateInstalled), 30, TimeUnit.SECONDS);
}
protected void waitForMonitoringIndices() throws Exception {
awaitIndexExists(MONITORING_INDICES_PREFIX + "*");
assertBusy(this::ensureMonitoringIndicesYellow);
@ -411,18 +425,6 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), value, timeUnit)));
}
public class MockDataIndexNameResolver extends MonitoringIndexNameResolver.Data<MonitoringDoc> {
public MockDataIndexNameResolver(String version) {
super(version);
}
@Override
protected void buildXContent(MonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
throw new UnsupportedOperationException("MockDataIndexNameResolver does not support resolving building XContent");
}
}
protected class MockTimestampedIndexNameResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringDoc> {
public MockTimestampedIndexNameResolver(MonitoredSystem system, Settings settings, String version) {
@ -466,8 +468,8 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
" 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," +
" 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete'," +
" 'cluster:admin/ingest/pipeline/get', 'cluster:admin/ingest/pipeline/put', 'cluster:admin/ingest/pipeline/delete'," +
" 'cluster:monitor/xpack/watcher/watch/get', 'cluster:monitor/xpack/watcher/watch/put', " +
" 'cluster:monitor/xpack/watcher/watch/delete'," +
" 'cluster:monitor/xpack/watcher/watch/get', 'cluster:admin/xpack/watcher/watch/put', " +
" 'cluster:admin/xpack/watcher/watch/delete'," +
" 'cluster:monitor/task', 'cluster:admin/xpack/monitoring/bulk' ]\n" +
" indices:\n" +
" - names: '*'\n" +

2
plugin/src/test/java/org/elasticsearch/xpack/template/TemplateUtilsTests.java
View File

@ -36,7 +36,7 @@ public class TemplateUtilsTests extends ESTestCase {
assertTemplate(source, equalTo("{\n" +
" \"index_patterns\": \".monitoring-data-" + version + "\",\n" +
" \"mappings\": {\n" +
" \"type_1\": {\n" +
" \"doc\": {\n" +
" \"_meta\": {\n" +
" \"template.version\": \"" + version + "\"\n" +
" }\n" +

2
plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
View File

@ -199,6 +199,6 @@ public class XPackRestIT extends XPackRestTestCase {
private boolean isMonitoringTest() {
String testName = getTestName();
return testName != null && testName.contains("=monitoring/");
return testName != null && (testName.contains("=monitoring/") || testName.contains("=monitoring\\"));
}
}

2
plugin/src/test/resources/monitoring-test.json
View File

@ -1,7 +1,7 @@
{
"index_patterns": ".monitoring-data-${monitoring.template.version}",
"mappings": {
"type_1": {
"doc": {
"_meta": {
"template.version": "${monitoring.template.version}"
}

72
plugin/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
View File

@ -4,7 +4,7 @@
- do:
xpack.monitoring.bulk:
system_id: "kibana"
system_api_version: "2"
system_api_version: "6"
interval: "10s"
body:
- index:
@ -32,16 +32,16 @@
- do:
search:
index: .monitoring-kibana-*
type: test_type
body: { "query": { "term" : { "type": "test_type" } } }
- match: { hits.total: 2 }
- do:
xpack.monitoring.bulk:
system_id: "kibana"
system_api_version: "2"
system_api_version: "6"
interval: "10000ms"
type: "default_type"
type: "default_type"
body:
- '{"index": {}}'
- '{"field_1": "value_1"}'
@ -60,18 +60,62 @@
- do:
search:
index: .monitoring-kibana-*
type: default_type
body: { "query": { "term" : { "type": "default_type" } } }
- match: { hits.total: 2 }
- do:
search:
index: .monitoring-kibana-*
type: custom_type
body: { "query": { "term" : { "type": "custom_type" } } }
- match: { hits.total: 1 }
# We actively ignore indexing requests made to .monitoring-data-N starting with 5.5
# We actively ignore indexing requests made to the _data index starting with 5.5
- do:
search:
index: .monitoring-data-*
type: kibana
- match: { hits.total: 0 }
# Old system_api_version should still be accepted
- do:
xpack.monitoring.bulk:
system_id: "kibana"
system_api_version: "2"
interval: "10000ms"
type: "default_type"
body:
- '{"index": {}}'
- '{"field_1": "value_1"}'
- '{"index": {"_type": "custom_type"}}'
- '{"field_1": "value_2"}'
- '{"index": {}}'
- '{"field_1": "value_3"}'
- '{"index": {"_index": "_data", "_type": "kibana"}}'
- '{"field_1": "value_4"}'
- is_false: errors
- do:
indices.refresh: {}
- do:
search:
index: .monitoring-kibana-*
body: { "query": { "term" : { "type": "default_type" } } }
- match: { hits.total: 4 }
- do:
search:
index: .monitoring-kibana-*
body: { "query": { "term" : { "type": "custom_type" } } }
- match: { hits.total: 2 }
# We actively ignore indexing requests made to the _data index starting with 5.5, even for the old versions
- do:
search:
index: .monitoring-data-*
@ -83,9 +127,9 @@
- do:
catch: request
xpack.monitoring.bulk:
system_api_version: "2"
system_api_version: "6"
interval: "10s"
type: "default_type"
type: "default_type"
body:
- '{"index": {}}'
- '{"field_1": "value_1"}'
@ -96,7 +140,7 @@
xpack.monitoring.bulk:
system_id: "kibana"
interval: "10s"
type: "default_type"
type: "default_type"
body:
- '{"index": {}}'
- '{"field_1": "value_1"}'
@ -106,8 +150,8 @@
catch: request
xpack.monitoring.bulk:
system_id: "kibana"
system_api_version: "2"
type: "default_type"
system_api_version: "6"
type: "default_type"
body:
- '{"index": {}}'
- '{"field_1": "value_1"}'
@ -118,7 +162,7 @@
- do:
xpack.monitoring.bulk:
system_id: "beats"
system_api_version: "2"
system_api_version: "6"
interval: "5s"
body:
- index:
@ -150,7 +194,7 @@
catch: /export_exception/
xpack.monitoring.bulk:
system_id: "beats"
system_api_version: "2"
system_api_version: "6"
interval: "5s"
body:
- index:

9
plugin/src/test/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml
View File

@ -84,7 +84,7 @@ teardown:
Authorization: "Basic bG9nc3Rhc2hfYWdlbnQ6czNrcml0"
xpack.monitoring.bulk:
system_id: "logstash"
system_api_version: "2"
system_api_version: "6"
interval: "10s"
body:
- index:
@ -104,14 +104,13 @@ teardown:
- do:
search:
index: .monitoring-logstash-*
type: logstash_metric
body: { "query": { "term" : { "type": "logstash_metric" } } }
- match: { hits.total: 1 }
# We actively ignore indexing requests made to .monitoring-data-N starting with 5.5
- do:
search:
index: .monitoring-data-*
type: logstash_info
- match: { hits.total: 0 }
- do:
@ -121,7 +120,7 @@ teardown:
Authorization: "Basic dW5rbm93bl9hZ2VudDpzM2tyaXQ="
xpack.monitoring.bulk:
system_id: "logstash"
system_api_version: "2"
system_api_version: "6"
interval: "10s"
body:
- index:
@ -137,5 +136,5 @@ teardown:
- do:
search:
index: .monitoring-logstash-*
type: logstash_metric
body: { "query": { "term" : { "type": "logstash_metric" } } }
- match: { hits.total: 1 }