Monitoring/Watcher: Load version of templates in a static way (elastic/elasticsearch#2040)
The old implementation was to use properties at build-time. This however did not work, as the tests could not be run in the IDE. This has been removed of monitoring for some time already, but needs to be removed from watcher as well. This commit uses static variables and refactors the code a bit. First, there is a generic TemplateUtils class, to be used in monitoring and watcher. Also the watcher code has been changed to copy the needed variables into the template registry class instead of keeping it in the WatcherModule. This commit also includes some refactoring to remove the version parameter in marvel, was static anyway Closes elastic/elasticsearch#1372 Original commit: elastic/x-pack-elasticsearch@fbfc22ea09
This commit is contained in:
parent
df3bbd42b9
commit
12ff8853f0
|
@ -5,16 +5,8 @@
|
||||||
*/
|
*/
|
||||||
package org.elasticsearch.marvel.agent.exporter;
|
package org.elasticsearch.marvel.agent.exporter;
|
||||||
|
|
||||||
import org.elasticsearch.ElasticsearchParseException;
|
import org.elasticsearch.xpack.template.TemplateUtils;
|
||||||
import org.elasticsearch.common.bytes.BytesArray;
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
|
||||||
import org.elasticsearch.common.compress.NotXContentException;
|
|
||||||
import org.elasticsearch.common.io.Streams;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
@ -29,56 +21,8 @@ public final class MarvelTemplateUtils {
|
||||||
private MarvelTemplateUtils() {
|
private MarvelTemplateUtils() {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public static String loadTemplate(String id) {
|
||||||
* Loads a built-in template and returns its source.
|
|
||||||
*/
|
|
||||||
public static String loadTemplate(String id, Integer version) {
|
|
||||||
String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id);
|
String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id);
|
||||||
try {
|
return TemplateUtils.loadTemplate(resource, String.valueOf(TEMPLATE_VERSION), TEMPLATE_VERSION_PROPERTY);
|
||||||
BytesReference source = load(resource);
|
|
||||||
validate(source);
|
|
||||||
|
|
||||||
return filter(source, version);
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new IllegalArgumentException("Unable to load monitoring template [" + resource + "]", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Loads a resource from the classpath and returns it as a {@link BytesReference}
|
|
||||||
*/
|
|
||||||
static BytesReference load(String name) throws IOException {
|
|
||||||
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) {
|
|
||||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
|
||||||
Streams.copy(is, out);
|
|
||||||
return new BytesArray(out.toByteArray());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parses and validates that the source is not empty.
|
|
||||||
*/
|
|
||||||
static void validate(BytesReference source) {
|
|
||||||
if (source == null) {
|
|
||||||
throw new ElasticsearchParseException("Monitoring template must not be null");
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
XContentHelper.convertToMap(source, false).v2();
|
|
||||||
} catch (NotXContentException e) {
|
|
||||||
throw new ElasticsearchParseException("Monitoring template must not be empty");
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new ElasticsearchParseException("Invalid monitoring template", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filters the source: replaces any template version property with the version number
|
|
||||||
*/
|
|
||||||
static String filter(BytesReference source, Integer version) {
|
|
||||||
return Pattern.compile(TEMPLATE_VERSION_PROPERTY)
|
|
||||||
.matcher(source.toUtf8())
|
|
||||||
.replaceAll(String.valueOf(version));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,12 +35,6 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
public static final String PREFIX = ".monitoring";
|
public static final String PREFIX = ".monitoring";
|
||||||
public static final String DELIMITER = "-";
|
public static final String DELIMITER = "-";
|
||||||
|
|
||||||
private final int version;
|
|
||||||
|
|
||||||
protected MonitoringIndexNameResolver(int version) {
|
|
||||||
this.version = version;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the name of the index in which the monitoring document must be indexed.
|
* Returns the name of the index in which the monitoring document must be indexed.
|
||||||
*
|
*
|
||||||
|
@ -111,10 +105,6 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
*/
|
*/
|
||||||
public abstract String template();
|
public abstract String template();
|
||||||
|
|
||||||
int getVersion() {
|
|
||||||
return version;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return the filters used when rendering the document.
|
* @return the filters used when rendering the document.
|
||||||
* If null or empty, no filtering is applied.
|
* If null or empty, no filtering is applied.
|
||||||
|
@ -142,9 +132,8 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
|
|
||||||
private final String index;
|
private final String index;
|
||||||
|
|
||||||
public Data(int version) {
|
public Data() {
|
||||||
super(version);
|
this.index = String.join(DELIMITER, PREFIX, DATA, String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||||
this.index = String.join(DELIMITER, PREFIX, DATA, String.valueOf(version));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -159,12 +148,12 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String templateName() {
|
public String templateName() {
|
||||||
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, getVersion());
|
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, MarvelTemplateUtils.TEMPLATE_VERSION);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String template() {
|
public String template() {
|
||||||
return MarvelTemplateUtils.loadTemplate(DATA, getVersion());
|
return MarvelTemplateUtils.loadTemplate(DATA);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -181,10 +170,9 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
private final DateTimeFormatter formatter;
|
private final DateTimeFormatter formatter;
|
||||||
private final String index;
|
private final String index;
|
||||||
|
|
||||||
public Timestamped(MonitoredSystem system, int version, Settings settings) {
|
public Timestamped(MonitoredSystem system, Settings settings) {
|
||||||
super(version);
|
|
||||||
this.system = system;
|
this.system = system;
|
||||||
this.index = String.join(DELIMITER, PREFIX, system.getSystem(), String.valueOf(getVersion()));
|
this.index = String.join(DELIMITER, PREFIX, system.getSystem(), String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||||
String format = INDEX_NAME_TIME_FORMAT_SETTING.get(settings);
|
String format = INDEX_NAME_TIME_FORMAT_SETTING.get(settings);
|
||||||
try {
|
try {
|
||||||
this.formatter = DateTimeFormat.forPattern(format).withZoneUTC();
|
this.formatter = DateTimeFormat.forPattern(format).withZoneUTC();
|
||||||
|
@ -211,12 +199,12 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String templateName() {
|
public String templateName() {
|
||||||
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), getVersion());
|
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), MarvelTemplateUtils.TEMPLATE_VERSION);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String template() {
|
public String template() {
|
||||||
return MarvelTemplateUtils.loadTemplate(getId(), getVersion());
|
return MarvelTemplateUtils.loadTemplate(getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
String getId() {
|
String getId() {
|
||||||
|
|
|
@ -19,7 +19,6 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMonitoringDoc;
|
||||||
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMonitoringDoc;
|
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMonitoringDoc;
|
||||||
import org.elasticsearch.marvel.agent.collector.node.NodeStatsMonitoringDoc;
|
import org.elasticsearch.marvel.agent.collector.node.NodeStatsMonitoringDoc;
|
||||||
import org.elasticsearch.marvel.agent.collector.shards.ShardMonitoringDoc;
|
import org.elasticsearch.marvel.agent.collector.shards.ShardMonitoringDoc;
|
||||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
|
||||||
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
||||||
import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver;
|
import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver;
|
||||||
import org.elasticsearch.marvel.agent.resolver.cluster.ClusterInfoResolver;
|
import org.elasticsearch.marvel.agent.resolver.cluster.ClusterInfoResolver;
|
||||||
|
@ -43,40 +42,37 @@ import static org.elasticsearch.marvel.MonitoredSystem.ES;
|
||||||
|
|
||||||
public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver> {
|
public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver> {
|
||||||
|
|
||||||
private static final int ES_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
|
|
||||||
private static final int KIBANA_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
|
|
||||||
|
|
||||||
private final List<Registration> registrations = new ArrayList<>();
|
private final List<Registration> registrations = new ArrayList<>();
|
||||||
|
|
||||||
public ResolversRegistry(Settings settings) {
|
public ResolversRegistry(Settings settings) {
|
||||||
// register built-in defaults resolvers
|
// register built-in defaults resolvers
|
||||||
registerBuiltIn(ES, ES_TEMPLATE_VERSION, settings);
|
registerBuiltIn(ES, settings);
|
||||||
|
|
||||||
// register resolvers for monitored systems
|
// register resolvers for monitored systems
|
||||||
registerMonitoredSystem(MonitoredSystem.KIBANA, KIBANA_TEMPLATE_VERSION, settings);
|
registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Registers resolvers for elasticsearch documents collected by the monitoring plugin
|
* Registers resolvers for elasticsearch documents collected by the monitoring plugin
|
||||||
*/
|
*/
|
||||||
private void registerBuiltIn(MonitoredSystem id, int version, Settings settings) {
|
private void registerBuiltIn(MonitoredSystem id, Settings settings) {
|
||||||
registrations.add(resolveByClass(ClusterInfoMonitoringDoc.class, new ClusterInfoResolver(version)));
|
registrations.add(resolveByClass(ClusterInfoMonitoringDoc.class, new ClusterInfoResolver()));
|
||||||
registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, version, settings)));
|
registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, version, settings)));
|
registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, version, settings)));
|
registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver(version)));
|
registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver()));
|
||||||
registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, version, settings)));
|
registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, version, settings)));
|
registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, version, settings)));
|
registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, version, settings)));
|
registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, settings)));
|
||||||
registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, version, settings)));
|
registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, settings)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Registers resolvers for monitored systems
|
* Registers resolvers for monitored systems
|
||||||
*/
|
*/
|
||||||
private void registerMonitoredSystem(MonitoredSystem id, int version, Settings settings) {
|
private void registerMonitoredSystem(MonitoredSystem id, Settings settings) {
|
||||||
final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, version, settings);
|
final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, settings);
|
||||||
registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver));
|
registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,8 +17,8 @@ import java.io.IOException;
|
||||||
|
|
||||||
public class MonitoringBulkResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringBulkDoc> {
|
public class MonitoringBulkResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringBulkDoc> {
|
||||||
|
|
||||||
public MonitoringBulkResolver(MonitoredSystem id, int version, Settings settings) {
|
public MonitoringBulkResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -22,10 +22,6 @@ public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data<Cluste
|
||||||
|
|
||||||
public static final String TYPE = "cluster_info";
|
public static final String TYPE = "cluster_info";
|
||||||
|
|
||||||
public ClusterInfoResolver(int version) {
|
|
||||||
super(version);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String type(ClusterInfoMonitoringDoc document) {
|
public String type(ClusterInfoMonitoringDoc document) {
|
||||||
return TYPE;
|
return TYPE;
|
||||||
|
|
|
@ -18,8 +18,8 @@ public class ClusterStateNodeResolver extends MonitoringIndexNameResolver.Timest
|
||||||
|
|
||||||
public static final String TYPE = "node";
|
public static final String TYPE = "node";
|
||||||
|
|
||||||
public ClusterStateNodeResolver(MonitoredSystem id, int version, Settings settings) {
|
public ClusterStateNodeResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -31,8 +31,8 @@ public class ClusterStateResolver extends MonitoringIndexNameResolver.Timestampe
|
||||||
"cluster_state.nodes",
|
"cluster_state.nodes",
|
||||||
};
|
};
|
||||||
|
|
||||||
public ClusterStateResolver(MonitoredSystem id, int version, Settings settings) {
|
public ClusterStateResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -38,8 +38,8 @@ public class ClusterStatsResolver extends MonitoringIndexNameResolver.Timestampe
|
||||||
"cluster_stats.nodes.versions",
|
"cluster_stats.nodes.versions",
|
||||||
};
|
};
|
||||||
|
|
||||||
public ClusterStatsResolver(MonitoredSystem id, int version, Settings settings) {
|
public ClusterStatsResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -18,10 +18,6 @@ public class DiscoveryNodeResolver extends MonitoringIndexNameResolver.Data<Disc
|
||||||
|
|
||||||
public static final String TYPE = "node";
|
public static final String TYPE = "node";
|
||||||
|
|
||||||
public DiscoveryNodeResolver(int version) {
|
|
||||||
super(version);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String type(DiscoveryNodeMonitoringDoc document) {
|
public String type(DiscoveryNodeMonitoringDoc document) {
|
||||||
return TYPE;
|
return TYPE;
|
||||||
|
|
|
@ -22,8 +22,8 @@ public class IndexRecoveryResolver extends MonitoringIndexNameResolver.Timestamp
|
||||||
|
|
||||||
public static final String TYPE = "index_recovery";
|
public static final String TYPE = "index_recovery";
|
||||||
|
|
||||||
public IndexRecoveryResolver(MonitoredSystem id, int version, Settings settings) {
|
public IndexRecoveryResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -66,8 +66,8 @@ public class IndexStatsResolver extends MonitoringIndexNameResolver.Timestamped<
|
||||||
"index_stats.total.refresh.total_time_in_millis",
|
"index_stats.total.refresh.total_time_in_millis",
|
||||||
};
|
};
|
||||||
|
|
||||||
public IndexStatsResolver(MonitoredSystem id, int version, Settings settings) {
|
public IndexStatsResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -41,8 +41,8 @@ public class IndicesStatsResolver extends MonitoringIndexNameResolver.Timestampe
|
||||||
"indices_stats._all.total.store.size_in_bytes",
|
"indices_stats._all.total.store.size_in_bytes",
|
||||||
};
|
};
|
||||||
|
|
||||||
public IndicesStatsResolver(MonitoredSystem id, int version, Settings settings) {
|
public IndicesStatsResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -87,8 +87,8 @@ public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped<N
|
||||||
"node_stats.thread_pool.watcher.rejected",
|
"node_stats.thread_pool.watcher.rejected",
|
||||||
};
|
};
|
||||||
|
|
||||||
public NodeStatsResolver(MonitoredSystem id, int version, Settings settings) {
|
public NodeStatsResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -32,8 +32,8 @@ public class ShardsResolver extends MonitoringIndexNameResolver.Timestamped<Shar
|
||||||
"shard.index",
|
"shard.index",
|
||||||
};
|
};
|
||||||
|
|
||||||
public ShardsResolver(MonitoredSystem id, int version, Settings settings) {
|
public ShardsResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -8,8 +8,8 @@ package org.elasticsearch.marvel.agent.exporter;
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.marvel.MonitoringSettings;
|
|
||||||
import org.elasticsearch.marvel.MonitoredSystem;
|
import org.elasticsearch.marvel.MonitoredSystem;
|
||||||
|
import org.elasticsearch.marvel.MonitoringSettings;
|
||||||
import org.elasticsearch.marvel.agent.collector.Collector;
|
import org.elasticsearch.marvel.agent.collector.Collector;
|
||||||
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
|
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
|
||||||
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
|
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
|
||||||
|
@ -25,8 +25,6 @@ import static org.hamcrest.Matchers.notNullValue;
|
||||||
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
|
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
|
||||||
public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase {
|
public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase {
|
||||||
|
|
||||||
private final Integer currentVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Settings nodeSettings(int nodeOrdinal) {
|
protected Settings nodeSettings(int nodeOrdinal) {
|
||||||
Settings.Builder settings = Settings.builder()
|
Settings.Builder settings = Settings.builder()
|
||||||
|
@ -43,7 +41,7 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
|
|
||||||
protected abstract void deleteTemplates() throws Exception;
|
protected abstract void deleteTemplates() throws Exception;
|
||||||
|
|
||||||
protected abstract void putTemplate(String name, int version) throws Exception;
|
protected abstract void putTemplate(String name) throws Exception;
|
||||||
|
|
||||||
protected abstract void assertTemplateExist(String name) throws Exception;
|
protected abstract void assertTemplateExist(String name) throws Exception;
|
||||||
|
|
||||||
|
@ -70,15 +68,14 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
public void testCreateWhenExistingTemplatesAreOld() throws Exception {
|
public void testCreateWhenExistingTemplatesAreOld() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
final Integer version = randomIntBetween(0, currentVersion - 1);
|
putTemplate(indexTemplateName());
|
||||||
putTemplate(indexTemplateName(version), version);
|
putTemplate(dataTemplateName());
|
||||||
putTemplate(dataTemplateName(version), version);
|
|
||||||
|
|
||||||
doExporting();
|
doExporting();
|
||||||
|
|
||||||
logger.debug("--> existing templates are old");
|
logger.debug("--> existing templates are old");
|
||||||
assertTemplateExist(dataTemplateName(version));
|
assertTemplateExist(dataTemplateName());
|
||||||
assertTemplateExist(indexTemplateName(version));
|
assertTemplateExist(indexTemplateName());
|
||||||
|
|
||||||
logger.debug("--> existing templates are old: new templates should be created");
|
logger.debug("--> existing templates are old: new templates should be created");
|
||||||
for (String template : monitoringTemplates().keySet()) {
|
for (String template : monitoringTemplates().keySet()) {
|
||||||
|
@ -95,8 +92,8 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
public void testCreateWhenExistingTemplateAreUpToDate() throws Exception {
|
public void testCreateWhenExistingTemplateAreUpToDate() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
putTemplate(indexTemplateName(currentVersion), currentVersion);
|
putTemplate(indexTemplateName());
|
||||||
putTemplate(dataTemplateName(currentVersion), currentVersion);
|
putTemplate(dataTemplateName());
|
||||||
|
|
||||||
doExporting();
|
doExporting();
|
||||||
|
|
||||||
|
@ -106,8 +103,8 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug("--> existing templates has the same version: they should not be changed");
|
logger.debug("--> existing templates has the same version: they should not be changed");
|
||||||
assertTemplateNotUpdated(indexTemplateName(currentVersion));
|
assertTemplateNotUpdated(indexTemplateName());
|
||||||
assertTemplateNotUpdated(dataTemplateName(currentVersion));
|
assertTemplateNotUpdated(dataTemplateName());
|
||||||
|
|
||||||
doExporting();
|
doExporting();
|
||||||
|
|
||||||
|
@ -116,47 +113,6 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
awaitIndexExists(currentTimestampedIndexName());
|
awaitIndexExists(currentTimestampedIndexName());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRandomTemplates() throws Exception {
|
|
||||||
internalCluster().startNode();
|
|
||||||
|
|
||||||
int previousIndexTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
|
|
||||||
boolean previousIndexTemplateExist = randomBoolean();
|
|
||||||
if (previousIndexTemplateExist) {
|
|
||||||
logger.debug("--> creating index template in version [{}]", previousIndexTemplateVersion);
|
|
||||||
putTemplate(indexTemplateName(previousIndexTemplateVersion), previousIndexTemplateVersion);
|
|
||||||
}
|
|
||||||
|
|
||||||
int previousDataTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
|
|
||||||
boolean previousDataTemplateExist = randomBoolean();
|
|
||||||
if (previousDataTemplateExist) {
|
|
||||||
logger.debug("--> creating data template in version [{}]", previousDataTemplateVersion);
|
|
||||||
putTemplate(dataTemplateName(previousDataTemplateVersion), previousDataTemplateVersion);
|
|
||||||
}
|
|
||||||
|
|
||||||
doExporting();
|
|
||||||
|
|
||||||
logger.debug("--> templates should exist in current version");
|
|
||||||
for (String template : monitoringTemplates().keySet()) {
|
|
||||||
assertTemplateExist(template);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (previousIndexTemplateExist) {
|
|
||||||
logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion);
|
|
||||||
assertTemplateExist(indexTemplateName(previousIndexTemplateVersion));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (previousDataTemplateExist) {
|
|
||||||
logger.debug("--> data template should exist in version [{}]", previousDataTemplateVersion);
|
|
||||||
assertTemplateExist(dataTemplateName(previousDataTemplateVersion));
|
|
||||||
}
|
|
||||||
|
|
||||||
doExporting();
|
|
||||||
|
|
||||||
logger.debug("--> indices should exist in current version");
|
|
||||||
awaitIndexExists(currentDataIndexName());
|
|
||||||
awaitIndexExists(currentTimestampedIndexName());
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void doExporting() throws Exception {
|
protected void doExporting() throws Exception {
|
||||||
Collector collector = internalCluster().getInstance(ClusterStatsCollector.class);
|
Collector collector = internalCluster().getInstance(ClusterStatsCollector.class);
|
||||||
assertNotNull(collector);
|
assertNotNull(collector);
|
||||||
|
@ -169,18 +125,18 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
exporters.export(collector.collect());
|
exporters.export(collector.collect());
|
||||||
}
|
}
|
||||||
|
|
||||||
private String dataTemplateName(Integer version) {
|
private String dataTemplateName() {
|
||||||
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(version);
|
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver();
|
||||||
return resolver.templateName();
|
return resolver.templateName();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String indexTemplateName(Integer version) {
|
private String indexTemplateName() {
|
||||||
MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version, exporterSettings());
|
MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, exporterSettings());
|
||||||
return resolver.templateName();
|
return resolver.templateName();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String currentDataIndexName() {
|
private String currentDataIndexName() {
|
||||||
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(currentVersion);
|
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver();
|
||||||
return resolver.index(null);
|
return resolver.index(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,13 +144,12 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
||||||
MonitoringDoc doc = new MonitoringDoc(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString());
|
MonitoringDoc doc = new MonitoringDoc(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString());
|
||||||
doc.setTimestamp(System.currentTimeMillis());
|
doc.setTimestamp(System.currentTimeMillis());
|
||||||
|
|
||||||
MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, currentVersion,
|
MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, exporterSettings());
|
||||||
exporterSettings());
|
|
||||||
return resolver.index(doc);
|
return resolver.index(doc);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Generates a basic template **/
|
/** Generates a basic template **/
|
||||||
protected static BytesReference generateTemplateSource(String name, Integer version) throws IOException {
|
protected static BytesReference generateTemplateSource(String name) throws IOException {
|
||||||
return jsonBuilder().startObject()
|
return jsonBuilder().startObject()
|
||||||
.field("template", name)
|
.field("template", name)
|
||||||
.startObject("settings")
|
.startObject("settings")
|
||||||
|
|
|
@ -5,100 +5,31 @@
|
||||||
*/
|
*/
|
||||||
package org.elasticsearch.marvel.agent.exporter;
|
package org.elasticsearch.marvel.agent.exporter;
|
||||||
|
|
||||||
import org.apache.lucene.util.Constants;
|
|
||||||
import org.elasticsearch.ElasticsearchParseException;
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.common.bytes.BytesArray;
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
|
||||||
import org.elasticsearch.test.ESTestCase;
|
import org.elasticsearch.test.ESTestCase;
|
||||||
import org.hamcrest.Matcher;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.rules.ExpectedException;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.elasticsearch.xpack.template.TemplateUtilsTests.assertTemplate;
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static org.hamcrest.Matchers.equalTo;
|
||||||
import static org.hamcrest.Matchers.greaterThan;
|
import static org.hamcrest.Matchers.greaterThan;
|
||||||
import static org.hamcrest.Matchers.notNullValue;
|
import static org.hamcrest.Matchers.notNullValue;
|
||||||
|
|
||||||
public class MarvelTemplateUtilsTests extends ESTestCase {
|
public class MarvelTemplateUtilsTests extends ESTestCase {
|
||||||
|
|
||||||
private static final String TEST_TEMPLATE = "/monitoring-test.json";
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public ExpectedException expectedException = ExpectedException.none();
|
|
||||||
|
|
||||||
public void testLoadTemplate() throws IOException {
|
public void testLoadTemplate() throws IOException {
|
||||||
final int version = randomIntBetween(0, 10_000);
|
String source = MarvelTemplateUtils.loadTemplate("test");
|
||||||
String source = MarvelTemplateUtils.loadTemplate("test", version);
|
|
||||||
|
|
||||||
assertThat(source, notNullValue());
|
assertThat(source, notNullValue());
|
||||||
assertThat(source.length(), greaterThan(0));
|
assertThat(source.length(), greaterThan(0));
|
||||||
assertTemplate(source, equalTo("{\n" +
|
assertTemplate(source, equalTo("{\n" +
|
||||||
" \"template\": \".monitoring-data-" + version + "\",\n" +
|
" \"template\": \".monitoring-data-" + MarvelTemplateUtils.TEMPLATE_VERSION + "\",\n" +
|
||||||
" \"mappings\": {\n" +
|
" \"mappings\": {\n" +
|
||||||
" \"type_1\": {\n" +
|
" \"type_1\": {\n" +
|
||||||
" \"_meta\": {\n" +
|
" \"_meta\": {\n" +
|
||||||
" \"template.version\": \"" + version + "\"\n" +
|
" \"template.version\": \"" + MarvelTemplateUtils.TEMPLATE_VERSION + "\"\n" +
|
||||||
" }\n" +
|
" }\n" +
|
||||||
" }\n" +
|
" }\n" +
|
||||||
" }\n" +
|
" }\n" +
|
||||||
"}"));
|
"}"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testLoad() throws IOException {
|
|
||||||
BytesReference source = MarvelTemplateUtils.load(TEST_TEMPLATE);
|
|
||||||
assertThat(source, notNullValue());
|
|
||||||
assertThat(source.length(), greaterThan(0));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testValidateNullSource() {
|
|
||||||
expectedException.expect(ElasticsearchParseException.class);
|
|
||||||
expectedException.expectMessage("Monitoring template must not be null");
|
|
||||||
MarvelTemplateUtils.validate(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testValidateEmptySource() {
|
|
||||||
expectedException.expect(ElasticsearchParseException.class);
|
|
||||||
expectedException.expectMessage("Monitoring template must not be empty");
|
|
||||||
MarvelTemplateUtils.validate(new BytesArray(""));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testValidateInvalidSource() {
|
|
||||||
expectedException.expect(ElasticsearchParseException.class);
|
|
||||||
expectedException.expectMessage("Invalid monitoring template");
|
|
||||||
MarvelTemplateUtils.validate(new BytesArray("{\"foo\": \"bar"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testValidate() {
|
|
||||||
try {
|
|
||||||
MarvelTemplateUtils.validate(MarvelTemplateUtils.load(TEST_TEMPLATE));
|
|
||||||
} catch (Exception e) {
|
|
||||||
fail("failed to validate test template: " + e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testFilter() {
|
|
||||||
assertTemplate(MarvelTemplateUtils.filter(new BytesArray("${monitoring.template.version}"), 0), equalTo("0"));
|
|
||||||
assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), 1),
|
|
||||||
equalTo("{\"template\": \"test-1\"}"));
|
|
||||||
assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), 2),
|
|
||||||
equalTo("{\"template\": \"2-test\"}"));
|
|
||||||
assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), 3),
|
|
||||||
equalTo("{\"template\": \"test-3-test\"}"));
|
|
||||||
|
|
||||||
final int version = randomIntBetween(0, 100);
|
|
||||||
assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " +
|
|
||||||
"\"bar-${monitoring.template.version}\"}"), version),
|
|
||||||
equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void assertTemplate(String actual, Matcher<? super String> matcher) {
|
|
||||||
if (Constants.WINDOWS) {
|
|
||||||
// translate Windows line endings (\r\n) to standard ones (\n)
|
|
||||||
actual = Strings.replace(actual, System.lineSeparator(), "\n");
|
|
||||||
}
|
|
||||||
assertThat(actual, matcher);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,8 +74,8 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void putTemplate(String name, int version) throws Exception {
|
protected void putTemplate(String name) throws Exception {
|
||||||
dispatcher.templates.put(name, generateTemplateSource(name, version));
|
dispatcher.templates.put(name, generateTemplateSource(name));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -26,9 +26,9 @@ public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void putTemplate(String name, int version) throws Exception {
|
protected void putTemplate(String name) throws Exception {
|
||||||
waitNoPendingTasksOnAll();
|
waitNoPendingTasksOnAll();
|
||||||
assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name, version)).get());
|
assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name)).get());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -10,6 +10,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
|
||||||
import org.elasticsearch.common.transport.DummyTransportAddress;
|
import org.elasticsearch.common.transport.DummyTransportAddress;
|
||||||
import org.elasticsearch.common.xcontent.ToXContent;
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||||
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -20,11 +21,9 @@ import static org.hamcrest.Matchers.equalTo;
|
||||||
|
|
||||||
public class DataResolverTests extends MonitoringIndexNameResolverTestCase {
|
public class DataResolverTests extends MonitoringIndexNameResolverTestCase {
|
||||||
|
|
||||||
private int randomVersion = randomIntBetween(0, 100);
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
|
protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
|
||||||
return newDataResolver(randomVersion);
|
return newDataResolver();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -52,12 +51,11 @@ public class DataResolverTests extends MonitoringIndexNameResolverTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDataResolver() {
|
public void testDataResolver() {
|
||||||
assertThat(newDataResolver(randomVersion).index(newMarvelDoc()), equalTo(".monitoring-data-" + randomVersion));
|
assertThat(newDataResolver().index(newMarvelDoc()), equalTo(".monitoring-data-" + MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||||
assertThat(newDataResolver(42).index(newMarvelDoc()), equalTo(".monitoring-data-42"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private MonitoringIndexNameResolver.Data<MonitoringDoc> newDataResolver(int randomVersion) {
|
private MonitoringIndexNameResolver.Data<MonitoringDoc> newDataResolver() {
|
||||||
return new MonitoringIndexNameResolver.Data<MonitoringDoc>(randomVersion) {
|
return new MonitoringIndexNameResolver.Data<MonitoringDoc>() {
|
||||||
@Override
|
@Override
|
||||||
public String type(MonitoringDoc document) {
|
public String type(MonitoringDoc document) {
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -11,6 +11,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
|
||||||
import org.elasticsearch.common.xcontent.XContentType;
|
import org.elasticsearch.common.xcontent.XContentType;
|
||||||
import org.elasticsearch.common.xcontent.support.XContentMapValues;
|
import org.elasticsearch.common.xcontent.support.XContentMapValues;
|
||||||
import org.elasticsearch.marvel.MonitoredSystem;
|
import org.elasticsearch.marvel.MonitoredSystem;
|
||||||
|
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||||
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
||||||
import org.elasticsearch.test.ESTestCase;
|
import org.elasticsearch.test.ESTestCase;
|
||||||
|
|
||||||
|
@ -128,18 +129,17 @@ public abstract class MonitoringIndexNameResolverTestCase<M extends MonitoringDo
|
||||||
public void testResolver() {
|
public void testResolver() {
|
||||||
MonitoringIndexNameResolver resolver = newResolver();
|
MonitoringIndexNameResolver resolver = newResolver();
|
||||||
assertThat(resolver, notNullValue());
|
assertThat(resolver, notNullValue());
|
||||||
assertThat(resolver.getVersion(), greaterThanOrEqualTo(0));
|
|
||||||
|
|
||||||
if (resolver instanceof MonitoringIndexNameResolver.Timestamped) {
|
if (resolver instanceof MonitoringIndexNameResolver.Timestamped) {
|
||||||
MonitoringIndexNameResolver.Timestamped timestamped = (MonitoringIndexNameResolver.Timestamped) resolver;
|
MonitoringIndexNameResolver.Timestamped timestamped = (MonitoringIndexNameResolver.Timestamped) resolver;
|
||||||
assertThat(resolver.index(newMarvelDoc()),
|
assertThat(resolver.index(newMarvelDoc()),
|
||||||
startsWith(PREFIX + DELIMITER + timestamped.getId() + DELIMITER + timestamped.getVersion() + DELIMITER));
|
startsWith(PREFIX + DELIMITER + timestamped.getId() + DELIMITER + MarvelTemplateUtils.TEMPLATE_VERSION + DELIMITER));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (resolver instanceof MonitoringIndexNameResolver.Data) {
|
if (resolver instanceof MonitoringIndexNameResolver.Data) {
|
||||||
MonitoringIndexNameResolver.Data data = (MonitoringIndexNameResolver.Data) resolver;
|
MonitoringIndexNameResolver.Data data = (MonitoringIndexNameResolver.Data) resolver;
|
||||||
assertThat(resolver.index(newMarvelDoc()),
|
assertThat(resolver.index(newMarvelDoc()),
|
||||||
equalTo(PREFIX + DELIMITER + MonitoringIndexNameResolver.Data.DATA + DELIMITER + String.valueOf(data.getVersion())));
|
equalTo(PREFIX + DELIMITER + MonitoringIndexNameResolver.Data.DATA + DELIMITER + MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ import org.elasticsearch.common.transport.DummyTransportAddress;
|
||||||
import org.elasticsearch.common.xcontent.ToXContent;
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.marvel.MonitoredSystem;
|
import org.elasticsearch.marvel.MonitoredSystem;
|
||||||
|
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||||
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
|
||||||
import org.joda.time.format.DateTimeFormat;
|
import org.joda.time.format.DateTimeFormat;
|
||||||
|
|
||||||
|
@ -28,11 +29,10 @@ import static org.hamcrest.Matchers.notNullValue;
|
||||||
public class TimestampedResolverTests extends MonitoringIndexNameResolverTestCase {
|
public class TimestampedResolverTests extends MonitoringIndexNameResolverTestCase {
|
||||||
|
|
||||||
private MonitoredSystem randomId = randomFrom(MonitoredSystem.values());
|
private MonitoredSystem randomId = randomFrom(MonitoredSystem.values());
|
||||||
private int randomVersion = randomIntBetween(0, 100);
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
|
protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
|
||||||
return newTimestampedResolver(randomId, randomVersion, Settings.EMPTY);
|
return newTimestampedResolver(randomId, Settings.EMPTY);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -72,19 +72,17 @@ public class TimestampedResolverTests extends MonitoringIndexNameResolverTestCas
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
MonitoringIndexNameResolver.Timestamped resolver = newTimestampedResolver(randomId, randomVersion, settings);
|
MonitoringIndexNameResolver.Timestamped resolver = newTimestampedResolver(randomId, settings);
|
||||||
assertThat(resolver, notNullValue());
|
assertThat(resolver, notNullValue());
|
||||||
assertThat(resolver.getId(), equalTo(randomId.getSystem()));
|
assertThat(resolver.getId(), equalTo(randomId.getSystem()));
|
||||||
assertThat(resolver.getVersion(), equalTo(randomVersion));
|
|
||||||
assertThat(resolver.index(doc),
|
assertThat(resolver.index(doc),
|
||||||
equalTo(PREFIX + DELIMITER + resolver.getId() + DELIMITER + String.valueOf(resolver.getVersion())
|
equalTo(PREFIX + DELIMITER + resolver.getId() + DELIMITER + String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION)
|
||||||
+ DELIMITER + DateTimeFormat.forPattern(format).withZoneUTC().print(1437580442979L)));
|
+ DELIMITER + DateTimeFormat.forPattern(format).withZoneUTC().print(1437580442979L)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private MonitoringIndexNameResolver.Timestamped<MonitoringDoc> newTimestampedResolver(MonitoredSystem id, int version,
|
private MonitoringIndexNameResolver.Timestamped<MonitoringDoc> newTimestampedResolver(MonitoredSystem id, Settings settings) {
|
||||||
Settings settings) {
|
return new MonitoringIndexNameResolver.Timestamped<MonitoringDoc>(id, settings) {
|
||||||
return new MonitoringIndexNameResolver.Timestamped<MonitoringDoc>(id, version, settings) {
|
|
||||||
@Override
|
@Override
|
||||||
public String type(MonitoringDoc document) {
|
public String type(MonitoringDoc document) {
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -107,7 +107,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
|
||||||
public void testClusterStateNodes() throws Exception {
|
public void testClusterStateNodes() throws Exception {
|
||||||
final long nbNodes = internalCluster().size();
|
final long nbNodes = internalCluster().size();
|
||||||
|
|
||||||
MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES, TEMPLATE_VERSION);
|
MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES);
|
||||||
assertNotNull(timestampedResolver);
|
assertNotNull(timestampedResolver);
|
||||||
|
|
||||||
String timestampedIndex = timestampedResolver.indexPattern();
|
String timestampedIndex = timestampedResolver.indexPattern();
|
||||||
|
@ -152,7 +152,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
|
||||||
public void testDiscoveryNodes() throws Exception {
|
public void testDiscoveryNodes() throws Exception {
|
||||||
final long nbNodes = internalCluster().size();
|
final long nbNodes = internalCluster().size();
|
||||||
|
|
||||||
MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(TEMPLATE_VERSION);
|
MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver();
|
||||||
assertNotNull(dataResolver);
|
assertNotNull(dataResolver);
|
||||||
|
|
||||||
String dataIndex = dataResolver.indexPattern();
|
String dataIndex = dataResolver.indexPattern();
|
||||||
|
|
|
@ -19,6 +19,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
|
||||||
import org.elasticsearch.test.VersionUtils;
|
import org.elasticsearch.test.VersionUtils;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.joda.time.format.DateTimeFormat;
|
||||||
|
import org.joda.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
|
||||||
|
@ -46,7 +48,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
public void testDeleteIndex() throws Exception {
|
public void testDeleteIndex() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10));
|
createTimestampedIndex(now().minusDays(10));
|
||||||
assertIndicesCount(1);
|
assertIndicesCount(1);
|
||||||
|
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
@ -57,7 +59,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
public void testIgnoreCurrentDataIndex() throws Exception {
|
public void testIgnoreCurrentDataIndex() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
createDataIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10));
|
createDataIndex(now().minusDays(10));
|
||||||
assertIndicesCount(1);
|
assertIndicesCount(1);
|
||||||
|
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
@ -69,8 +71,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1));
|
createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1));
|
||||||
createDataIndex(0, now().minusDays(10));
|
createDataIndex(now().minusDays(10));
|
||||||
createDataIndex(Integer.MAX_VALUE, now().minusDays(20));
|
String olderIndex = String.join(MonitoringIndexNameResolver.DELIMITER, MonitoringIndexNameResolver.PREFIX,
|
||||||
|
MonitoringIndexNameResolver.Data.DATA, "1");
|
||||||
|
createIndex(olderIndex, now().minusDays(20));
|
||||||
|
|
||||||
assertIndicesCount(3);
|
assertIndicesCount(3);
|
||||||
|
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
@ -81,8 +86,8 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
public void testIgnoreCurrentTimestampedIndex() throws Exception {
|
public void testIgnoreCurrentTimestampedIndex() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10));
|
createTimestampedIndex(now().minusDays(10));
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now());
|
createTimestampedIndex(now());
|
||||||
assertIndicesCount(2);
|
assertIndicesCount(2);
|
||||||
|
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
@ -93,8 +98,13 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception {
|
public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception {
|
||||||
internalCluster().startNode();
|
internalCluster().startNode();
|
||||||
|
|
||||||
createTimestampedIndex(0, now().minusDays(10));
|
createTimestampedIndex(now().minusDays(10));
|
||||||
createTimestampedIndex(Integer.MAX_VALUE, now().minusDays(10));
|
// create an older index based on an older template
|
||||||
|
DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC();
|
||||||
|
String index = String.join(MonitoringIndexNameResolver.DELIMITER, MonitoringIndexNameResolver.PREFIX,
|
||||||
|
MonitoredSystem.ES.getSystem(), "1");
|
||||||
|
String timestampedIndex = String.join("-", index, formatter.print(now().minusDays(10)));
|
||||||
|
createIndex(timestampedIndex, now().minusDays(10));
|
||||||
assertIndicesCount(2);
|
assertIndicesCount(2);
|
||||||
|
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
@ -108,11 +118,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
CleanerService.Listener listener = getListener();
|
CleanerService.Listener listener = getListener();
|
||||||
|
|
||||||
final DateTime now = now();
|
final DateTime now = now();
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusYears(1));
|
createTimestampedIndex(now.minusYears(1));
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(6));
|
createTimestampedIndex(now.minusMonths(6));
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(1));
|
createTimestampedIndex(now.minusMonths(1));
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(10));
|
createTimestampedIndex(now.minusDays(10));
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(1));
|
createTimestampedIndex(now.minusDays(1));
|
||||||
assertIndicesCount(5);
|
assertIndicesCount(5);
|
||||||
|
|
||||||
// Clean indices that have expired two years ago
|
// Clean indices that have expired two years ago
|
||||||
|
@ -148,7 +158,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
|
|
||||||
final DateTime now = now();
|
final DateTime now = now();
|
||||||
for (int i = 0; i < max; i++) {
|
for (int i = 0; i < max; i++) {
|
||||||
createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(i));
|
createTimestampedIndex(now.minusDays(i));
|
||||||
}
|
}
|
||||||
assertIndicesCount(max);
|
assertIndicesCount(max);
|
||||||
|
|
||||||
|
@ -175,18 +185,18 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase
|
||||||
/**
|
/**
|
||||||
* Creates a monitoring data index in a given version.
|
* Creates a monitoring data index in a given version.
|
||||||
*/
|
*/
|
||||||
protected void createDataIndex(int version, DateTime creationDate) {
|
protected void createDataIndex(DateTime creationDate) {
|
||||||
createIndex(new MockDataIndexNameResolver(version).index(randomMonitoringDoc()), creationDate);
|
createIndex(new MockDataIndexNameResolver().index(randomMonitoringDoc()), creationDate);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a monitoring timestamped index in a given version.
|
* Creates a monitoring timestamped index in a given version.
|
||||||
*/
|
*/
|
||||||
protected void createTimestampedIndex(int version, DateTime creationDate) {
|
protected void createTimestampedIndex(DateTime creationDate) {
|
||||||
MonitoringDoc monitoringDoc = randomMonitoringDoc();
|
MonitoringDoc monitoringDoc = randomMonitoringDoc();
|
||||||
monitoringDoc.setTimestamp(creationDate.getMillis());
|
monitoringDoc.setTimestamp(creationDate.getMillis());
|
||||||
|
|
||||||
MonitoringIndexNameResolver.Timestamped resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version);
|
MonitoringIndexNameResolver.Timestamped resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES);
|
||||||
createIndex(resolver.index(monitoringDoc), creationDate);
|
createIndex(resolver.index(monitoringDoc), creationDate);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,8 +25,8 @@ import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
|
||||||
import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
|
import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
|
||||||
import org.elasticsearch.marvel.client.MonitoringClient;
|
import org.elasticsearch.marvel.client.MonitoringClient;
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
import org.elasticsearch.shield.authc.file.FileRealm;
|
|
||||||
import org.elasticsearch.shield.Security;
|
import org.elasticsearch.shield.Security;
|
||||||
|
import org.elasticsearch.shield.authc.file.FileRealm;
|
||||||
import org.elasticsearch.shield.authc.support.Hasher;
|
import org.elasticsearch.shield.authc.support.Hasher;
|
||||||
import org.elasticsearch.shield.authc.support.SecuredString;
|
import org.elasticsearch.shield.authc.support.SecuredString;
|
||||||
import org.elasticsearch.shield.authz.store.FileRolesStore;
|
import org.elasticsearch.shield.authz.store.FileRolesStore;
|
||||||
|
@ -438,11 +438,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
|
||||||
Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), value, timeUnit)));
|
Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), value, timeUnit)));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected class MockDataIndexNameResolver extends MonitoringIndexNameResolver.Data<MonitoringDoc> {
|
public class MockDataIndexNameResolver extends MonitoringIndexNameResolver.Data<MonitoringDoc> {
|
||||||
|
|
||||||
public MockDataIndexNameResolver(int version) {
|
|
||||||
super(version);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String type(MonitoringDoc document) {
|
public String type(MonitoringDoc document) {
|
||||||
|
@ -462,12 +458,12 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
|
||||||
|
|
||||||
protected class MockTimestampedIndexNameResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringDoc> {
|
protected class MockTimestampedIndexNameResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringDoc> {
|
||||||
|
|
||||||
public MockTimestampedIndexNameResolver(MonitoredSystem id, int version, Settings settings) {
|
public MockTimestampedIndexNameResolver(MonitoredSystem id, Settings settings) {
|
||||||
super(id, version, settings);
|
super(id, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MockTimestampedIndexNameResolver(MonitoredSystem id, int version) {
|
public MockTimestampedIndexNameResolver(MonitoredSystem id) {
|
||||||
this(id, version, Settings.EMPTY);
|
this(id, Settings.EMPTY);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
package org.elasticsearch.xpack.template;
|
||||||
|
|
||||||
|
import org.elasticsearch.ElasticsearchParseException;
|
||||||
|
import org.elasticsearch.common.bytes.BytesArray;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.common.compress.NotXContentException;
|
||||||
|
import org.elasticsearch.common.io.Streams;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handling versioned templates for time-based indices in x-pack
|
||||||
|
*/
|
||||||
|
public class TemplateUtils {
|
||||||
|
|
||||||
|
private TemplateUtils() {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads a built-in template and returns its source.
|
||||||
|
*/
|
||||||
|
public static String loadTemplate(String resource, String version, String versionProperty) {
|
||||||
|
try {
|
||||||
|
BytesReference source = load(resource);
|
||||||
|
validate(source);
|
||||||
|
|
||||||
|
return filter(source, version, versionProperty);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalArgumentException("Unable to load template [" + resource + "]", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads a resource from the classpath and returns it as a {@link BytesReference}
|
||||||
|
*/
|
||||||
|
public static BytesReference load(String name) throws IOException {
|
||||||
|
try (InputStream is = TemplateUtils.class.getResourceAsStream(name)) {
|
||||||
|
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||||
|
Streams.copy(is, out);
|
||||||
|
return new BytesArray(out.toByteArray());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses and validates that the source is not empty.
|
||||||
|
*/
|
||||||
|
public static void validate(BytesReference source) {
|
||||||
|
if (source == null) {
|
||||||
|
throw new ElasticsearchParseException("Template must not be null");
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
XContentHelper.convertToMap(source, false).v2();
|
||||||
|
} catch (NotXContentException e) {
|
||||||
|
throw new ElasticsearchParseException("Template must not be empty");
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ElasticsearchParseException("Invalid template", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Filters the source: replaces any template version property with the version number
|
||||||
|
*/
|
||||||
|
public static String filter(BytesReference source, String version, String versionProperty) {
|
||||||
|
return Pattern.compile(versionProperty)
|
||||||
|
.matcher(source.toUtf8())
|
||||||
|
.replaceAll(version);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,103 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
package org.elasticsearch.xpack.template;
|
||||||
|
|
||||||
|
import org.apache.lucene.util.Constants;
|
||||||
|
import org.elasticsearch.ElasticsearchParseException;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.bytes.BytesArray;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.test.ESTestCase;
|
||||||
|
import org.hamcrest.Matcher;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import static org.hamcrest.Matchers.equalTo;
|
||||||
|
import static org.hamcrest.Matchers.greaterThan;
|
||||||
|
import static org.hamcrest.Matchers.notNullValue;
|
||||||
|
import static org.hamcrest.core.Is.is;
|
||||||
|
|
||||||
|
public class TemplateUtilsTests extends ESTestCase {
|
||||||
|
|
||||||
|
private static final String TEST_TEMPLATE = "/monitoring-%s.json";
|
||||||
|
|
||||||
|
public void testLoadTemplate() throws IOException {
|
||||||
|
final int version = randomIntBetween(0, 10_000);
|
||||||
|
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
|
||||||
|
String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), Pattern.quote("${monitoring.template.version}"));
|
||||||
|
|
||||||
|
assertThat(source, notNullValue());
|
||||||
|
assertThat(source.length(), greaterThan(0));
|
||||||
|
assertTemplate(source, equalTo("{\n" +
|
||||||
|
" \"template\": \".monitoring-data-" + version + "\",\n" +
|
||||||
|
" \"mappings\": {\n" +
|
||||||
|
" \"type_1\": {\n" +
|
||||||
|
" \"_meta\": {\n" +
|
||||||
|
" \"template.version\": \"" + version + "\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
"}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testLoad() throws IOException {
|
||||||
|
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
|
||||||
|
BytesReference source = TemplateUtils.load(resource);
|
||||||
|
assertThat(source, notNullValue());
|
||||||
|
assertThat(source.length(), greaterThan(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testValidateNullSource() {
|
||||||
|
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> TemplateUtils.validate(null));
|
||||||
|
assertThat(exception.getMessage(), is("Template must not be null"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testValidateEmptySource() {
|
||||||
|
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
|
||||||
|
() -> TemplateUtils.validate(new BytesArray("")));
|
||||||
|
assertThat(exception.getMessage(), is("Template must not be empty"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testValidateInvalidSource() {
|
||||||
|
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
|
||||||
|
() -> TemplateUtils.validate(new BytesArray("{\"foo\": \"bar")));
|
||||||
|
assertThat(exception.getMessage(), is("Invalid template"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testValidate() throws IOException {
|
||||||
|
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
|
||||||
|
TemplateUtils.validate(TemplateUtils.load(resource));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testFilter() {
|
||||||
|
assertTemplate(TemplateUtils.filter(new BytesArray("${monitoring.template.version}"), "0",
|
||||||
|
Pattern.quote("${monitoring.template.version}")), equalTo("0"));
|
||||||
|
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), "1",
|
||||||
|
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-1\"}"));
|
||||||
|
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), "2",
|
||||||
|
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"2-test\"}"));
|
||||||
|
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), "3",
|
||||||
|
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-3-test\"}"));
|
||||||
|
|
||||||
|
final int version = randomIntBetween(0, 100);
|
||||||
|
assertTemplate(TemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " +
|
||||||
|
"\"bar-${monitoring.template.version}\"}"), String.valueOf(version),
|
||||||
|
Pattern.quote("${monitoring.template.version}")),
|
||||||
|
equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void assertTemplate(String actual, Matcher<? super String> matcher) {
|
||||||
|
if (Constants.WINDOWS) {
|
||||||
|
// translate Windows line endings (\r\n) to standard ones (\n)
|
||||||
|
actual = Strings.replace(actual, System.lineSeparator(), "\n");
|
||||||
|
}
|
||||||
|
assertThat(actual, matcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -47,6 +47,7 @@ import org.elasticsearch.watcher.rest.action.RestHijackOperationAction;
|
||||||
import org.elasticsearch.watcher.rest.action.RestPutWatchAction;
|
import org.elasticsearch.watcher.rest.action.RestPutWatchAction;
|
||||||
import org.elasticsearch.watcher.rest.action.RestWatchServiceAction;
|
import org.elasticsearch.watcher.rest.action.RestWatchServiceAction;
|
||||||
import org.elasticsearch.watcher.rest.action.RestWatcherStatsAction;
|
import org.elasticsearch.watcher.rest.action.RestWatcherStatsAction;
|
||||||
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
||||||
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
|
||||||
import org.elasticsearch.watcher.support.clock.ClockModule;
|
import org.elasticsearch.watcher.support.clock.ClockModule;
|
||||||
import org.elasticsearch.watcher.support.http.HttpClient;
|
import org.elasticsearch.watcher.support.http.HttpClient;
|
||||||
|
@ -168,7 +169,7 @@ public class Watcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void onModule(SettingsModule module) {
|
public void onModule(SettingsModule module) {
|
||||||
for (TemplateConfig templateConfig : WatcherModule.TEMPLATE_CONFIGS) {
|
for (TemplateConfig templateConfig : WatcherIndexTemplateRegistry.TEMPLATE_CONFIGS) {
|
||||||
module.registerSetting(templateConfig.getSetting());
|
module.registerSetting(templateConfig.getSetting());
|
||||||
}
|
}
|
||||||
module.registerSetting(InternalSlackService.SLACK_ACCOUNT_SETTING);
|
module.registerSetting(InternalSlackService.SLACK_ACCOUNT_SETTING);
|
||||||
|
|
|
@ -6,44 +6,18 @@
|
||||||
package org.elasticsearch.watcher;
|
package org.elasticsearch.watcher;
|
||||||
|
|
||||||
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.common.inject.AbstractModule;
|
import org.elasticsearch.common.inject.AbstractModule;
|
||||||
import org.elasticsearch.common.inject.multibindings.Multibinder;
|
import org.elasticsearch.common.inject.multibindings.Multibinder;
|
||||||
import org.elasticsearch.common.inject.util.Providers;
|
import org.elasticsearch.common.inject.util.Providers;
|
||||||
import org.elasticsearch.common.settings.Setting;
|
import org.elasticsearch.common.settings.Setting;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
||||||
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
|
|
||||||
import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation;
|
import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation;
|
||||||
import org.elasticsearch.xpack.XPackPlugin;
|
import org.elasticsearch.xpack.XPackPlugin;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
|
|
||||||
public class WatcherModule extends AbstractModule {
|
public class WatcherModule extends AbstractModule {
|
||||||
|
|
||||||
static final String PROPERTIES_FILE = "/watcher.properties";
|
|
||||||
static final String VERSION_FIELD = "xpack.watcher.template.version";
|
|
||||||
|
|
||||||
public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + getHistoryIndexTemplateVersion();
|
|
||||||
public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches";
|
|
||||||
public static final String WATCHES_TEMPLATE_NAME = "watches";
|
|
||||||
public static final Setting<Settings> HISTORY_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.history.index.",
|
|
||||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
|
||||||
public static final Setting<Settings> TRIGGERED_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.triggered_watches.index.",
|
|
||||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
|
||||||
public static final Setting<Settings> WATCHES_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.watches.index.",
|
|
||||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
|
||||||
|
|
||||||
|
|
||||||
public final static TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{
|
|
||||||
new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING),
|
|
||||||
new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING),
|
|
||||||
new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING)
|
|
||||||
};
|
|
||||||
|
|
||||||
private final boolean enabled;
|
private final boolean enabled;
|
||||||
private final boolean transportClientMode;
|
private final boolean transportClientMode;
|
||||||
|
|
||||||
|
@ -64,28 +38,6 @@ public class WatcherModule extends AbstractModule {
|
||||||
bind(WatcherLicensee.class).asEagerSingleton();
|
bind(WatcherLicensee.class).asEagerSingleton();
|
||||||
bind(WatcherLifeCycleService.class).asEagerSingleton();
|
bind(WatcherLifeCycleService.class).asEagerSingleton();
|
||||||
bind(WatcherSettingsValidation.class).asEagerSingleton();
|
bind(WatcherSettingsValidation.class).asEagerSingleton();
|
||||||
|
|
||||||
bind(WatcherIndexTemplateRegistry.class).asEagerSingleton();
|
bind(WatcherIndexTemplateRegistry.class).asEagerSingleton();
|
||||||
Multibinder<TemplateConfig> multibinder
|
|
||||||
= Multibinder.newSetBinder(binder(), TemplateConfig.class);
|
|
||||||
for (TemplateConfig templateConfig : TEMPLATE_CONFIGS) {
|
|
||||||
multibinder.addBinding().toInstance(templateConfig);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Integer getHistoryIndexTemplateVersion() {
|
|
||||||
try (InputStream is = WatcherModule.class.getResourceAsStream(PROPERTIES_FILE)) {
|
|
||||||
Properties properties = new Properties();
|
|
||||||
properties.load(is);
|
|
||||||
String version = properties.getProperty(VERSION_FIELD);
|
|
||||||
if (Strings.hasLength(version)) {
|
|
||||||
return Integer.parseInt(version);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
} catch (NumberFormatException e) {
|
|
||||||
throw new IllegalArgumentException("failed to parse watcher template version");
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new IllegalArgumentException("failed to load watcher template version");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
import org.elasticsearch.index.engine.VersionConflictEngineException;
|
import org.elasticsearch.index.engine.VersionConflictEngineException;
|
||||||
import org.elasticsearch.watcher.WatcherModule;
|
import org.elasticsearch.watcher.WatcherModule;
|
||||||
import org.elasticsearch.watcher.execution.ExecutionState;
|
import org.elasticsearch.watcher.execution.ExecutionState;
|
||||||
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
||||||
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
import org.joda.time.format.DateTimeFormat;
|
import org.joda.time.format.DateTimeFormat;
|
||||||
|
@ -31,7 +32,7 @@ import static org.elasticsearch.watcher.support.Exceptions.ioException;
|
||||||
*/
|
*/
|
||||||
public class HistoryStore extends AbstractComponent {
|
public class HistoryStore extends AbstractComponent {
|
||||||
|
|
||||||
public static final String INDEX_PREFIX = ".watcher-history-" + WatcherModule.getHistoryIndexTemplateVersion() + "-";
|
public static final String INDEX_PREFIX = ".watcher-history-" + WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION + "-";
|
||||||
public static final String DOC_TYPE = "watch_record";
|
public static final String DOC_TYPE = "watch_record";
|
||||||
|
|
||||||
static final DateTimeFormatter indexTimeFormat = DateTimeFormat.forPattern("YYYY.MM.dd");
|
static final DateTimeFormatter indexTimeFormat = DateTimeFormat.forPattern("YYYY.MM.dd");
|
||||||
|
|
|
@ -5,54 +5,69 @@
|
||||||
*/
|
*/
|
||||||
package org.elasticsearch.watcher.support;
|
package org.elasticsearch.watcher.support;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
||||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
|
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
|
||||||
import org.elasticsearch.cluster.ClusterChangedEvent;
|
import org.elasticsearch.cluster.ClusterChangedEvent;
|
||||||
import org.elasticsearch.cluster.service.ClusterService;
|
|
||||||
import org.elasticsearch.cluster.ClusterState;
|
import org.elasticsearch.cluster.ClusterState;
|
||||||
import org.elasticsearch.cluster.ClusterStateListener;
|
import org.elasticsearch.cluster.ClusterStateListener;
|
||||||
|
import org.elasticsearch.cluster.service.ClusterService;
|
||||||
import org.elasticsearch.common.component.AbstractComponent;
|
import org.elasticsearch.common.component.AbstractComponent;
|
||||||
import org.elasticsearch.common.inject.Inject;
|
import org.elasticsearch.common.inject.Inject;
|
||||||
import org.elasticsearch.common.io.Streams;
|
|
||||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
|
||||||
import org.elasticsearch.common.settings.ClusterSettings;
|
import org.elasticsearch.common.settings.ClusterSettings;
|
||||||
import org.elasticsearch.common.settings.Setting;
|
import org.elasticsearch.common.settings.Setting;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
import org.elasticsearch.gateway.GatewayService;
|
import org.elasticsearch.gateway.GatewayService;
|
||||||
import org.elasticsearch.threadpool.ThreadPool;
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
||||||
import org.elasticsearch.watcher.watch.WatchStore;
|
import org.elasticsearch.xpack.template.TemplateUtils;
|
||||||
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.Executor;
|
import java.util.concurrent.Executor;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
import static java.util.Collections.unmodifiableMap;
|
import static java.util.Collections.unmodifiableMap;
|
||||||
import static java.util.Collections.unmodifiableSet;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*/
|
*/
|
||||||
public class WatcherIndexTemplateRegistry extends AbstractComponent implements ClusterStateListener {
|
public class WatcherIndexTemplateRegistry extends AbstractComponent implements ClusterStateListener {
|
||||||
|
|
||||||
private static final String FORBIDDEN_INDEX_SETTING = "index.mapper.dynamic";
|
private static final String FORBIDDEN_INDEX_SETTING = "index.mapper.dynamic";
|
||||||
|
public static final String INDEX_TEMPLATE_VERSION = "1";
|
||||||
|
|
||||||
|
public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + INDEX_TEMPLATE_VERSION;
|
||||||
|
public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches";
|
||||||
|
public static final String WATCHES_TEMPLATE_NAME = "watches";
|
||||||
|
|
||||||
|
public static final Setting<Settings> HISTORY_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.history.index.",
|
||||||
|
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||||
|
public static final Setting<Settings> TRIGGERED_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.triggered_watches.index.",
|
||||||
|
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||||
|
public static final Setting<Settings> WATCHES_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.watches.index.",
|
||||||
|
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||||
|
public static final TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{
|
||||||
|
new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING),
|
||||||
|
new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING),
|
||||||
|
new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING)
|
||||||
|
};
|
||||||
|
|
||||||
private final WatcherClientProxy client;
|
private final WatcherClientProxy client;
|
||||||
private final ThreadPool threadPool;
|
private final ThreadPool threadPool;
|
||||||
private final ClusterService clusterService;
|
private final ClusterService clusterService;
|
||||||
private final Set<TemplateConfig> indexTemplates;
|
private final TemplateConfig[] indexTemplates;
|
||||||
|
|
||||||
private volatile Map<String, Settings> customIndexSettings;
|
private volatile Map<String, Settings> customIndexSettings;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public WatcherIndexTemplateRegistry(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
|
public WatcherIndexTemplateRegistry(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
|
||||||
ThreadPool threadPool, WatcherClientProxy client, Set<TemplateConfig> configs) {
|
ThreadPool threadPool, WatcherClientProxy client) {
|
||||||
super(settings);
|
super(settings);
|
||||||
this.client = client;
|
this.client = client;
|
||||||
this.threadPool = threadPool;
|
this.threadPool = threadPool;
|
||||||
this.clusterService = clusterService;
|
this.clusterService = clusterService;
|
||||||
this.indexTemplates = unmodifiableSet(new HashSet<>(configs));
|
this.indexTemplates = TEMPLATE_CONFIGS;
|
||||||
clusterService.add(this);
|
clusterService.add(this);
|
||||||
|
|
||||||
Map<String, Settings> customIndexSettings = new HashMap<>();
|
Map<String, Settings> customIndexSettings = new HashMap<>();
|
||||||
|
@ -150,21 +165,12 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
|
||||||
} else {
|
} else {
|
||||||
executor = threadPool.generic();
|
executor = threadPool.generic();
|
||||||
}
|
}
|
||||||
executor.execute(new Runnable() {
|
executor.execute(() -> {
|
||||||
@Override
|
final byte[] template = TemplateUtils.loadTemplate("/" + config.getFileName()+ ".json", INDEX_TEMPLATE_VERSION,
|
||||||
public void run() {
|
Pattern.quote("${xpack.watcher.template.version}")).getBytes(Charsets.UTF_8);
|
||||||
try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getFileName()+ ".json")) {
|
|
||||||
if (is == null) {
|
|
||||||
logger.error("Resource [/{}.json] not found in classpath", config.getFileName());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
final byte[] template;
|
|
||||||
try (BytesStreamOutput out = new BytesStreamOutput()) {
|
|
||||||
Streams.copy(is, out);
|
|
||||||
template = out.bytes().toBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template);
|
PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template);
|
||||||
|
request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
|
||||||
Settings customSettings = customIndexSettings.get(config.getSetting().getKey());
|
Settings customSettings = customIndexSettings.get(config.getSetting().getKey());
|
||||||
if (customSettings != null && customSettings.names().size() > 0) {
|
if (customSettings != null && customSettings.names().size() > 0) {
|
||||||
Settings updatedSettings = Settings.builder()
|
Settings updatedSettings = Settings.builder()
|
||||||
|
@ -174,9 +180,8 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
|
||||||
request.settings(updatedSettings);
|
request.settings(updatedSettings);
|
||||||
}
|
}
|
||||||
PutIndexTemplateResponse response = client.putTemplate(request);
|
PutIndexTemplateResponse response = client.putTemplate(request);
|
||||||
} catch (Exception e) {
|
if (response.isAcknowledged() == false) {
|
||||||
logger.error("failed to load [{}.json]", e, config.getTemplateName());
|
logger.error("Error adding watcher template [{}], request was not acknowledged", config.getTemplateName());
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.test.ESIntegTestCase;
|
import org.elasticsearch.test.ESIntegTestCase;
|
||||||
import org.elasticsearch.test.junit.annotations.TestLogging;
|
import org.elasticsearch.test.junit.annotations.TestLogging;
|
||||||
import org.elasticsearch.watcher.WatcherModule;
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
||||||
import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase;
|
import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase;
|
||||||
|
|
||||||
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
|
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
|
||||||
|
@ -23,8 +23,10 @@ import static org.hamcrest.core.Is.is;
|
||||||
@TestLogging("cluster:DEBUG,action.admin.cluster.settings:DEBUG,watcher:DEBUG")
|
@TestLogging("cluster:DEBUG,action.admin.cluster.settings:DEBUG,watcher:DEBUG")
|
||||||
@ESIntegTestCase.ClusterScope(scope = TEST, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 1)
|
@ESIntegTestCase.ClusterScope(scope = TEST, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 1)
|
||||||
public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCase {
|
public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCase {
|
||||||
|
|
||||||
public void testChangeSettings() throws Exception {
|
public void testChangeSettings() throws Exception {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1"));
|
||||||
// this isn't defined in the template, so we rely on ES's default, which is zero
|
// this isn't defined in the template, so we rely on ES's default, which is zero
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), nullValue());
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), nullValue());
|
||||||
|
@ -44,7 +46,7 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices()
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
.prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2"));
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), equalTo("2"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), equalTo("2"));
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.refresh_interval"), equalTo("5m"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.refresh_interval"), equalTo("5m"));
|
||||||
|
@ -53,7 +55,8 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testChangeSettingsIgnoringForbiddenSetting() throws Exception {
|
public void testChangeSettingsIgnoringForbiddenSetting() throws Exception {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1"));
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
||||||
|
|
||||||
|
@ -70,7 +73,7 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices()
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
.prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2"));
|
assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2"));
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase;
|
||||||
import org.elasticsearch.watcher.WatcherModule;
|
import org.elasticsearch.watcher.WatcherModule;
|
||||||
import org.elasticsearch.watcher.execution.ExecutionState;
|
import org.elasticsearch.watcher.execution.ExecutionState;
|
||||||
import org.elasticsearch.watcher.execution.Wid;
|
import org.elasticsearch.watcher.execution.Wid;
|
||||||
|
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
|
||||||
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
|
||||||
import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent;
|
import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
@ -70,7 +71,7 @@ public class HistoryStoreTests extends ESTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testIndexNameGeneration() {
|
public void testIndexNameGeneration() {
|
||||||
Integer indexTemplateVersion = WatcherModule.getHistoryIndexTemplateVersion();
|
String indexTemplateVersion = WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION;
|
||||||
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(0, UTC)),
|
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(0, UTC)),
|
||||||
equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01"));
|
equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01"));
|
||||||
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(100000000000L, UTC)),
|
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(100000000000L, UTC)),
|
||||||
|
|
|
@ -47,7 +47,7 @@ public class WatcherIndexTemplateRegistryTests extends AbstractWatcherIntegratio
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices()
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
.prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().size(), equalTo(1));
|
assertThat(response.getIndexTemplates().size(), equalTo(1));
|
||||||
// setting from the file on the classpath:
|
// setting from the file on the classpath:
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
||||||
|
@ -57,13 +57,13 @@ public class WatcherIndexTemplateRegistryTests extends AbstractWatcherIntegratio
|
||||||
});
|
});
|
||||||
|
|
||||||
// Now delete the index template and verify the index template gets added back:
|
// Now delete the index template and verify the index template gets added back:
|
||||||
assertAcked(client().admin().indices().prepareDeleteTemplate(WatcherModule.HISTORY_TEMPLATE_NAME).get());
|
assertAcked(client().admin().indices().prepareDeleteTemplate(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get());
|
||||||
|
|
||||||
assertBusy(new Runnable() {
|
assertBusy(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
GetIndexTemplatesResponse response = client().admin().indices()
|
GetIndexTemplatesResponse response = client().admin().indices()
|
||||||
.prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get();
|
.prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get();
|
||||||
assertThat(response.getIndexTemplates().size(), equalTo(1));
|
assertThat(response.getIndexTemplates().size(), equalTo(1));
|
||||||
// setting from the file on the classpath:
|
// setting from the file on the classpath:
|
||||||
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false));
|
||||||
|
|
|
@ -27,8 +27,8 @@ import org.elasticsearch.plugins.Plugin;
|
||||||
import org.elasticsearch.script.MockMustacheScriptEngine;
|
import org.elasticsearch.script.MockMustacheScriptEngine;
|
||||||
import org.elasticsearch.search.SearchHit;
|
import org.elasticsearch.search.SearchHit;
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||||
import org.elasticsearch.shield.authc.file.FileRealm;
|
|
||||||
import org.elasticsearch.shield.Security;
|
import org.elasticsearch.shield.Security;
|
||||||
|
import org.elasticsearch.shield.authc.file.FileRealm;
|
||||||
import org.elasticsearch.shield.authc.support.Hasher;
|
import org.elasticsearch.shield.authc.support.Hasher;
|
||||||
import org.elasticsearch.shield.authc.support.SecuredString;
|
import org.elasticsearch.shield.authc.support.SecuredString;
|
||||||
import org.elasticsearch.shield.authz.store.FileRolesStore;
|
import org.elasticsearch.shield.authz.store.FileRolesStore;
|
||||||
|
@ -40,7 +40,6 @@ import org.elasticsearch.test.store.MockFSIndexStore;
|
||||||
import org.elasticsearch.test.transport.AssertingLocalTransport;
|
import org.elasticsearch.test.transport.AssertingLocalTransport;
|
||||||
import org.elasticsearch.test.transport.MockTransportService;
|
import org.elasticsearch.test.transport.MockTransportService;
|
||||||
import org.elasticsearch.watcher.WatcherLifeCycleService;
|
import org.elasticsearch.watcher.WatcherLifeCycleService;
|
||||||
import org.elasticsearch.watcher.WatcherModule;
|
|
||||||
import org.elasticsearch.watcher.WatcherService;
|
import org.elasticsearch.watcher.WatcherService;
|
||||||
import org.elasticsearch.watcher.WatcherState;
|
import org.elasticsearch.watcher.WatcherState;
|
||||||
import org.elasticsearch.watcher.actions.email.service.Authentication;
|
import org.elasticsearch.watcher.actions.email.service.Authentication;
|
||||||
|
@ -91,9 +90,9 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
|
||||||
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
|
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
|
||||||
import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
|
import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
|
||||||
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
|
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
|
||||||
import static org.elasticsearch.watcher.WatcherModule.HISTORY_TEMPLATE_NAME;
|
import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME;
|
||||||
import static org.elasticsearch.watcher.WatcherModule.TRIGGERED_TEMPLATE_NAME;
|
import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME;
|
||||||
import static org.elasticsearch.watcher.WatcherModule.WATCHES_TEMPLATE_NAME;
|
import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME;
|
||||||
import static org.hamcrest.Matchers.emptyArray;
|
import static org.hamcrest.Matchers.emptyArray;
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static org.hamcrest.Matchers.equalTo;
|
||||||
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
||||||
|
@ -143,7 +142,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
|
||||||
@Override
|
@Override
|
||||||
protected Set<String> excludeTemplates() {
|
protected Set<String> excludeTemplates() {
|
||||||
Set<String> excludes = new HashSet<>();
|
Set<String> excludes = new HashSet<>();
|
||||||
for (WatcherIndexTemplateRegistry.TemplateConfig templateConfig : WatcherModule.TEMPLATE_CONFIGS) {
|
for (WatcherIndexTemplateRegistry.TemplateConfig templateConfig : WatcherIndexTemplateRegistry.TEMPLATE_CONFIGS) {
|
||||||
excludes.add(templateConfig.getTemplateName());
|
excludes.add(templateConfig.getTemplateName());
|
||||||
}
|
}
|
||||||
return excludes;
|
return excludes;
|
||||||
|
|
Loading…
Reference in New Issue