diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java index 5c3163b2da8..65ef16e108e 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java @@ -5,16 +5,8 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.NotXContentException; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xpack.template.TemplateUtils; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.Locale; import java.util.regex.Pattern; @@ -29,56 +21,8 @@ public final class MarvelTemplateUtils { private MarvelTemplateUtils() { } - /** - * Loads a built-in template and returns its source. - */ - public static String loadTemplate(String id, Integer version) { + public static String loadTemplate(String id) { String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id); - try { - BytesReference source = load(resource); - validate(source); - - return filter(source, version); - } catch (Exception e) { - throw new IllegalArgumentException("Unable to load monitoring template [" + resource + "]", e); - } - } - - /** - * Loads a resource from the classpath and returns it as a {@link BytesReference} - */ - static BytesReference load(String name) throws IOException { - try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) { - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - Streams.copy(is, out); - return new BytesArray(out.toByteArray()); - } - } - } - - /** - * Parses and validates that the source is not empty. - */ - static void validate(BytesReference source) { - if (source == null) { - throw new ElasticsearchParseException("Monitoring template must not be null"); - } - - try { - XContentHelper.convertToMap(source, false).v2(); - } catch (NotXContentException e) { - throw new ElasticsearchParseException("Monitoring template must not be empty"); - } catch (Exception e) { - throw new ElasticsearchParseException("Invalid monitoring template", e); - } - } - - /** - * Filters the source: replaces any template version property with the version number - */ - static String filter(BytesReference source, Integer version) { - return Pattern.compile(TEMPLATE_VERSION_PROPERTY) - .matcher(source.toUtf8()) - .replaceAll(String.valueOf(version)); + return TemplateUtils.loadTemplate(resource, String.valueOf(TEMPLATE_VERSION), TEMPLATE_VERSION_PROPERTY); } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java index add17936baa..6b89eec592b 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java @@ -35,12 +35,6 @@ public abstract class MonitoringIndexNameResolver { public static final String PREFIX = ".monitoring"; public static final String DELIMITER = "-"; - private final int version; - - protected MonitoringIndexNameResolver(int version) { - this.version = version; - } - /** * Returns the name of the index in which the monitoring document must be indexed. * @@ -111,10 +105,6 @@ public abstract class MonitoringIndexNameResolver { */ public abstract String template(); - int getVersion() { - return version; - } - /** * @return the filters used when rendering the document. * If null or empty, no filtering is applied. @@ -142,9 +132,8 @@ public abstract class MonitoringIndexNameResolver { private final String index; - public Data(int version) { - super(version); - this.index = String.join(DELIMITER, PREFIX, DATA, String.valueOf(version)); + public Data() { + this.index = String.join(DELIMITER, PREFIX, DATA, String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION)); } @Override @@ -159,12 +148,12 @@ public abstract class MonitoringIndexNameResolver { @Override public String templateName() { - return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, getVersion()); + return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, MarvelTemplateUtils.TEMPLATE_VERSION); } @Override public String template() { - return MarvelTemplateUtils.loadTemplate(DATA, getVersion()); + return MarvelTemplateUtils.loadTemplate(DATA); } } @@ -181,10 +170,9 @@ public abstract class MonitoringIndexNameResolver { private final DateTimeFormatter formatter; private final String index; - public Timestamped(MonitoredSystem system, int version, Settings settings) { - super(version); + public Timestamped(MonitoredSystem system, Settings settings) { this.system = system; - this.index = String.join(DELIMITER, PREFIX, system.getSystem(), String.valueOf(getVersion())); + this.index = String.join(DELIMITER, PREFIX, system.getSystem(), String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION)); String format = INDEX_NAME_TIME_FORMAT_SETTING.get(settings); try { this.formatter = DateTimeFormat.forPattern(format).withZoneUTC(); @@ -211,12 +199,12 @@ public abstract class MonitoringIndexNameResolver { @Override public String templateName() { - return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), getVersion()); + return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), MarvelTemplateUtils.TEMPLATE_VERSION); } @Override public String template() { - return MarvelTemplateUtils.loadTemplate(getId(), getVersion()); + return MarvelTemplateUtils.loadTemplate(getId()); } String getId() { diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java index 6f14a53c77a..86d5d9c3913 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java @@ -19,7 +19,6 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMonitoringDoc; import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMonitoringDoc; import org.elasticsearch.marvel.agent.collector.node.NodeStatsMonitoringDoc; import org.elasticsearch.marvel.agent.collector.shards.ShardMonitoringDoc; -import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterInfoResolver; @@ -43,40 +42,37 @@ import static org.elasticsearch.marvel.MonitoredSystem.ES; public class ResolversRegistry implements Iterable { - private static final int ES_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION; - private static final int KIBANA_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION; - private final List registrations = new ArrayList<>(); public ResolversRegistry(Settings settings) { // register built-in defaults resolvers - registerBuiltIn(ES, ES_TEMPLATE_VERSION, settings); + registerBuiltIn(ES, settings); // register resolvers for monitored systems - registerMonitoredSystem(MonitoredSystem.KIBANA, KIBANA_TEMPLATE_VERSION, settings); + registerMonitoredSystem(MonitoredSystem.KIBANA, settings); } /** * Registers resolvers for elasticsearch documents collected by the monitoring plugin */ - private void registerBuiltIn(MonitoredSystem id, int version, Settings settings) { - registrations.add(resolveByClass(ClusterInfoMonitoringDoc.class, new ClusterInfoResolver(version))); - registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, version, settings))); - registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, version, settings))); - registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, version, settings))); - registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver(version))); - registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, version, settings))); - registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, version, settings))); - registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, version, settings))); - registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, version, settings))); - registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, version, settings))); + private void registerBuiltIn(MonitoredSystem id, Settings settings) { + registrations.add(resolveByClass(ClusterInfoMonitoringDoc.class, new ClusterInfoResolver())); + registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, settings))); + registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, settings))); + registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings))); + registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver())); + registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings))); + registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, settings))); + registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings))); + registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, settings))); + registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, settings))); } /** * Registers resolvers for monitored systems */ - private void registerMonitoredSystem(MonitoredSystem id, int version, Settings settings) { - final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, version, settings); + private void registerMonitoredSystem(MonitoredSystem id, Settings settings) { + final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, settings); registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver)); } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java index e92b3e34452..e4fa402075d 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java @@ -17,8 +17,8 @@ import java.io.IOException; public class MonitoringBulkResolver extends MonitoringIndexNameResolver.Timestamped { - public MonitoringBulkResolver(MonitoredSystem id, int version, Settings settings) { - super(id, version, settings); + public MonitoringBulkResolver(MonitoredSystem id, Settings settings) { + super(id, settings); } @Override diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoResolver.java index 407d3eb2659..cab0a37cb88 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoResolver.java @@ -22,10 +22,6 @@ public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data existing templates are old"); - assertTemplateExist(dataTemplateName(version)); - assertTemplateExist(indexTemplateName(version)); + assertTemplateExist(dataTemplateName()); + assertTemplateExist(indexTemplateName()); logger.debug("--> existing templates are old: new templates should be created"); for (String template : monitoringTemplates().keySet()) { @@ -95,8 +92,8 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa public void testCreateWhenExistingTemplateAreUpToDate() throws Exception { internalCluster().startNode(); - putTemplate(indexTemplateName(currentVersion), currentVersion); - putTemplate(dataTemplateName(currentVersion), currentVersion); + putTemplate(indexTemplateName()); + putTemplate(dataTemplateName()); doExporting(); @@ -106,8 +103,8 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa } logger.debug("--> existing templates has the same version: they should not be changed"); - assertTemplateNotUpdated(indexTemplateName(currentVersion)); - assertTemplateNotUpdated(dataTemplateName(currentVersion)); + assertTemplateNotUpdated(indexTemplateName()); + assertTemplateNotUpdated(dataTemplateName()); doExporting(); @@ -116,47 +113,6 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa awaitIndexExists(currentTimestampedIndexName()); } - public void testRandomTemplates() throws Exception { - internalCluster().startNode(); - - int previousIndexTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1); - boolean previousIndexTemplateExist = randomBoolean(); - if (previousIndexTemplateExist) { - logger.debug("--> creating index template in version [{}]", previousIndexTemplateVersion); - putTemplate(indexTemplateName(previousIndexTemplateVersion), previousIndexTemplateVersion); - } - - int previousDataTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1); - boolean previousDataTemplateExist = randomBoolean(); - if (previousDataTemplateExist) { - logger.debug("--> creating data template in version [{}]", previousDataTemplateVersion); - putTemplate(dataTemplateName(previousDataTemplateVersion), previousDataTemplateVersion); - } - - doExporting(); - - logger.debug("--> templates should exist in current version"); - for (String template : monitoringTemplates().keySet()) { - assertTemplateExist(template); - } - - if (previousIndexTemplateExist) { - logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion); - assertTemplateExist(indexTemplateName(previousIndexTemplateVersion)); - } - - if (previousDataTemplateExist) { - logger.debug("--> data template should exist in version [{}]", previousDataTemplateVersion); - assertTemplateExist(dataTemplateName(previousDataTemplateVersion)); - } - - doExporting(); - - logger.debug("--> indices should exist in current version"); - awaitIndexExists(currentDataIndexName()); - awaitIndexExists(currentTimestampedIndexName()); - } - protected void doExporting() throws Exception { Collector collector = internalCluster().getInstance(ClusterStatsCollector.class); assertNotNull(collector); @@ -169,18 +125,18 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa exporters.export(collector.collect()); } - private String dataTemplateName(Integer version) { - MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(version); + private String dataTemplateName() { + MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(); return resolver.templateName(); } - private String indexTemplateName(Integer version) { - MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version, exporterSettings()); + private String indexTemplateName() { + MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, exporterSettings()); return resolver.templateName(); } private String currentDataIndexName() { - MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(currentVersion); + MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(); return resolver.index(null); } @@ -188,13 +144,12 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa MonitoringDoc doc = new MonitoringDoc(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString()); doc.setTimestamp(System.currentTimeMillis()); - MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, currentVersion, - exporterSettings()); + MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, exporterSettings()); return resolver.index(doc); } /** Generates a basic template **/ - protected static BytesReference generateTemplateSource(String name, Integer version) throws IOException { + protected static BytesReference generateTemplateSource(String name) throws IOException { return jsonBuilder().startObject() .field("template", name) .startObject("settings") diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java index 8138d8520e5..c70566898b8 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java @@ -5,100 +5,31 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.apache.lucene.util.Constants; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matcher; -import org.junit.Rule; -import org.junit.rules.ExpectedException; import java.io.IOException; +import static org.elasticsearch.xpack.template.TemplateUtilsTests.assertTemplate; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.notNullValue; public class MarvelTemplateUtilsTests extends ESTestCase { - private static final String TEST_TEMPLATE = "/monitoring-test.json"; - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - public void testLoadTemplate() throws IOException { - final int version = randomIntBetween(0, 10_000); - String source = MarvelTemplateUtils.loadTemplate("test", version); + String source = MarvelTemplateUtils.loadTemplate("test"); assertThat(source, notNullValue()); assertThat(source.length(), greaterThan(0)); assertTemplate(source, equalTo("{\n" + - " \"template\": \".monitoring-data-" + version + "\",\n" + + " \"template\": \".monitoring-data-" + MarvelTemplateUtils.TEMPLATE_VERSION + "\",\n" + " \"mappings\": {\n" + " \"type_1\": {\n" + " \"_meta\": {\n" + - " \"template.version\": \"" + version + "\"\n" + + " \"template.version\": \"" + MarvelTemplateUtils.TEMPLATE_VERSION + "\"\n" + " }\n" + " }\n" + " }\n" + "}")); } - - public void testLoad() throws IOException { - BytesReference source = MarvelTemplateUtils.load(TEST_TEMPLATE); - assertThat(source, notNullValue()); - assertThat(source.length(), greaterThan(0)); - } - - public void testValidateNullSource() { - expectedException.expect(ElasticsearchParseException.class); - expectedException.expectMessage("Monitoring template must not be null"); - MarvelTemplateUtils.validate(null); - } - - public void testValidateEmptySource() { - expectedException.expect(ElasticsearchParseException.class); - expectedException.expectMessage("Monitoring template must not be empty"); - MarvelTemplateUtils.validate(new BytesArray("")); - } - - public void testValidateInvalidSource() { - expectedException.expect(ElasticsearchParseException.class); - expectedException.expectMessage("Invalid monitoring template"); - MarvelTemplateUtils.validate(new BytesArray("{\"foo\": \"bar")); - } - - public void testValidate() { - try { - MarvelTemplateUtils.validate(MarvelTemplateUtils.load(TEST_TEMPLATE)); - } catch (Exception e) { - fail("failed to validate test template: " + e.getMessage()); - } - } - - public void testFilter() { - assertTemplate(MarvelTemplateUtils.filter(new BytesArray("${monitoring.template.version}"), 0), equalTo("0")); - assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), 1), - equalTo("{\"template\": \"test-1\"}")); - assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), 2), - equalTo("{\"template\": \"2-test\"}")); - assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), 3), - equalTo("{\"template\": \"test-3-test\"}")); - - final int version = randomIntBetween(0, 100); - assertTemplate(MarvelTemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " + - "\"bar-${monitoring.template.version}\"}"), version), - equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}")); - } - - public static void assertTemplate(String actual, Matcher matcher) { - if (Constants.WINDOWS) { - // translate Windows line endings (\r\n) to standard ones (\n) - actual = Strings.replace(actual, System.lineSeparator(), "\n"); - } - assertThat(actual, matcher); - } - } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java index bb0d6afea5e..2ce259b6665 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java @@ -74,8 +74,8 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase } @Override - protected void putTemplate(String name, int version) throws Exception { - dispatcher.templates.put(name, generateTemplateSource(name, version)); + protected void putTemplate(String name) throws Exception { + dispatcher.templates.put(name, generateTemplateSource(name)); } @Override diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java index 035254747c6..8ec319f2c8d 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java @@ -26,9 +26,9 @@ public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase } @Override - protected void putTemplate(String name, int version) throws Exception { + protected void putTemplate(String name) throws Exception { waitNoPendingTasksOnAll(); - assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name, version)).get()); + assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name)).get()); } @Override diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/DataResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/DataResolverTests.java index a526dfd20a4..471234ed3eb 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/DataResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/DataResolverTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import java.io.IOException; @@ -20,11 +21,9 @@ import static org.hamcrest.Matchers.equalTo; public class DataResolverTests extends MonitoringIndexNameResolverTestCase { - private int randomVersion = randomIntBetween(0, 100); - @Override protected MonitoringIndexNameResolver newResolver() { - return newDataResolver(randomVersion); + return newDataResolver(); } @Override @@ -52,12 +51,11 @@ public class DataResolverTests extends MonitoringIndexNameResolverTestCase { } public void testDataResolver() { - assertThat(newDataResolver(randomVersion).index(newMarvelDoc()), equalTo(".monitoring-data-" + randomVersion)); - assertThat(newDataResolver(42).index(newMarvelDoc()), equalTo(".monitoring-data-42")); + assertThat(newDataResolver().index(newMarvelDoc()), equalTo(".monitoring-data-" + MarvelTemplateUtils.TEMPLATE_VERSION)); } - private MonitoringIndexNameResolver.Data newDataResolver(int randomVersion) { - return new MonitoringIndexNameResolver.Data(randomVersion) { + private MonitoringIndexNameResolver.Data newDataResolver() { + return new MonitoringIndexNameResolver.Data() { @Override public String type(MonitoringDoc document) { return null; diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java index 4b540ced892..8443e960297 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.test.ESTestCase; @@ -128,18 +129,17 @@ public abstract class MonitoringIndexNameResolverTestCase newResolver() { - return newTimestampedResolver(randomId, randomVersion, Settings.EMPTY); + return newTimestampedResolver(randomId, Settings.EMPTY); } @Override @@ -72,19 +72,17 @@ public class TimestampedResolverTests extends MonitoringIndexNameResolverTestCas .build(); } - MonitoringIndexNameResolver.Timestamped resolver = newTimestampedResolver(randomId, randomVersion, settings); + MonitoringIndexNameResolver.Timestamped resolver = newTimestampedResolver(randomId, settings); assertThat(resolver, notNullValue()); assertThat(resolver.getId(), equalTo(randomId.getSystem())); - assertThat(resolver.getVersion(), equalTo(randomVersion)); assertThat(resolver.index(doc), - equalTo(PREFIX + DELIMITER + resolver.getId() + DELIMITER + String.valueOf(resolver.getVersion()) + equalTo(PREFIX + DELIMITER + resolver.getId() + DELIMITER + String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION) + DELIMITER + DateTimeFormat.forPattern(format).withZoneUTC().print(1437580442979L))); } } - private MonitoringIndexNameResolver.Timestamped newTimestampedResolver(MonitoredSystem id, int version, - Settings settings) { - return new MonitoringIndexNameResolver.Timestamped(id, version, settings) { + private MonitoringIndexNameResolver.Timestamped newTimestampedResolver(MonitoredSystem id, Settings settings) { + return new MonitoringIndexNameResolver.Timestamped(id, settings) { @Override public String type(MonitoringDoc document) { return null; diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java index e1d1ffd7415..9b172ec25e7 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java @@ -107,7 +107,7 @@ public class ClusterStateTests extends MarvelIntegTestCase { public void testClusterStateNodes() throws Exception { final long nbNodes = internalCluster().size(); - MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES, TEMPLATE_VERSION); + MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES); assertNotNull(timestampedResolver); String timestampedIndex = timestampedResolver.indexPattern(); @@ -152,7 +152,7 @@ public class ClusterStateTests extends MarvelIntegTestCase { public void testDiscoveryNodes() throws Exception { final long nbNodes = internalCluster().size(); - MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(TEMPLATE_VERSION); + MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(); assertNotNull(dataResolver); String dataIndex = dataResolver.indexPattern(); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java index 39060b09797..90846a2fc53 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java @@ -19,6 +19,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.VersionUtils; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; import java.util.Locale; @@ -46,7 +48,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase public void testDeleteIndex() throws Exception { internalCluster().startNode(); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); + createTimestampedIndex(now().minusDays(10)); assertIndicesCount(1); CleanerService.Listener listener = getListener(); @@ -57,7 +59,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase public void testIgnoreCurrentDataIndex() throws Exception { internalCluster().startNode(); - createDataIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); + createDataIndex(now().minusDays(10)); assertIndicesCount(1); CleanerService.Listener listener = getListener(); @@ -69,8 +71,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase internalCluster().startNode(); createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1)); - createDataIndex(0, now().minusDays(10)); - createDataIndex(Integer.MAX_VALUE, now().minusDays(20)); + createDataIndex(now().minusDays(10)); + String olderIndex = String.join(MonitoringIndexNameResolver.DELIMITER, MonitoringIndexNameResolver.PREFIX, + MonitoringIndexNameResolver.Data.DATA, "1"); + createIndex(olderIndex, now().minusDays(20)); + assertIndicesCount(3); CleanerService.Listener listener = getListener(); @@ -81,8 +86,8 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase public void testIgnoreCurrentTimestampedIndex() throws Exception { internalCluster().startNode(); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now()); + createTimestampedIndex(now().minusDays(10)); + createTimestampedIndex(now()); assertIndicesCount(2); CleanerService.Listener listener = getListener(); @@ -93,8 +98,13 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception { internalCluster().startNode(); - createTimestampedIndex(0, now().minusDays(10)); - createTimestampedIndex(Integer.MAX_VALUE, now().minusDays(10)); + createTimestampedIndex(now().minusDays(10)); + // create an older index based on an older template + DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC(); + String index = String.join(MonitoringIndexNameResolver.DELIMITER, MonitoringIndexNameResolver.PREFIX, + MonitoredSystem.ES.getSystem(), "1"); + String timestampedIndex = String.join("-", index, formatter.print(now().minusDays(10))); + createIndex(timestampedIndex, now().minusDays(10)); assertIndicesCount(2); CleanerService.Listener listener = getListener(); @@ -108,11 +118,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase CleanerService.Listener listener = getListener(); final DateTime now = now(); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusYears(1)); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(6)); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(1)); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(10)); - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(1)); + createTimestampedIndex(now.minusYears(1)); + createTimestampedIndex(now.minusMonths(6)); + createTimestampedIndex(now.minusMonths(1)); + createTimestampedIndex(now.minusDays(10)); + createTimestampedIndex(now.minusDays(1)); assertIndicesCount(5); // Clean indices that have expired two years ago @@ -148,7 +158,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase final DateTime now = now(); for (int i = 0; i < max; i++) { - createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(i)); + createTimestampedIndex(now.minusDays(i)); } assertIndicesCount(max); @@ -175,18 +185,18 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase /** * Creates a monitoring data index in a given version. */ - protected void createDataIndex(int version, DateTime creationDate) { - createIndex(new MockDataIndexNameResolver(version).index(randomMonitoringDoc()), creationDate); + protected void createDataIndex(DateTime creationDate) { + createIndex(new MockDataIndexNameResolver().index(randomMonitoringDoc()), creationDate); } /** * Creates a monitoring timestamped index in a given version. */ - protected void createTimestampedIndex(int version, DateTime creationDate) { + protected void createTimestampedIndex(DateTime creationDate) { MonitoringDoc monitoringDoc = randomMonitoringDoc(); monitoringDoc.setTimestamp(creationDate.getMillis()); - MonitoringIndexNameResolver.Timestamped resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version); + MonitoringIndexNameResolver.Timestamped resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES); createIndex(resolver.index(monitoringDoc), creationDate); } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index c007990500e..f0d86963560 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -25,8 +25,8 @@ import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import org.elasticsearch.marvel.agent.resolver.ResolversRegistry; import org.elasticsearch.marvel.client.MonitoringClient; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.shield.authc.file.FileRealm; import org.elasticsearch.shield.Security; +import org.elasticsearch.shield.authc.file.FileRealm; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.authz.store.FileRolesStore; @@ -438,11 +438,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), value, timeUnit))); } - protected class MockDataIndexNameResolver extends MonitoringIndexNameResolver.Data { - - public MockDataIndexNameResolver(int version) { - super(version); - } + public class MockDataIndexNameResolver extends MonitoringIndexNameResolver.Data { @Override public String type(MonitoringDoc document) { @@ -462,12 +458,12 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { protected class MockTimestampedIndexNameResolver extends MonitoringIndexNameResolver.Timestamped { - public MockTimestampedIndexNameResolver(MonitoredSystem id, int version, Settings settings) { - super(id, version, settings); + public MockTimestampedIndexNameResolver(MonitoredSystem id, Settings settings) { + super(id, settings); } - public MockTimestampedIndexNameResolver(MonitoredSystem id, int version) { - this(id, version, Settings.EMPTY); + public MockTimestampedIndexNameResolver(MonitoredSystem id) { + this(id, Settings.EMPTY); } @Override diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/template/TemplateUtils.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/template/TemplateUtils.java new file mode 100644 index 00000000000..c4ca9a71899 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/template/TemplateUtils.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.template; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.NotXContentException; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.xcontent.XContentHelper; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.regex.Pattern; + +/** + * Handling versioned templates for time-based indices in x-pack + */ +public class TemplateUtils { + + private TemplateUtils() {} + + /** + * Loads a built-in template and returns its source. + */ + public static String loadTemplate(String resource, String version, String versionProperty) { + try { + BytesReference source = load(resource); + validate(source); + + return filter(source, version, versionProperty); + } catch (Exception e) { + throw new IllegalArgumentException("Unable to load template [" + resource + "]", e); + } + } + + /** + * Loads a resource from the classpath and returns it as a {@link BytesReference} + */ + public static BytesReference load(String name) throws IOException { + try (InputStream is = TemplateUtils.class.getResourceAsStream(name)) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + Streams.copy(is, out); + return new BytesArray(out.toByteArray()); + } + } + } + + /** + * Parses and validates that the source is not empty. + */ + public static void validate(BytesReference source) { + if (source == null) { + throw new ElasticsearchParseException("Template must not be null"); + } + + try { + XContentHelper.convertToMap(source, false).v2(); + } catch (NotXContentException e) { + throw new ElasticsearchParseException("Template must not be empty"); + } catch (Exception e) { + throw new ElasticsearchParseException("Invalid template", e); + } + } + + /** + * Filters the source: replaces any template version property with the version number + */ + public static String filter(BytesReference source, String version, String versionProperty) { + return Pattern.compile(versionProperty) + .matcher(source.toUtf8()) + .replaceAll(version); + } + +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/template/TemplateUtilsTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/template/TemplateUtilsTests.java new file mode 100644 index 00000000000..ab656373e25 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/template/TemplateUtilsTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.template; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.Locale; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.core.Is.is; + +public class TemplateUtilsTests extends ESTestCase { + + private static final String TEST_TEMPLATE = "/monitoring-%s.json"; + + public void testLoadTemplate() throws IOException { + final int version = randomIntBetween(0, 10_000); + String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); + String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), Pattern.quote("${monitoring.template.version}")); + + assertThat(source, notNullValue()); + assertThat(source.length(), greaterThan(0)); + assertTemplate(source, equalTo("{\n" + + " \"template\": \".monitoring-data-" + version + "\",\n" + + " \"mappings\": {\n" + + " \"type_1\": {\n" + + " \"_meta\": {\n" + + " \"template.version\": \"" + version + "\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}")); + } + + public void testLoad() throws IOException { + String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); + BytesReference source = TemplateUtils.load(resource); + assertThat(source, notNullValue()); + assertThat(source.length(), greaterThan(0)); + } + + public void testValidateNullSource() { + ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> TemplateUtils.validate(null)); + assertThat(exception.getMessage(), is("Template must not be null")); + } + + public void testValidateEmptySource() { + ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, + () -> TemplateUtils.validate(new BytesArray(""))); + assertThat(exception.getMessage(), is("Template must not be empty")); + } + + public void testValidateInvalidSource() { + ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, + () -> TemplateUtils.validate(new BytesArray("{\"foo\": \"bar"))); + assertThat(exception.getMessage(), is("Invalid template")); + } + + public void testValidate() throws IOException { + String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); + TemplateUtils.validate(TemplateUtils.load(resource)); + } + + public void testFilter() { + assertTemplate(TemplateUtils.filter(new BytesArray("${monitoring.template.version}"), "0", + Pattern.quote("${monitoring.template.version}")), equalTo("0")); + assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), "1", + Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-1\"}")); + assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), "2", + Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"2-test\"}")); + assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), "3", + Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-3-test\"}")); + + final int version = randomIntBetween(0, 100); + assertTemplate(TemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " + + "\"bar-${monitoring.template.version}\"}"), String.valueOf(version), + Pattern.quote("${monitoring.template.version}")), + equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}")); + } + + public static void assertTemplate(String actual, Matcher matcher) { + if (Constants.WINDOWS) { + // translate Windows line endings (\r\n) to standard ones (\n) + actual = Strings.replace(actual, System.lineSeparator(), "\n"); + } + assertThat(actual, matcher); + } + + +} diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java index 0e08369d642..cc539d63b38 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java @@ -47,6 +47,7 @@ import org.elasticsearch.watcher.rest.action.RestHijackOperationAction; import org.elasticsearch.watcher.rest.action.RestPutWatchAction; import org.elasticsearch.watcher.rest.action.RestWatchServiceAction; import org.elasticsearch.watcher.rest.action.RestWatcherStatsAction; +import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig; import org.elasticsearch.watcher.support.clock.ClockModule; import org.elasticsearch.watcher.support.http.HttpClient; @@ -168,7 +169,7 @@ public class Watcher { } public void onModule(SettingsModule module) { - for (TemplateConfig templateConfig : WatcherModule.TEMPLATE_CONFIGS) { + for (TemplateConfig templateConfig : WatcherIndexTemplateRegistry.TEMPLATE_CONFIGS) { module.registerSetting(templateConfig.getSetting()); } module.registerSetting(InternalSlackService.SLACK_ACCOUNT_SETTING); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java index 61aa80b6cac..2367d082912 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java @@ -6,44 +6,18 @@ package org.elasticsearch.watcher; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.inject.util.Providers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry; -import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig; import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation; import org.elasticsearch.xpack.XPackPlugin; -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - public class WatcherModule extends AbstractModule { - static final String PROPERTIES_FILE = "/watcher.properties"; - static final String VERSION_FIELD = "xpack.watcher.template.version"; - - public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + getHistoryIndexTemplateVersion(); - public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches"; - public static final String WATCHES_TEMPLATE_NAME = "watches"; - public static final Setting HISTORY_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.history.index.", - Setting.Property.Dynamic, Setting.Property.NodeScope); - public static final Setting TRIGGERED_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.triggered_watches.index.", - Setting.Property.Dynamic, Setting.Property.NodeScope); - public static final Setting WATCHES_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.watches.index.", - Setting.Property.Dynamic, Setting.Property.NodeScope); - - - public final static TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{ - new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING), - new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING), - new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING) - }; - private final boolean enabled; private final boolean transportClientMode; @@ -64,28 +38,6 @@ public class WatcherModule extends AbstractModule { bind(WatcherLicensee.class).asEagerSingleton(); bind(WatcherLifeCycleService.class).asEagerSingleton(); bind(WatcherSettingsValidation.class).asEagerSingleton(); - bind(WatcherIndexTemplateRegistry.class).asEagerSingleton(); - Multibinder multibinder - = Multibinder.newSetBinder(binder(), TemplateConfig.class); - for (TemplateConfig templateConfig : TEMPLATE_CONFIGS) { - multibinder.addBinding().toInstance(templateConfig); - } - } - - public static Integer getHistoryIndexTemplateVersion() { - try (InputStream is = WatcherModule.class.getResourceAsStream(PROPERTIES_FILE)) { - Properties properties = new Properties(); - properties.load(is); - String version = properties.getProperty(VERSION_FIELD); - if (Strings.hasLength(version)) { - return Integer.parseInt(version); - } - return null; - } catch (NumberFormatException e) { - throw new IllegalArgumentException("failed to parse watcher template version"); - } catch (IOException e) { - throw new IllegalArgumentException("failed to load watcher template version"); - } } } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java index d067c2c7c4c..12f4785812b 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.watcher.WatcherModule; import org.elasticsearch.watcher.execution.ExecutionState; +import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -31,7 +32,7 @@ import static org.elasticsearch.watcher.support.Exceptions.ioException; */ public class HistoryStore extends AbstractComponent { - public static final String INDEX_PREFIX = ".watcher-history-" + WatcherModule.getHistoryIndexTemplateVersion() + "-"; + public static final String INDEX_PREFIX = ".watcher-history-" + WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION + "-"; public static final String DOC_TYPE = "watch_record"; static final DateTimeFormatter indexTimeFormat = DateTimeFormat.forPattern("YYYY.MM.dd"); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java index 14949ba0cb9..6a88ca174dc 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java @@ -5,54 +5,69 @@ */ package org.elasticsearch.watcher.support; +import com.google.common.base.Charsets; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy; -import org.elasticsearch.watcher.watch.WatchStore; +import org.elasticsearch.xpack.template.TemplateUtils; -import java.io.InputStream; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.concurrent.Executor; +import java.util.regex.Pattern; import static java.util.Collections.unmodifiableMap; -import static java.util.Collections.unmodifiableSet; /** */ public class WatcherIndexTemplateRegistry extends AbstractComponent implements ClusterStateListener { + private static final String FORBIDDEN_INDEX_SETTING = "index.mapper.dynamic"; + public static final String INDEX_TEMPLATE_VERSION = "1"; + + public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + INDEX_TEMPLATE_VERSION; + public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches"; + public static final String WATCHES_TEMPLATE_NAME = "watches"; + + public static final Setting HISTORY_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.history.index.", + Setting.Property.Dynamic, Setting.Property.NodeScope); + public static final Setting TRIGGERED_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.triggered_watches.index.", + Setting.Property.Dynamic, Setting.Property.NodeScope); + public static final Setting WATCHES_TEMPLATE_SETTING = Setting.groupSetting("xpack.watcher.watches.index.", + Setting.Property.Dynamic, Setting.Property.NodeScope); + public static final TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{ + new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING), + new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING), + new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING) + }; private final WatcherClientProxy client; private final ThreadPool threadPool; private final ClusterService clusterService; - private final Set indexTemplates; + private final TemplateConfig[] indexTemplates; private volatile Map customIndexSettings; @Inject public WatcherIndexTemplateRegistry(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, - ThreadPool threadPool, WatcherClientProxy client, Set configs) { + ThreadPool threadPool, WatcherClientProxy client) { super(settings); this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; - this.indexTemplates = unmodifiableSet(new HashSet<>(configs)); + this.indexTemplates = TEMPLATE_CONFIGS; clusterService.add(this); Map customIndexSettings = new HashMap<>(); @@ -150,33 +165,23 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C } else { executor = threadPool.generic(); } - executor.execute(new Runnable() { - @Override - public void run() { - try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getFileName()+ ".json")) { - if (is == null) { - logger.error("Resource [/{}.json] not found in classpath", config.getFileName()); - return; - } - final byte[] template; - try (BytesStreamOutput out = new BytesStreamOutput()) { - Streams.copy(is, out); - template = out.bytes().toBytes(); - } + executor.execute(() -> { + final byte[] template = TemplateUtils.loadTemplate("/" + config.getFileName()+ ".json", INDEX_TEMPLATE_VERSION, + Pattern.quote("${xpack.watcher.template.version}")).getBytes(Charsets.UTF_8); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template); - Settings customSettings = customIndexSettings.get(config.getSetting().getKey()); - if (customSettings != null && customSettings.names().size() > 0) { - Settings updatedSettings = Settings.builder() - .put(request.settings()) - .put(customSettings) - .build(); - request.settings(updatedSettings); - } - PutIndexTemplateResponse response = client.putTemplate(request); - } catch (Exception e) { - logger.error("failed to load [{}.json]", e, config.getTemplateName()); - } + PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template); + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + Settings customSettings = customIndexSettings.get(config.getSetting().getKey()); + if (customSettings != null && customSettings.names().size() > 0) { + Settings updatedSettings = Settings.builder() + .put(request.settings()) + .put(customSettings) + .build(); + request.settings(updatedSettings); + } + PutIndexTemplateResponse response = client.putTemplate(request); + if (response.isAcknowledged() == false) { + logger.error("Error adding watcher template [{}], request was not acknowledged", config.getTemplateName()); } }); } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java index 2b2d03d67c4..6cb1031b619 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.watcher.WatcherModule; +import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; @@ -23,8 +23,10 @@ import static org.hamcrest.core.Is.is; @TestLogging("cluster:DEBUG,action.admin.cluster.settings:DEBUG,watcher:DEBUG") @ESIntegTestCase.ClusterScope(scope = TEST, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 1) public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCase { + public void testChangeSettings() throws Exception { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + GetIndexTemplatesResponse response = client().admin().indices() + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1")); // this isn't defined in the template, so we rely on ES's default, which is zero assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), nullValue()); @@ -44,7 +46,7 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas @Override public void run() { GetIndexTemplatesResponse response = client().admin().indices() - .prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2")); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_replicas"), equalTo("2")); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.refresh_interval"), equalTo("5m")); @@ -53,7 +55,8 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas } public void testChangeSettingsIgnoringForbiddenSetting() throws Exception { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + GetIndexTemplatesResponse response = client().admin().indices() + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("1")); assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false)); @@ -70,7 +73,7 @@ public class HistoryStoreSettingsTests extends AbstractWatcherIntegrationTestCas @Override public void run() { GetIndexTemplatesResponse response = client().admin().indices() - .prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().get(0).getSettings().get("index.number_of_shards"), equalTo("2")); assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false)); } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java index 4fe7bbcdfe6..2522e341f59 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.watcher.WatcherModule; import org.elasticsearch.watcher.execution.ExecutionState; import org.elasticsearch.watcher.execution.Wid; +import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy; import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent; import org.joda.time.DateTime; @@ -70,7 +71,7 @@ public class HistoryStoreTests extends ESTestCase { } public void testIndexNameGeneration() { - Integer indexTemplateVersion = WatcherModule.getHistoryIndexTemplateVersion(); + String indexTemplateVersion = WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION; assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(0, UTC)), equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01")); assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(100000000000L, UTC)), diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java index 4adfd5d4631..bdae08dc757 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -47,7 +47,7 @@ public class WatcherIndexTemplateRegistryTests extends AbstractWatcherIntegratio @Override public void run() { GetIndexTemplatesResponse response = client().admin().indices() - .prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().size(), equalTo(1)); // setting from the file on the classpath: assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false)); @@ -57,13 +57,13 @@ public class WatcherIndexTemplateRegistryTests extends AbstractWatcherIntegratio }); // Now delete the index template and verify the index template gets added back: - assertAcked(client().admin().indices().prepareDeleteTemplate(WatcherModule.HISTORY_TEMPLATE_NAME).get()); + assertAcked(client().admin().indices().prepareDeleteTemplate(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get()); assertBusy(new Runnable() { @Override public void run() { GetIndexTemplatesResponse response = client().admin().indices() - .prepareGetTemplates(WatcherModule.HISTORY_TEMPLATE_NAME).get(); + .prepareGetTemplates(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME).get(); assertThat(response.getIndexTemplates().size(), equalTo(1)); // setting from the file on the classpath: assertThat(response.getIndexTemplates().get(0).getSettings().getAsBoolean("index.mapper.dynamic", null), is(false)); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java index 9dd18ae6566..c57ac04e35d 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -27,8 +27,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockMustacheScriptEngine; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.shield.authc.file.FileRealm; import org.elasticsearch.shield.Security; +import org.elasticsearch.shield.authc.file.FileRealm; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.authz.store.FileRolesStore; @@ -40,7 +40,6 @@ import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.AssertingLocalTransport; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.watcher.WatcherLifeCycleService; -import org.elasticsearch.watcher.WatcherModule; import org.elasticsearch.watcher.WatcherService; import org.elasticsearch.watcher.WatcherState; import org.elasticsearch.watcher.actions.email.service.Authentication; @@ -91,9 +90,9 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -import static org.elasticsearch.watcher.WatcherModule.HISTORY_TEMPLATE_NAME; -import static org.elasticsearch.watcher.WatcherModule.TRIGGERED_TEMPLATE_NAME; -import static org.elasticsearch.watcher.WatcherModule.WATCHES_TEMPLATE_NAME; +import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME; +import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME; +import static org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -143,7 +142,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase @Override protected Set excludeTemplates() { Set excludes = new HashSet<>(); - for (WatcherIndexTemplateRegistry.TemplateConfig templateConfig : WatcherModule.TEMPLATE_CONFIGS) { + for (WatcherIndexTemplateRegistry.TemplateConfig templateConfig : WatcherIndexTemplateRegistry.TEMPLATE_CONFIGS) { excludes.add(templateConfig.getTemplateName()); } return excludes;