diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index bc70ca0b9c5..9f972ec5b80 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -120,7 +120,6 @@ public class MetaDataCreateIndexService { * These index patterns will be converted to hidden indices, at which point they should be removed from this list. */ private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton( - ".slm-history-*", ".watch-history-*", ".ml-anomalies-*", ".ml-notifications-*", diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 7b014a21ef8..8a37659147e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -641,7 +641,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { public void testIndexNameExclusionsList() { // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty List excludedNames = Arrays.asList( - ".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index aae90b7c37d..6d999a5b2cf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -599,8 +599,11 @@ public abstract class ESRestTestCase extends ESTestCase { } protected static void wipeAllIndices() throws IOException { + boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0); try { - final Response response = adminClient().performRequest(new Request("DELETE", "*")); + final Request deleteReq = new Request("DELETE", "*"); + deleteReq.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + final Response response = adminClient().performRequest(deleteReq); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); } @@ -1235,7 +1238,7 @@ public abstract class ESRestTestCase extends ESTestCase { final Request request = new Request("GET", "_nodes"); request.addParameter("filter_path", "nodes.*.version"); - final Response response = client().performRequest(request); + final Response response = adminClient().performRequest(request); final Map nodes = ObjectPath.createFromResponse(response).evaluate("nodes"); Version minVersion = null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index 659ef74b3fd..ef768cd992c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -35,7 +35,8 @@ import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HISTORY_IND public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial - public static final String INDEX_TEMPLATE_VERSION = "1"; + // version 2: converted to hidden index + public static final int INDEX_TEMPLATE_VERSION = 2; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java index 5eb219f11a7..4565f7a93b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java @@ -16,7 +16,7 @@ public class IndexTemplateConfig { private final String templateName; private final String fileName; - private final String version; + private final int version; private final String versionProperty; /** @@ -27,14 +27,17 @@ public class IndexTemplateConfig { * {@code {"myTemplateVersion": "${my.version.property}"}} * With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}. * - * @param templateName The name that will be used for the index template. Literal, include the version in this string if + * Note that this code does not automatically insert the {@code version} index template property - include that in the JSON file + * defining the template, preferably using the version variable provided to this constructor. + * + * @param templateName The name that will be used for the index template. Literal, include the version in this string if * it should be used. * @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and * extension if necessary. * @param version The version of the template. Substituted for {@code versionProperty} as described above. * @param versionProperty The property that will be replaced with the {@code version} string as described above. */ - public IndexTemplateConfig(String templateName, String fileName, String version, String versionProperty) { + public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) { this.templateName = templateName; this.fileName = fileName; this.version = version; @@ -49,14 +52,20 @@ public class IndexTemplateConfig { return templateName; } + public int getVersion() { + return version; + } + /** * Loads the template from disk as a UTF-8 byte array. * @return The template as a UTF-8 byte array. */ public byte[] loadBytes() { - String template = TemplateUtils.loadTemplate(fileName, version, - Pattern.quote("${" + versionProperty + "}")); + final String versionPattern = Pattern.quote("${" + versionProperty + "}"); + String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern); assert template != null && template.length() > 0; + assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find() + : "index template must have a version property set to the given version property"; return template.getBytes(StandardCharsets.UTF_8); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 870887e9b08..f5d95204a79 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -31,6 +32,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -130,16 +132,24 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener { private void addTemplatesIfMissing(ClusterState state) { final List indexTemplates = getTemplateConfigs(); - for (IndexTemplateConfig template : indexTemplates) { - final String templateName = template.getTemplateName(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); if (creationCheck.compareAndSet(false, true)) { - if (!state.metaData().getTemplates().containsKey(templateName)) { + IndexTemplateMetaData currentTemplate = state.metaData().getTemplates().get(templateName); + if (Objects.isNull(currentTemplate)) { logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putTemplate(template, creationCheck); + putTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion()); + putTemplate(newTemplate, creationCheck); } else { creationCheck.set(false); - logger.trace("not adding index template [{}] for [{}], because it already exists", templateName, getOrigin()); + logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.getVersion()); } } else { logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index 6b25f7855f1..3c7e9cdfb90 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -59,9 +59,10 @@ public class TemplateUtils { public static String loadTemplate(String resource, String version, String versionProperty) { try { BytesReference source = load(resource); - validate(source); + final String filteredJson = filter(source, version, versionProperty); + validate(new BytesArray(filteredJson)); - return filter(source, version, versionProperty); + return filteredJson; } catch (Exception e) { throw new IllegalArgumentException("Unable to load template [" + resource + "]", e); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 821ea9a4335..9eaf0237ffd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -16,7 +16,7 @@ public final class WatcherIndexTemplateRegistryField { // version 9: add a user field defining which user executed the watch // version 10: add support for foreach path in actions // Note: if you change this, also inform the kibana team around the watcher-ui - public static final String INDEX_TEMPLATE_VERSION = "10"; + public static final int INDEX_TEMPLATE_VERSION = 10; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index ae9c50552b3..bb79af74f20 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -9,6 +9,7 @@ "index.auto_expand_replicas": "0-1", "index.lifecycle.name": "ilm-history-ilm-policy", "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}", + "index.hidden": true, "index.format": 1 }, "mappings": { @@ -79,5 +80,6 @@ } } } - } + }, + "version": ${xpack.ilm_history.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/slm-history.json b/x-pack/plugin/core/src/main/resources/slm-history.json index 76631602a3f..7ddd995fb21 100644 --- a/x-pack/plugin/core/src/main/resources/slm-history.json +++ b/x-pack/plugin/core/src/main/resources/slm-history.json @@ -9,6 +9,7 @@ "index.auto_expand_replicas": "0-1", "index.lifecycle.name": "slm-history-ilm-policy", "index.lifecycle.rollover_alias": ".slm-history-${xpack.slm.template.version}", + "index.hidden": true, "index.format": 1 }, "mappings": { @@ -55,5 +56,6 @@ } } } - } -} \ No newline at end of file + }, + "version": ${xpack.slm.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index c57d3c23a31..28f3b0fea65 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -36,5 +36,6 @@ } } } - } + }, + "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 8e91e02549c..5b3186a9a6c 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -611,5 +611,6 @@ } } } - } + }, + "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index 8b6bc435d2e..d35ddd9afd2 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -562,5 +562,6 @@ } } } - } + }, + "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index d02c2ffa24c..5181dfdc864 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -58,5 +58,6 @@ } } } - } + }, + "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java index 5dc2973bc42..bccaaf54407 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java @@ -62,6 +62,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.mock.orig.Mockito.when; import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING; +import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.SLM_POLICY_NAME; import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.SLM_TEMPLATE_NAME; import static org.hamcrest.Matchers.equalTo; @@ -113,27 +114,10 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); AtomicInteger calledTimes = new AtomicInteger(0); - client.setVerifier((action, request, listener) -> { - if (action instanceof PutIndexTemplateAction) { - calledTimes.incrementAndGet(); - assertThat(action, instanceOf(PutIndexTemplateAction.class)); - assertThat(request, instanceOf(PutIndexTemplateRequest.class)); - final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request; - assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME)); - assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); - assertNotNull(listener); - return new TestPutIndexTemplateResponse(true); - } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return new PutLifecycleAction.Response(true); - } else { - fail("client called with unexpected request:" + request.toString()); - return null; - } - }); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); @@ -145,7 +129,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { // will not be issued anymore, leaving calledTimes to 0 assertBusy(() -> { // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(newEvent); assertThat(calledTimes.get(), greaterThan(1)); }); @@ -174,7 +158,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { } }); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); } @@ -203,7 +187,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { return null; }); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); } @@ -235,11 +219,61 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) { LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName()); policyMap.put(policy.getName(), different); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); } } + public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION - 1), + nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + } + + public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + } + + + public void testSameOrHigherVersionTemplateNotUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent sameVersionEvent = createClusterChangedEvent( + Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION), nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutIndexTemplateAction) { + fail("template should not have been re-installed"); + return null; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + }); + registry.clusterChanged(sameVersionEvent); + + ClusterChangedEvent higherVersionEvent = createClusterChangedEvent( + Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION + randomIntBetween(1, 1000)), nodes); + registry.clusterChanged(higherVersionEvent); + } + public void testThatMissingMasterNodeDoesNothing() { DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); @@ -249,20 +283,25 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { return null; }); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(SLM_TEMPLATE_NAME), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes); registry.clusterChanged(event); } public void testValidate() { - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), null))); - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME), - Collections.emptyMap(), null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, + Collections.singletonMap(SLM_TEMPLATE_NAME, null), + Collections.emptyMap(), + null))); Map policyMap = new HashMap<>(); policyMap.put(SLM_POLICY_NAME, new LifecyclePolicy(SLM_POLICY_NAME, new HashMap<>())); - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), policyMap, null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), policyMap, null))); - assertTrue(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME), policyMap, null))); + assertTrue(registry.validate(createClusterState(Settings.EMPTY, + Collections.singletonMap(SLM_TEMPLATE_NAME, null), + policyMap, + null))); } // ------------- @@ -299,14 +338,35 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { } } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, DiscoveryNodes nodes) { - return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes); + private ActionResponse verifyTemplateInstalled( + AtomicInteger calledTimes, ActionType action, ActionRequest request, ActionListener listener) { + if (action instanceof PutIndexTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutIndexTemplateAction.class)); + assertThat(request, instanceOf(PutIndexTemplateRequest.class)); + final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request; + assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME)); + assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); + assertThat(putRequest.version(), equalTo(INDEX_TEMPLATE_VERSION)); + assertNotNull(listener); + return new TestPutIndexTemplateResponse(true); + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, DiscoveryNodes nodes) { + return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { - ClusterState cs = createClusterState(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes); + ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes); ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs, ClusterState.builder(new ClusterName("test")).build()); ClusterChangedEvent event = spy(realEvent); @@ -315,13 +375,15 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase { return event; } - private ClusterState createClusterState(Settings nodeSettings, - List existingTemplateNames, - Map existingPolicies, - DiscoveryNodes nodes) { + private ClusterState createClusterState(Settings nodeSettings, Map existingTemplates, + Map existingPolicies, DiscoveryNodes nodes) { ImmutableOpenMap.Builder indexTemplates = ImmutableOpenMap.builder(); - for (String name : existingTemplateNames) { - indexTemplates.put(name, mock(IndexTemplateMetaData.class)); + for (Map.Entry template : existingTemplates.entrySet()) { + final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class); + when(mockTemplate.version()).thenReturn(template.getValue()); + when(mockTemplate.getVersion()).thenReturn(template.getValue()); + + indexTemplates.put(template.getKey(), mockTemplate); } Map existingILMMeta = existingPolicies.entrySet().stream() diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 21b2d16afdc..149d4297caf 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -27,7 +27,8 @@ import java.util.List; public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial - public static final String INDEX_TEMPLATE_VERSION = "1"; + // version 2: convert to hidden index + public static final int INDEX_TEMPLATE_VERSION = 2; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java index 8a6e53ec024..59aef01f180 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java @@ -105,7 +105,7 @@ public class HistoryStoreTests extends ESTestCase { } public void testIndexNameGeneration() { - String indexTemplateVersion = INDEX_TEMPLATE_VERSION; + String indexTemplateVersion = Integer.toString(INDEX_TEMPLATE_VERSION); assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli((long) 0).atZone(ZoneOffset.UTC)), equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01")); assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(100000000000L).atZone(ZoneOffset.UTC)), diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index 85dc50c670b..f0be71155dc 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -58,6 +58,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.mock.orig.Mockito.verify; import static org.elasticsearch.mock.orig.Mockito.when; +import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -111,14 +112,16 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(4)).putTemplate(captor.capture(), anyObject()); @@ -136,14 +139,16 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { registry = new WatcherIndexTemplateRegistry(Settings.builder() .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(), clusterService, threadPool, client, xContentRegistry); - ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(5)).putTemplate(captor.capture(), anyObject()); @@ -155,7 +160,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); verify(client, times(1)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -171,7 +176,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { assertThat(policies, hasSize(1)); LifecyclePolicy policy = policies.get(0); policyMap.put(policy.getName(), policy); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -183,7 +188,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { registry = new WatcherIndexTemplateRegistry(Settings.builder() .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(), clusterService, threadPool, client, xContentRegistry); - ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -203,20 +208,44 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) { LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName()); policyMap.put(policy.getName(), different); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } } public void testThatTemplatesExist() { - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history")), is(false)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history", ".triggered_watches", ".watches")), - is(false)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - ".triggered_watches", ".watches")), is(true)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - ".triggered_watches", ".watches", "whatever", "else")), is(true)); + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false)); + } + + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false)); + } + + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true)); + } + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put("whatever", null); + existingTemplates.put("else", null); + + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true)); + } } // if a node is newer than the master node, the template needs to be applied as well @@ -226,8 +255,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watch-history-6", 6); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); @@ -240,8 +272,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); + existingTemplates.put(".watch-history-6", null); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); verifyZeroInteractions(client); @@ -251,24 +286,30 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); + existingTemplates.put(".watch-history-6", null); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); verifyZeroInteractions(client); } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, DiscoveryNodes nodes) { + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplateNames, DiscoveryNodes nodes) { return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes); } private ClusterState createClusterState(Settings nodeSettings, - List existingTemplateNames, + Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { ImmutableOpenMap.Builder indexTemplates = ImmutableOpenMap.builder(); - for (String name : existingTemplateNames) { - indexTemplates.put(name, mock(IndexTemplateMetaData.class)); + for (Map.Entry template : existingTemplates.entrySet()) { + final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class); + when(mockTemplate.version()).thenReturn(template.getValue()); + when(mockTemplate.getVersion()).thenReturn(template.getValue()); + indexTemplates.put(template.getKey(), mockTemplate); } Map existingILMMeta = existingPolicies.entrySet().stream() @@ -286,17 +327,17 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { .build(); } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplateNames, Map existingPolicies, DiscoveryNodes nodes) { return createClusterChangedEvent(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes); } private ClusterChangedEvent createClusterChangedEvent(Settings nodeSettings, - List existingTemplateNames, + Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { - ClusterState cs = createClusterState(nodeSettings, existingTemplateNames, existingPolicies, nodes); + ClusterState cs = createClusterState(nodeSettings, existingTemplates, existingPolicies, nodes); ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs, ClusterState.builder(new ClusterName("test")).build()); ClusterChangedEvent event = spy(realEvent); @@ -305,10 +346,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { return event; } - private ClusterState createClusterState(String ... existingTemplates) { + private ClusterState createClusterState(Map existingTemplates) { MetaData.Builder metaDataBuilder = MetaData.builder(); - for (String templateName : existingTemplates) { - metaDataBuilder.put(IndexTemplateMetaData.builder(templateName) + for (Map.Entry template : existingTemplates.entrySet()) { + metaDataBuilder.put(IndexTemplateMetaData.builder(template.getKey()) + .version(template.getValue()) .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))); }