From a81460dbf5a78ef6f7307928b8ddd030bd2cff89 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Fri, 6 Mar 2020 09:47:03 -0700 Subject: [PATCH] Make watch history indices hidden (#52974) This commit updates the template used for watch history indices with the hidden index setting so that new indices will be created as hidden. Relates #50251 Backport of #52962 --- .../action/support/IndicesOptions.java | 3 + .../metadata/MetaDataCreateIndexService.java | 1 - .../MetaDataCreateIndexServiceTests.java | 1 - .../elasticsearch/test/ESIntegTestCase.java | 3 +- .../core/template/IndexTemplateRegistry.java | 2 + .../WatcherIndexTemplateRegistryField.java | 5 +- .../src/main/resources/watch-history-10.json | 567 ++++++++++++++++ .../resources/watch-history-no-ilm-10.json | 616 ++++++++++++++++++ .../main/resources/watch-history-no-ilm.json | 1 + .../src/main/resources/watch-history.json | 1 + .../local/LocalIndicesCleanerTests.java | 2 +- .../support/WatcherIndexTemplateRegistry.java | 49 +- .../WatcherIndexTemplateRegistryTests.java | 30 +- .../upgrades/WatcherRestartIT.java | 20 + 14 files changed, 1283 insertions(+), 18 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/watch-history-10.json create mode 100644 x-pack/plugin/core/src/main/resources/watch-history-no-ilm-10.json diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 7bb7c4fdb10..4a0f53ad119 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -126,6 +126,9 @@ public class IndicesOptions implements ToXContentFragment { new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED)); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES), EnumSet.of(WildcardStates.OPEN)); + public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = + new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES), + EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN)); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES, Option.IGNORE_THROTTLED), EnumSet.of(WildcardStates.OPEN)); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 17340dce80f..80fc99b94f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -120,7 +120,6 @@ public class MetaDataCreateIndexService { * These index patterns will be converted to hidden indices, at which point they should be removed from this list. */ private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton( - ".watch-history-*", ".data-frame-notifications-*", ".transform-notifications-*" )); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 5e11bdd817d..cbb89dee716 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -641,7 +641,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { public void testIndexNameExclusionsList() { // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty List excludedNames = Arrays.asList( - ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT) ); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 4523961fd37..fb1c2d48805 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1253,7 +1253,8 @@ public abstract class ESIntegTestCase extends ESTestCase { protected final RefreshResponse refresh(String... indices) { waitForRelocation(); // TODO RANDOMIZE with flush? - RefreshResponse actionGet = client().admin().indices().prepareRefresh(indices).execute().actionGet(); + RefreshResponse actionGet = client().admin().indices().prepareRefresh(indices) + .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED).execute().actionGet(); assertNoFailures(actionGet); return actionGet; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 79802d08ee4..433c5e3e542 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -52,6 +52,7 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener { protected final Client client; protected final ThreadPool threadPool; protected final NamedXContentRegistry xContentRegistry; + protected final ClusterService clusterService; protected final ConcurrentMap templateCreationsInProgress = new ConcurrentHashMap<>(); protected final ConcurrentMap policyCreationsInProgress = new ConcurrentHashMap<>(); @@ -61,6 +62,7 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener { this.client = client; this.threadPool = threadPool; this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; clusterService.addListener(this); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 9eaf0237ffd..4a2524beecf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -15,10 +15,13 @@ public final class WatcherIndexTemplateRegistryField { // version 8: fix slack attachment property not to be dynamic, causing field type issues // version 9: add a user field defining which user executed the watch // version 10: add support for foreach path in actions + // version 11: watch history indices are hidden // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 10; + public static final int INDEX_TEMPLATE_VERSION = 11; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; + public static final String HISTORY_TEMPLATE_NAME_10 = ".watch-history-10"; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; + public static final String HISTORY_TEMPLATE_NAME_NO_ILM_10 = ".watch-history-no-ilm-10"; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; public static final String WATCHES_TEMPLATE_NAME = ".watches"; public static final String[] TEMPLATE_NAMES = new String[] { diff --git a/x-pack/plugin/core/src/main/resources/watch-history-10.json b/x-pack/plugin/core/src/main/resources/watch-history-10.json new file mode 100644 index 00000000000..78c989c0569 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/watch-history-10.json @@ -0,0 +1,567 @@ +{ + "index_patterns": [ ".watcher-history-10*" ], + "order": 2147483647, + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "watch-history-ilm-policy", + "index.format": 6 + }, + "mappings": { + "_doc": { + "_meta": { + "watcher-history-version": "10" + }, + "dynamic_templates": [ + { + "disabled_payload_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.payload", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_search_request_body_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.search\\.request\\.(body|template)", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_exception_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*)|actions)\\.error", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_jira_custom_fields": { + "path_match": "result.actions.jira.fields.customfield_*", + "mapping": { + "type": "object", + "enabled": false + } + } + } + ], + "dynamic": false, + "properties": { + "watch_id": { + "type": "keyword" + }, + "node": { + "type": "keyword" + }, + "trigger_event": { + "type": "object", + "dynamic": true, + "properties": { + "type" : { + "type" : "keyword" + }, + "triggered_time": { + "type": "date" + }, + "manual": { + "type": "object", + "dynamic": true, + "properties": { + "schedule": { + "type": "object", + "dynamic": true, + "properties": { + "scheduled_time": { + "type": "date" + } + } + } + } + }, + "schedule": { + "type": "object", + "dynamic": true, + "properties": { + "scheduled_time": { + "type": "date" + } + } + } + } + }, + "vars" : { + "type" : "object", + "enabled" : false + }, + "input": { + "type": "object", + "enabled": false + }, + "condition": { + "type": "object", + "enabled": false + }, + "state": { + "type": "keyword" + }, + "status": { + "type": "object", + "enabled" : false, + "dynamic" : true + }, + "messages": { + "type": "text" + }, + "user": { + "type": "text" + }, + "exception" : { + "type" : "object", + "enabled" : false + }, + "result": { + "type": "object", + "dynamic": true, + "properties": { + "execution_time": { + "type": "date" + }, + "execution_duration": { + "type": "long" + }, + "input": { + "type": "object", + "dynamic": true, + "properties": { + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "payload" : { + "type" : "object", + "enabled" : false + }, + "search": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "search_type": { + "type": "keyword" + }, + "indices": { + "type": "keyword" + }, + "types": { + "type": "keyword" + } + } + } + } + }, + "http": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "path": { + "type": "keyword" + }, + "host": { + "type": "keyword" + } + } + } + } + } + } + }, + "condition" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "met" : { + "type" : "boolean" + }, + "compare" : { + "type" : "object", + "enabled" : false + }, + "array_compare" : { + "type" : "object", + "enabled" : false + }, + "script" : { + "type" : "object", + "enabled" : false + } + } + }, + "transform" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "search" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "request" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "indices" : { + "type" : "keyword" + }, + "types" : { + "type" : "keyword" + } + } + } + } + } + } + }, + "actions": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "id" : { + "type" : "keyword" + }, + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "reason" : { + "type" : "keyword" + }, + "number_of_actions_executed": { + "type": "integer" + }, + "foreach" : { + "type": "object", + "enabled" : false + }, + "email": { + "type": "object", + "dynamic": true, + "properties": { + "message": { + "type": "object", + "dynamic": true, + "properties": { + "id": { + "type": "keyword" + }, + "from": { + "type": "keyword" + }, + "reply_to": { + "type": "keyword" + }, + "to": { + "type": "keyword" + }, + "cc": { + "type": "keyword" + }, + "bcc": { + "type": "keyword" + } + } + } + } + }, + "webhook": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "path": { + "type": "keyword" + }, + "host": { + "type": "keyword" + } + } + } + } + }, + "index": { + "type": "object", + "dynamic": true, + "properties": { + "response": { + "type": "object", + "dynamic": true, + "properties": { + "index": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + } + } + }, + "jira" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "fields": { + "type": "object", + "dynamic": true, + "properties": { + "summary": { + "type": "text" + }, + "description": { + "type": "text" + }, + "labels" : { + "type": "text" + }, + "project" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "key" : { + "type" : "keyword" + }, + "id" : { + "type" : "keyword" + } + } + }, + "issuetype" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "name" : { + "type": "keyword" + }, + "id" : { + "type" : "keyword" + } + } + } + } + }, + "result": { + "type": "object", + "dynamic": true, + "properties" : { + "id" : { + "type" : "keyword" + }, + "key" : { + "type" : "keyword" + }, + "self" : { + "type" : "keyword" + } + } + } + } + }, + "slack" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "sent_messages": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "status": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "to" : { + "type": "keyword" + }, + "message" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "from" : { + "type" : "text" + }, + "icon" : { + "type" : "keyword" + }, + "text" : { + "type" : "text" + }, + "attachments" : { + "type" : "nested", + "include_in_parent": true, + "dynamic" : true, + "properties" : { + "color" : { + "type" : "keyword" + }, + "fields" : { + "properties" : { + "value" : { + "type" : "text" + } + } + } + } + } + } + } + } + } + } + }, + "pagerduty" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "sent_event": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "event" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "client" : { + "type" : "text" + }, + "client_url" : { + "type" : "keyword" + }, + "account" : { + "type" : "keyword" + }, + "attach_payload" : { + "type" : "boolean" + }, + "incident_key" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "context" : { + "type" : "nested", + "include_in_parent": true, + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "href" : { + "type" : "keyword" + }, + "src" : { + "type" : "keyword" + }, + "alt" : { + "type" : "text" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "metadata": { + "type": "object", + "dynamic": true + } + } + } + }, + "version": 10 +} diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm-10.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm-10.json new file mode 100644 index 00000000000..c763f992468 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm-10.json @@ -0,0 +1,616 @@ +{ + "index_patterns": [ ".watcher-history-10*" ], + "order": 2147483646, + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.format": 6 + }, + "mappings": { + "doc": { + "_meta": { + "watcher-history-version": "10" + }, + "dynamic_templates": [ + { + "disabled_payload_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.payload", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_search_request_body_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.search\\.request\\.(body|template)", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_exception_fields": { + "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*)|actions)\\.error", + "match_pattern": "regex", + "mapping": { + "type": "object", + "enabled": false + } + } + }, + { + "disabled_jira_custom_fields": { + "path_match": "result.actions.jira.fields.customfield_*", + "mapping": { + "type": "object", + "enabled": false + } + } + } + ], + "dynamic": false, + "properties": { + "watch_id": { + "type": "keyword" + }, + "node": { + "type": "keyword" + }, + "trigger_event": { + "type": "object", + "dynamic": true, + "properties": { + "type" : { + "type" : "keyword" + }, + "triggered_time": { + "type": "date" + }, + "manual": { + "type": "object", + "dynamic": true, + "properties": { + "schedule": { + "type": "object", + "dynamic": true, + "properties": { + "scheduled_time": { + "type": "date" + } + } + } + } + }, + "schedule": { + "type": "object", + "dynamic": true, + "properties": { + "scheduled_time": { + "type": "date" + } + } + } + } + }, + "vars" : { + "type" : "object", + "enabled" : false + }, + "input": { + "type": "object", + "enabled": false + }, + "condition": { + "type": "object", + "enabled": false + }, + "state": { + "type": "keyword" + }, + "status": { + "type": "object", + "enabled" : false, + "dynamic" : true + }, + "messages": { + "type": "text" + }, + "user": { + "type": "text" + }, + "exception" : { + "type" : "object", + "enabled" : false + }, + "result": { + "type": "object", + "dynamic": true, + "properties": { + "execution_time": { + "type": "date" + }, + "execution_duration": { + "type": "long" + }, + "input": { + "type": "object", + "dynamic": true, + "properties": { + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "payload" : { + "type" : "object", + "enabled" : false + }, + "search": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "search_type": { + "type": "keyword" + }, + "indices": { + "type": "keyword" + }, + "types": { + "type": "keyword" + } + } + } + } + }, + "http": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "path": { + "type": "keyword" + }, + "host": { + "type": "keyword" + } + } + } + } + } + } + }, + "condition" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "met" : { + "type" : "boolean" + }, + "compare" : { + "type" : "object", + "enabled" : false + }, + "array_compare" : { + "type" : "object", + "enabled" : false + }, + "script" : { + "type" : "object", + "enabled" : false + } + } + }, + "transform" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "search" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "request" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "indices" : { + "type" : "keyword" + }, + "types" : { + "type" : "keyword" + } + } + } + } + } + } + }, + "actions": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "id" : { + "type" : "keyword" + }, + "type" : { + "type" : "keyword" + }, + "status" : { + "type" : "keyword" + }, + "reason" : { + "type" : "keyword" + }, + "email": { + "type": "object", + "dynamic": true, + "properties": { + "message": { + "type": "object", + "dynamic": true, + "properties": { + "id": { + "type": "keyword" + }, + "from": { + "type": "keyword" + }, + "reply_to": { + "type": "keyword" + }, + "to": { + "type": "keyword" + }, + "cc": { + "type": "keyword" + }, + "bcc": { + "type": "keyword" + } + } + } + } + }, + "webhook": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "path": { + "type": "keyword" + }, + "host": { + "type": "keyword" + } + } + } + } + }, + "index": { + "type": "object", + "dynamic": true, + "properties": { + "response": { + "type": "object", + "dynamic": true, + "properties": { + "index": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + } + } + }, + "hipchat" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "sent_messages": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "status": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "room" : { + "type": "keyword" + }, + "user" : { + "type": "keyword" + }, + "message" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "message_format" : { + "type" : "keyword" + }, + "color" : { + "type" : "keyword" + }, + "notify" : { + "type" : "boolean" + }, + "message" : { + "type" : "text" + }, + "from" : { + "type" : "text" + } + } + } + } + } + } + }, + "jira" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "fields": { + "type": "object", + "dynamic": true, + "properties": { + "summary": { + "type": "text" + }, + "description": { + "type": "text" + }, + "labels" : { + "type": "text" + }, + "project" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "key" : { + "type" : "keyword" + }, + "id" : { + "type" : "keyword" + } + } + }, + "issuetype" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "name" : { + "type": "keyword" + }, + "id" : { + "type" : "keyword" + } + } + } + } + }, + "result": { + "type": "object", + "dynamic": true, + "properties" : { + "id" : { + "type" : "keyword" + }, + "key" : { + "type" : "keyword" + }, + "self" : { + "type" : "keyword" + } + } + } + } + }, + "slack" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "sent_messages": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "status": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "to" : { + "type": "keyword" + }, + "message" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "from" : { + "type" : "text" + }, + "icon" : { + "type" : "keyword" + }, + "text" : { + "type" : "text" + }, + "attachments" : { + "type" : "nested", + "include_in_parent": true, + "dynamic" : true, + "properties" : { + "color" : { + "type" : "keyword" + }, + "fields" : { + "properties" : { + "value" : { + "type" : "text" + } + } + } + } + } + } + } + } + } + } + }, + "pagerduty" : { + "type": "object", + "dynamic": true, + "properties": { + "account": { + "type": "keyword" + }, + "sent_event": { + "type": "nested", + "include_in_parent": true, + "dynamic": true, + "properties": { + "reason": { + "type": "text" + }, + "request" : { + "type" : "object", + "enabled" : false + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "event" : { + "type" : "object", + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "client" : { + "type" : "text" + }, + "client_url" : { + "type" : "keyword" + }, + "account" : { + "type" : "keyword" + }, + "attach_payload" : { + "type" : "boolean" + }, + "incident_key" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "context" : { + "type" : "nested", + "include_in_parent": true, + "dynamic" : true, + "properties" : { + "type" : { + "type" : "keyword" + }, + "href" : { + "type" : "keyword" + }, + "src" : { + "type" : "keyword" + }, + "alt" : { + "type" : "text" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "metadata": { + "type": "object", + "dynamic": true + } + } + } + }, + "version": 10 +} diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 5b3186a9a6c..7b591050839 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -5,6 +5,7 @@ "index.number_of_shards": 1, "index.number_of_replicas": 0, "index.auto_expand_replicas": "0-1", + "index.hidden": true, "index.format": 6 }, "mappings": { diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index d35ddd9afd2..109ca95d751 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -6,6 +6,7 @@ "index.number_of_replicas": 0, "index.auto_expand_replicas": "0-1", "index.lifecycle.name": "watch-history-ilm-policy", + "index.hidden": true, "index.format": 6 }, "mappings": { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/local/LocalIndicesCleanerTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/local/LocalIndicesCleanerTests.java index 21624ee9548..1b4d2d2d318 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/local/LocalIndicesCleanerTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/local/LocalIndicesCleanerTests.java @@ -57,7 +57,7 @@ public class LocalIndicesCleanerTests extends AbstractIndicesCleanerTestCase { //so when es core gets the request with the explicit index name, it throws an index not found exception as that index //doesn't exist anymore. If we ignore unavailable instead no error will be thrown. GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings() - .setIndicesOptions(IndicesOptions.fromOptions(true, true, true, true)).get(); + .setIndicesOptions(IndicesOptions.fromOptions(true, true, true, true, true)).get(); assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(count)); }); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index be844d5d1e4..f7b39690837 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.watcher.support; +import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; @@ -36,11 +37,21 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry { "/watch-history.json", WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION, WATCHER_TEMPLATE_VERSION_VARIABLE); + public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCH_HISTORY_10 = new IndexTemplateConfig( + WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10, + "/watch-history-10.json", + 10, + WATCHER_TEMPLATE_VERSION_VARIABLE); public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM = new IndexTemplateConfig( WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM, "/watch-history-no-ilm.json", WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION, WATCHER_TEMPLATE_VERSION_VARIABLE); + public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_10 = new IndexTemplateConfig( + WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10, + "/watch-history-no-ilm-10.json", + 10, + WATCHER_TEMPLATE_VERSION_VARIABLE); public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCHES = new IndexTemplateConfig( WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, "/watches.json", @@ -50,22 +61,29 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry { public static final LifecyclePolicyConfig POLICY_WATCH_HISTORY = new LifecyclePolicyConfig("watch-history-ilm-policy", "/watch-history-ilm-policy.json"); - private final List templatesToUse; + private final boolean ilmEnabled; public WatcherIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, NamedXContentRegistry xContentRegistry) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - boolean ilmEnabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings); - templatesToUse = Arrays.asList( - ilmEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM, - TEMPLATE_CONFIG_TRIGGERED_WATCHES, - TEMPLATE_CONFIG_WATCHES - ); + this.ilmEnabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings); } @Override protected List getTemplateConfigs() { - return templatesToUse; + if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { + return Arrays.asList( + ilmEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM, + TEMPLATE_CONFIG_TRIGGERED_WATCHES, + TEMPLATE_CONFIG_WATCHES + ); + } else { + return Arrays.asList( + ilmEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY_10 : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_10, + TEMPLATE_CONFIG_TRIGGERED_WATCHES, + TEMPLATE_CONFIG_WATCHES + ); + } } @Override @@ -79,10 +97,17 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry { } public static boolean validate(ClusterState state) { - return (state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) || - state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) && - state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && - state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); + if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { + return (state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) || + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) && + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); + } else { + return (state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10) || + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10)) && + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && + state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); + } } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index f0be71155dc..ac474038bc1 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -264,6 +264,34 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(1)).putTemplate(argumentCaptor.capture(), anyObject()); + assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)); + } + + public void testThatTemplatesWithHiddenAreAppliedOnNewerNodes() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(node).add(masterNode).build(); + + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watch-history-6", 6); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); + registry.clusterChanged(event); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); + verify(client.admin().indices(), times(1)).putTemplate(argumentCaptor.capture(), anyObject()); + assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)); + + existingTemplates.remove(".watch-history-6"); + existingTemplates.put(".watch-history-10", 10); + masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(masterNode).add(node).build(); + event = createClusterChangedEvent(existingTemplates, nodes); + registry.clusterChanged(event); + + argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); + verify(client.admin().indices(), times(2)).putTemplate(argumentCaptor.capture(), anyObject()); assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)); } @@ -342,7 +370,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { ClusterState.builder(new ClusterName("test")).build()); ClusterChangedEvent event = spy(realEvent); when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); - + when(clusterService.state()).thenReturn(cs); return event; } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java index 755991a6e01..58e0a7c0ccf 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java @@ -7,11 +7,14 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningsHandler; import java.nio.charset.StandardCharsets; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public class WatcherRestartIT extends AbstractUpgradeTestCase { @@ -22,6 +25,23 @@ public class WatcherRestartIT extends AbstractUpgradeTestCase { client().performRequest(new Request("POST", "/_watcher/_start")); ensureWatcherStarted(); + + validateHistoryTemplate(); + } + + private void validateHistoryTemplate() throws Exception { + if (ClusterType.MIXED == CLUSTER_TYPE) { + final Request request = new Request("HEAD", "/_template/.watch-history-10"); + request.addParameter("include_type_name", "false"); + RequestOptions.Builder builder = request.getOptions().toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); + request.setOptions(builder); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } else if (ClusterType.UPGRADED == CLUSTER_TYPE) { + Response response = client().performRequest(new Request("HEAD", "/_template/.watch-history-11")); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } } private void ensureWatcherStopped() throws Exception {