Watcher: Load versioned index template for watch history

This loads an index template for the watch history to make sure,
that field changes are taken into account.

Also, the dynamic mapping for the watch history template has been
changed from strict to false.

This means that new fields can be included in a document, but they
will not indexed and are not searchable.

In addition the index names have been changed from .watch_history-$date to
.watcher-history-$template-$date - using dashes to be more consistent.

Closes elastic/elasticsearch#1299

Original commit: elastic/x-pack-elasticsearch@794f982234
This commit is contained in:
Alexander Reelsen 2016-02-09 09:39:07 +01:00
parent 25c3a66502
commit e8ad8cbb36
20 changed files with 118 additions and 62 deletions

View File

@ -98,7 +98,7 @@ public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegr
assertThat(mappingsResponse, notNullValue());
assertThat(mappingsResponse.getMappings().isEmpty(), is(false));
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> metadatas : mappingsResponse.getMappings()) {
if (!metadatas.key.startsWith(".watch_history")) {
if (!metadatas.key.startsWith(".watcher-history")) {
continue;
}
MappingMetaData metadata = metadatas.value.get("watch_record");

View File

@ -6,15 +6,15 @@ admin:
watcher_manager:
cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health
indices:
'.watch_history-*': all
'.watcher-history-*': all
watcher_monitor:
cluster: monitor_watcher
indices:
'.watch_history-*': read
'.watcher-history-*': read
crapy_role:
cluster:
- cluster:monitor/nodes/info
- cluster:monitor/health
- cluster:monitor/nodes/liveness
- cluster:monitor/nodes/liveness

View File

@ -58,5 +58,5 @@
- do:
search:
index: .watch_history-*
index: .watcher-history-*
- match: { hits.hits.0._source.result.actions.0.logging.logged_text: "value1 value2" }

View File

@ -6,13 +6,13 @@ admin:
watcher_manager:
cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health
indices:
'.watch_history-*': all
'.watcher-history-*': all
run_as: powerless_user, watcher_manager
watcher_monitor:
cluster: monitor_watcher
indices:
'.watch_history-*': read
'.watcher-history-*': read
crappy_role:
cluster:

View File

@ -74,6 +74,9 @@ ext.expansions = [
// Used in marvel index templates
'marvel.plugin.version': version,
'marvel.template.version': '1',
// Used in watcher index template
'watcher.plugin.version': version,
'watcher.template.version': '1',
]
processResources {

View File

@ -6,6 +6,7 @@
package org.elasticsearch.watcher;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.multibindings.Multibinder;
import org.elasticsearch.common.settings.Setting;
@ -14,10 +15,17 @@ import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
public class WatcherModule extends AbstractModule {
public static final String HISTORY_TEMPLATE_NAME = "watch_history";
static final String PROPERTIES_FILE = "/watcher.properties";
static final String VERSION_FIELD = "watcher.template.version";
public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + getHistoryIndexTemplateVersion();
public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches";
public static final String WATCHES_TEMPLATE_NAME = "watches";
public static final Setting<Settings> HISTORY_TEMPLATE_SETTING = Setting.groupSetting("watcher.history.index.", true,
@ -30,7 +38,7 @@ public class WatcherModule extends AbstractModule {
public final static TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{
new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING),
new TemplateConfig(HISTORY_TEMPLATE_NAME, HISTORY_TEMPLATE_SETTING),
new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING),
new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING)
};
@ -53,4 +61,19 @@ public class WatcherModule extends AbstractModule {
}
}
public static final Integer getHistoryIndexTemplateVersion() {
try (InputStream is = WatcherModule.class.getResourceAsStream(PROPERTIES_FILE)) {
Properties properties = new Properties();
properties.load(is);
String version = properties.getProperty(VERSION_FIELD);
if (Strings.hasLength(version)) {
return Integer.parseInt(version);
}
return null;
} catch (NumberFormatException e) {
throw new IllegalArgumentException("failed to parse watcher template version");
} catch (IOException e) {
throw new IllegalArgumentException("failed to load watcher template version");
}
}
}

View File

@ -91,13 +91,19 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;
import static org.elasticsearch.common.settings.Setting.Scope.CLUSTER;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
public class WatcherPlugin extends Plugin {
public static final String NAME = "watcher";
public static final String ENABLED_SETTING = NAME + ".enabled";
public static final Setting<String> INDEX_WATCHER_VERSION_SETTING =
new Setting<>("index.watcher.plugin.version", "", Function.identity(), false, Setting.Scope.INDEX);
public static final Setting<String> INDEX_WATCHER_TEMPLATE_VERSION_SETTING =
new Setting<>("index.watcher.template.version", "", Function.identity(), false, Setting.Scope.INDEX);
private final static ESLogger logger = Loggers.getLogger(XPackPlugin.class);
@ -193,37 +199,40 @@ public class WatcherPlugin extends Plugin {
module.registerSetting(InternalEmailService.EMAIL_ACCOUNT_SETTING);
module.registerSetting(InternalHipChatService.HIPCHAT_ACCOUNT_SETTING);
module.registerSetting(InternalPagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
module.registerSetting(Setting.intSetting("watcher.execution.scroll.size", 0, false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.intSetting("watcher.watch.scroll.size", 0, false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.boolSetting("watcher.enabled", false, false, Setting.Scope.CLUSTER));
module.registerSetting(INDEX_WATCHER_VERSION_SETTING);
module.registerSetting(INDEX_WATCHER_TEMPLATE_VERSION_SETTING);
module.registerSetting(Setting.intSetting("watcher.execution.scroll.size", 0, false, CLUSTER));
module.registerSetting(Setting.intSetting("watcher.watch.scroll.size", 0, false, CLUSTER));
module.registerSetting(Setting.boolSetting("watcher.enabled", false, false, CLUSTER));
module.registerSetting(ShieldSecretService.ENCRYPT_SENSITIVE_DATA_SETTING);
// TODO add real settings for these
module.registerSetting(Setting.simpleString("resource.reload.interval", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.enabled", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.interval.low", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.interval.medium", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.search.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.bulk.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.index.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.execution.default_throttle_period", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.http.default_read_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.groupSetting("watcher.http.ssl.", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.groupSetting("watcher.http.proxy.", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.actions.index.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.index.rest.direct_access", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.trigger.schedule.engine", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.input.search.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.transform.search.default_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.trigger.schedule.ticker.tick_interval", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.execution.scroll.timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.start_immediately", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("watcher.http.default_connection_timeout", false, Setting.Scope.CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.interval", false, CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.enabled", false, CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.interval.low", false, CLUSTER));
module.registerSetting(Setting.simpleString("resource.reload.interval.medium", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.search.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.bulk.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.internal.ops.index.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.execution.default_throttle_period", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.http.default_read_timeout", false, CLUSTER));
module.registerSetting(Setting.groupSetting("watcher.http.ssl.", false, CLUSTER));
module.registerSetting(Setting.groupSetting("watcher.http.proxy.", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.actions.index.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.index.rest.direct_access", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.trigger.schedule.engine", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.input.search.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.transform.search.default_timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.trigger.schedule.ticker.tick_interval", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.execution.scroll.timeout", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.start_immediately", false, CLUSTER));
module.registerSetting(Setting.simpleString("watcher.http.default_connection_timeout", false, CLUSTER));
module.registerSettingsFilter("watcher.actions.email.service.account.*.smtp.password");
module.registerSettingsFilter("watcher.actions.slack.service.account.*.url");
module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*.url");
module.registerSettingsFilter("watcher.actions.pagerduty.service." + PagerDutyAccount.SERVICE_KEY_SETTING);
module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*." +
PagerDutyAccount.SERVICE_KEY_SETTING);
module.registerSettingsFilter("watcher.actions.hipchat.service.account.*.auth_token");
}
@ -265,8 +274,9 @@ public class WatcherPlugin extends Plugin {
return;
}
String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too restrictive. disable " +
"[action.auto_create_index] or set it to [.watches,.triggered_watches,.watch_history*]", (Object) settings);
String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" +
" restrictive. disable [action.auto_create_index] or set it to " +
"[.watches,.triggered_watches,.watcher-history*]", (Object) settings);
if (Booleans.isExplicitFalse(value)) {
throw new IllegalArgumentException(errorMessage);
}
@ -312,9 +322,10 @@ public class WatcherPlugin extends Plugin {
throw new IllegalArgumentException(errorMessage);
}
}
logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. for the next 6 months daily history " +
"indices are allowed to be created, but please make sure that any future history indices after 6 months with the pattern " +
"[.watch_history-YYYY.MM.dd] are allowed to be created", value);
logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. " +
" for the next 6 months daily history indices are allowed to be created, but please make sure" +
" that any future history indices after 6 months with the pattern " +
"[.watcher-history-YYYY.MM.dd] are allowed to be created", value);
}
}

View File

@ -70,7 +70,7 @@ public abstract class WatchExecutionContext {
public abstract boolean skipThrottling(String actionId);
/**
* @return true if this execution should be recorded in the .watch_history index
* @return true if this execution should be recorded in the .watcher-history index
*/
public abstract boolean recordExecution();

View File

@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.watcher.WatcherModule;
import org.elasticsearch.watcher.execution.ExecutionState;
import org.elasticsearch.watcher.support.init.proxy.ClientProxy;
import org.joda.time.DateTime;
@ -30,7 +31,7 @@ import static org.elasticsearch.watcher.support.Exceptions.ioException;
*/
public class HistoryStore extends AbstractComponent {
public static final String INDEX_PREFIX = ".watch_history-";
public static final String INDEX_PREFIX = ".watcher-history-" + WatcherModule.getHistoryIndexTemplateVersion() + "-";
public static final String DOC_TYPE = "watch_record";
static final DateTimeFormatter indexTimeFormat = DateTimeFormat.forPattern("YYYY.MM.dd");

View File

@ -153,9 +153,9 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
executor.execute(new Runnable() {
@Override
public void run() {
try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getTemplateName() + ".json")) {
try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getFileName()+ ".json")) {
if (is == null) {
logger.error("Resource [/" + config.getTemplateName() + ".json] not found in classpath");
logger.error("Resource [/" + config.getFileName() + ".json] not found in classpath");
return;
}
final byte[] template;
@ -184,13 +184,23 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
public static class TemplateConfig {
private final String templateName;
private String fileName;
private final Setting<Settings> setting;
public TemplateConfig(String templateName, Setting<Settings> setting) {
this(templateName, templateName, setting);
}
public TemplateConfig(String templateName, String fileName, Setting<Settings> setting) {
this.templateName = templateName;
this.fileName = fileName;
this.setting = setting;
}
public String getFileName() {
return fileName;
}
public String getTemplateName() {
return templateName;
}

View File

@ -1,7 +1,9 @@
{
"template": ".watch_history*",
"template": ".watcher-history-${watcher.template.version}*",
"order": 2147483647,
"settings": {
"watcher.plugin.version": "${watcher.plugin.version}",
"watcher.template.version": "${watcher.template.version}",
"index.number_of_shards": 1,
"index.mapper.dynamic": false
},
@ -29,7 +31,7 @@
}
}
],
"dynamic": "strict",
"dynamic": false,
"_all": {
"enabled": false
},
@ -544,4 +546,4 @@
}
}
}
}
}

View File

@ -0,0 +1 @@
watcher.template.version=${watcher.template.version}

View File

@ -19,7 +19,7 @@ public class WatcherPluginTests extends ESTestCase {
} catch (IllegalArgumentException e) {
}
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index",
".watches,.triggered_watches,.watch_history*").build());
".watches,.triggered_watches,.watcher-history*").build());
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", "*w*").build());
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".w*,.t*").build());
try {
@ -33,7 +33,7 @@ public class WatcherPluginTests extends ESTestCase {
} catch (IllegalArgumentException e) {
}
try {
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watch_history*").build());
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watcher-history*").build());
fail("IllegalArgumentException expected");
} catch (IllegalArgumentException e) {
}

View File

@ -10,6 +10,7 @@ import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.watcher.WatcherModule;
import org.elasticsearch.watcher.execution.ExecutionState;
import org.elasticsearch.watcher.execution.Wid;
import org.elasticsearch.watcher.support.init.proxy.ClientProxy;
@ -45,7 +46,7 @@ public class HistoryStoreTests extends ESTestCase {
WatchRecord watchRecord = new WatchRecord(wid, event, ExecutionState.EXECUTED, null);
IndexResponse indexResponse = mock(IndexResponse.class);
IndexRequest indexRequest = indexRequest(".watch_history-1970.01.01", HistoryStore.DOC_TYPE, wid.value()
IndexRequest indexRequest = indexRequest(".watcher-history-1970.01.01", HistoryStore.DOC_TYPE, wid.value()
, IndexRequest.OpType.CREATE);
when(clientProxy.index(indexRequest, Matchers.<TimeValue>any())).thenReturn(indexResponse);
historyStore.put(watchRecord);
@ -69,9 +70,14 @@ public class HistoryStoreTests extends ESTestCase {
}
public void testIndexNameGeneration() {
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(0, UTC)), equalTo(".watch_history-1970.01.01"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(100000000000L, UTC)), equalTo(".watch_history-1973.03.03"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(1416582852000L, UTC)), equalTo(".watch_history-2014.11.21"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(2833165811000L, UTC)), equalTo(".watch_history-2059.10.12"));
Integer indexTemplateVersion = WatcherModule.getHistoryIndexTemplateVersion();
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(0, UTC)),
equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(100000000000L, UTC)),
equalTo(".watcher-history-" + indexTemplateVersion + "-1973.03.03"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(1416582852000L, UTC)),
equalTo(".watcher-history-" + indexTemplateVersion + "-2014.11.21"));
assertThat(HistoryStore.getHistoryIndexNameForTime(new DateTime(2833165811000L, UTC)),
equalTo(".watcher-history-" + indexTemplateVersion + "-2059.10.12"));
}
}

View File

@ -63,7 +63,7 @@ public class HistoryTemplateTimeMappingsTests extends AbstractWatcherIntegration
assertThat(mappingsResponse, notNullValue());
assertThat(mappingsResponse.getMappings().isEmpty(), is(false));
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> metadatas : mappingsResponse.getMappings()) {
if (!metadatas.key.startsWith(".watch_history")) {
if (!metadatas.key.startsWith(".watcher-history")) {
continue;
}
MappingMetaData metadata = metadatas.value.get("watch_record");

View File

@ -89,7 +89,7 @@ public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegr
assertThat(mappingsResponse, notNullValue());
assertThat(mappingsResponse.getMappings().isEmpty(), is(false));
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> metadatas : mappingsResponse.getMappings()) {
if (!metadatas.key.startsWith(".watch_history")) {
if (!metadatas.key.startsWith(".watcher-history")) {
continue;
}
MappingMetaData metadata = metadatas.value.get("watch_record");

View File

@ -6,7 +6,6 @@
package org.elasticsearch.watcher.support;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;

View File

@ -75,13 +75,13 @@ public class ActivateWatchTests extends AbstractWatcherIntegrationTestCase {
flush();
refresh();
long count1 = docCount(".watch_history*", "watch_record", matchAllQuery());
long count1 = docCount(".watcher-history*", "watch_record", matchAllQuery());
Thread.sleep(10000);
flush();
refresh();
long count2 = docCount(".watch_history*", "watch_record", matchAllQuery());
long count2 = docCount(".watcher-history*", "watch_record", matchAllQuery());
assertThat(count2, is(count1));
@ -96,7 +96,7 @@ public class ActivateWatchTests extends AbstractWatcherIntegrationTestCase {
assertThat(getWatchResponse.getStatus().state().isActive(), is(true));
Thread.sleep(10000);
long count3 = docCount(".watch_history*", "watch_record", matchAllQuery());
long count3 = docCount(".watcher-history*", "watch_record", matchAllQuery());
assertThat(count3, greaterThan(count1));
}

View File

@ -106,11 +106,11 @@
- do:
indices.refresh:
index: .watch_history-*
index: .watcher-history-*
- do:
search:
index: .watch_history-*
index: .watcher-history-*
body: >
{
"query": {

View File

@ -65,11 +65,11 @@
- do:
indices.refresh:
index: .watch_history-*
index: .watcher-history-*
- do:
search:
index: .watch_history-*
index: .watcher-history-*
body: >
{
"query": {