honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into immutable_map_be_gone 

Original commit: elastic/x-pack-elasticsearch@9eb0fb4bd7
This commit is contained in:
Nik Everett 2015-10-06 09:41:00 -04:00
parent dab504f6ba fec6ab7c22
commit 0ff0cb7cd5
6 changed files with 132 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
watcher/docs/release-notes.asciidoc
View File

@ -48,6 +48,10 @@ bin/plugin remove watcher
`watcher.dynamic_indices.time_zone` setting was set to `+01:00` and a watch has the following index name `<logstash-{now/d}>`
then after the upgrade you need to update this watch to use the following index name `<logstash-{now/d{YYYY.MM.dd|+01:00}}>`.
.Enhancements
* If `action.auto_create_index` setting has been configured then Watcher will check if the setting is too restrictive.
If the `action.auto_create_index` is too restrictive then Watcher will fail during startup with a descriptive error message.
.Bug Fixes
* Fixed url encoding issue in http input and webhook output. The url params were url encoded twice.

71
watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java
View File

@ -10,8 +10,14 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.logging.support.LoggerMessageFormat;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestModule;
@ -25,6 +31,7 @@ import org.elasticsearch.watcher.client.WatcherClientModule;
import org.elasticsearch.watcher.condition.ConditionModule;
import org.elasticsearch.watcher.execution.ExecutionModule;
import org.elasticsearch.watcher.history.HistoryModule;
import org.elasticsearch.watcher.history.HistoryStore;
import org.elasticsearch.watcher.input.InputModule;
import org.elasticsearch.watcher.license.LicenseModule;
import org.elasticsearch.watcher.license.LicenseService;
@ -63,10 +70,10 @@ import org.elasticsearch.watcher.transport.actions.stats.WatcherStatsAction;
import org.elasticsearch.watcher.trigger.TriggerModule;
import org.elasticsearch.watcher.trigger.schedule.ScheduleModule;
import org.elasticsearch.watcher.watch.WatchModule;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.*;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@ -75,6 +82,8 @@ public class WatcherPlugin extends Plugin {
public static final String NAME = "watcher";
public static final String ENABLED_SETTING = NAME + ".enabled";
private final static ESLogger logger = Loggers.getLogger(WatcherPlugin.class);
static {
MetaData.registerPrototype(WatcherMetaData.TYPE, WatcherMetaData.PROTO);
}
@ -87,6 +96,7 @@ public class WatcherPlugin extends Plugin {
this.settings = settings;
transportClient = "transport".equals(settings.get(Client.CLIENT_TYPE_SETTING));
enabled = watcherEnabled(settings);
validAutoCreateIndex(settings);
}
@Override public String name() {
@ -208,4 +218,59 @@ public class WatcherPlugin extends Plugin {
return settings.getAsBoolean(ENABLED_SETTING, true);
}
static void validAutoCreateIndex(Settings settings) {
String value = settings.get("action.auto_create_index");
if (value == null) {
return;
}
String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too restrictive. disable [action.auto_create_index] or set it to [.watches,.triggered_watches,.watch_history*]", (Object) settings);
if (Booleans.isExplicitFalse(value)) {
throw new IllegalArgumentException(errorMessage);
}
if (Booleans.isExplicitTrue(value)) {
return;
}
String[] matches = Strings.commaDelimitedListToStringArray(value);
List<String> indices = new ArrayList<>();
indices.add(".watches");
indices.add(".triggered_watches");
DateTime now = new DateTime(DateTimeZone.UTC);
indices.add(HistoryStore.getHistoryIndexNameForTime(now));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusDays(1)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(1)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(2)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(3)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(4)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(5)));
indices.add(HistoryStore.getHistoryIndexNameForTime(now.plusMonths(6)));
for (String index : indices) {
boolean matched = false;
for (String match : matches) {
char c = match.charAt(0);
if (c == '-') {
if (Regex.simpleMatch(match.substring(1), index)) {
throw new IllegalArgumentException(errorMessage);
}
} else if (c == '+') {
if (Regex.simpleMatch(match.substring(1), index)) {
matched = true;
break;
}
} else {
if (Regex.simpleMatch(match, index)) {
matched = true;
break;
}
}
}
if (!matched) {
throw new IllegalArgumentException(errorMessage);
}
}
logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. for the next 6 months daily history indices are allowed to be created, but please make sure that any future history indices after 6 months with the pattern [.watch_history-YYYY.MM.dd] are allowed to be created", value);
}
}

35
watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java
View File

@ -57,36 +57,6 @@ public class XMustacheScriptEngineService extends AbstractComponent implements S
return (new XMustacheFactory(xContentType)).compile(new FastStringReader(template), "query-template");
}
/**
* Execute a compiled template object (as retrieved from the compile method)
* and fill potential place holders with the variables given.
*
* @param compiledScript
* compiled template object.
* @param vars
* map of variables to use during substitution.
*
* @return the processed string with all given variables substitued.
* */
@Override
public Object execute(CompiledScript compiledScript, Map<String, Object> vars) {
BytesStreamOutput result = new BytesStreamOutput();
UTF8StreamWriter writer = utf8StreamWriter().setOutput(result);
((Mustache) compiledScript.compiled()).execute(writer, vars);
try {
writer.flush();
} catch (IOException e) {
logger.error("Could not execute query template (failed to flush writer): ", e);
} finally {
try {
writer.close();
} catch (IOException e) {
logger.error("Could not execute query template (failed to close writer): ", e);
}
}
return result.bytes();
}
@Override
public String[] types() {
return new String[] { NAME };
@ -114,11 +84,6 @@ public class XMustacheScriptEngineService extends AbstractComponent implements S
throw new UnsupportedOperationException();
}
@Override
public Object unwrap(Object value) {
return value;
}
@Override
public void close() {
// Nothing to do here

41
watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.watcher;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
public class WatcherPluginTests extends ESTestCase {
public void testValidAutoCreateIndex() {
WatcherPlugin.validAutoCreateIndex(Settings.EMPTY);
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", true).build());
try {
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", false).build());
fail("IllegalArgumentException expected");
} catch (IllegalArgumentException e) {
}
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watches,.triggered_watches,.watch_history*").build());
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", "*w*").build());
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".w*,.t*").build());
try {
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watches").build());
fail("IllegalArgumentException expected");
} catch (IllegalArgumentException e) {
}
try {
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".triggered_watch").build());
fail("IllegalArgumentException expected");
} catch (IllegalArgumentException e) {
}
try {
WatcherPlugin.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watch_history*").build());
fail("IllegalArgumentException expected");
} catch (IllegalArgumentException e) {
}
}
}

6
watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java
View File

@ -40,7 +40,7 @@ public class XMustacheScriptEngineTests extends ESTestCase {
Map<String, Object> vars = new HashMap<>();
vars.put("boost_val", "0.3");
CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template));
BytesReference o = (BytesReference) engine.execute(compiledScript, vars);
BytesReference o = (BytesReference) engine.executable(compiledScript, vars).run();
assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
+ "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}",
new String(o.toBytes(), Charset.forName("UTF-8")));
@ -52,7 +52,7 @@ public class XMustacheScriptEngineTests extends ESTestCase {
vars.put("boost_val", "0.3");
vars.put("body_val", "\"quick brown\"");
CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template));
BytesReference o = (BytesReference) engine.execute(compiledScript, vars);
BytesReference o = (BytesReference) engine.executable(compiledScript, vars).run();
assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
+ "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}",
new String(o.toBytes(), Charset.forName("UTF-8")));
@ -78,7 +78,7 @@ public class XMustacheScriptEngineTests extends ESTestCase {
vars.put("test_var1", var1Writer.toString());
vars.put("test_var2", var2Writer.toString());
CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template));
BytesReference o = (BytesReference) engine.execute(compiledScript, vars);
BytesReference o = (BytesReference) engine.executable(compiledScript, vars).run();
String s1 = o.toUtf8();
String s2 = prefix + " " + var1Writer.toString() + " " + var2Writer.toString();
assertEquals(s1, s2);

25
watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java
View File

@ -22,11 +22,12 @@ import java.io.StringWriter;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -54,15 +55,18 @@ public class XMustacheTests extends ESTestCase {
new String[] { "foo", "bar" },
Arrays.asList("foo", "bar"));
vars.put("data", data);
Object output = engine.execute(mustache, vars);
Object output = engine.executable(mustache, vars).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));
BytesReference bytes = (BytesReference) output;
assertThat(bytes.toUtf8(), equalTo("foo bar"));
// Sets can come out in any order
vars.put("data", newHashSet("foo", "bar"));
output = engine.execute(mustache, vars);
Set<String> setData = new HashSet<>();
setData.add("foo");
setData.add("bar");
vars.put("data", setData);
output = engine.executable(mustache, vars).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));
bytes = (BytesReference) output;
@ -80,7 +84,7 @@ public class XMustacheTests extends ESTestCase {
singleton(new String[] { "foo", "bar" })
);
vars.put("data", data);
Object output = engine.execute(mustache, vars);
Object output = engine.executable(mustache, vars).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));
BytesReference bytes = (BytesReference) output;
@ -96,15 +100,18 @@ public class XMustacheTests extends ESTestCase {
new Map[] { singletonMap("key", "foo"), singletonMap("key", "bar") },
Arrays.asList(singletonMap("key", "foo"), singletonMap("key", "bar")));
vars.put("data", data);
Object output = engine.execute(mustache, vars);
Object output = engine.executable(mustache, vars).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));
BytesReference bytes = (BytesReference) output;
assertThat(bytes.toUtf8(), equalTo("foo bar"));
// HashSet iteration order isn't fixed
vars.put("data", newHashSet(singletonMap("key", "foo"), singletonMap("key", "bar")));
output = engine.execute(mustache, vars);
Set<Object> setData = new HashSet<>();
setData.add(singletonMap("key", "foo"));
setData.add(singletonMap("key", "bar"));
vars.put("data", setData);
output = engine.executable(mustache, vars).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));
bytes = (BytesReference) output;
@ -156,7 +163,7 @@ public class XMustacheTests extends ESTestCase {
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("data", unescaped.toString());
CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template));
Object output = engine.execute(mustache, dataMap);
Object output = engine.executable(mustache, dataMap).run();
assertThat(output, notNullValue());
assertThat(output, instanceOf(BytesReference.class));