Merge remote-tracking branch 'upstream/master'

Original commit: elastic/x-pack-elasticsearch@a0112ee828

elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/LicenseSerializationTests.java

@@ -49,7 +49,7 @@ public class LicenseSerializationTests extends ESTestCase {
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true")));
         builder.flush();
-        Map<String, Object> map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false).v2();
 
         // should have an extra status field, human readable issue_data and expiry_date
         assertThat(map.get("status"), notNullValue());
@@ -59,14 +59,14 @@ public class LicenseSerializationTests extends ESTestCase {
         builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.flush();
-        map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        map = XContentHelper.convertToMap(builder.bytes(), false).v2();
         assertThat(map.get("status"), nullValue());
 
         license = TestUtils.generateLicenses(new TestUtils.LicenseSpec(validLicenseIssueDate, validLicenseExpiryDate));
         builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true")));
         builder.flush();
-        map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        map = XContentHelper.convertToMap(builder.bytes(), false).v2();
 
         // should have an extra status field, human readable issue_data and expiry_date
         assertThat(map.get("status"), notNullValue());
@@ -76,14 +76,14 @@ public class LicenseSerializationTests extends ESTestCase {
         builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.flush();
-        map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        map = XContentHelper.convertToMap(builder.bytes(), false).v2();
         assertThat(map.get("status"), nullValue());
 
         license = TestUtils.generateLicenses(new TestUtils.LicenseSpec(invalidLicenseIssueDate, validLicenseExpiryDate));
         builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true")));
         builder.flush();
-        map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        map = XContentHelper.convertToMap(builder.bytes(), false).v2();
 
         // should have an extra status field, human readable issue_data and expiry_date
         assertThat(map.get("status"), notNullValue());
@@ -93,7 +93,7 @@ public class LicenseSerializationTests extends ESTestCase {
         builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.flush();
-        map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2();
+        map = XContentHelper.convertToMap(builder.bytes(), false).v2();
         assertThat(map.get("status"), nullValue());
     }
 }

elasticsearch/qa/messy-test-watcher-with-groovy/build.gradle

@@ -9,4 +9,6 @@ apply plugin: 'elasticsearch.messy-test'
 dependencies {
   testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'testArtifacts')
   testCompile project(path: ':modules:lang-groovy', configuration: 'runtime')
+  // some tests depend on both groovy and mustache! this is really bad!
+  testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
 }

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/IndexActionIntegrationTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.sort.SortOrder;
@@ -48,6 +49,7 @@ public class IndexActionIntegrationTests extends AbstractWatcherIntegrationTestC
     protected List<Class<? extends Plugin>> pluginTypes() {
         List<Class<? extends Plugin>> types = super.pluginTypes();
         types.add(GroovyPlugin.class);
+        types.add(MustachePlugin.class);
         return types;
     }
     

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.watcher.support.Script;
 import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.watcher.test.WatcherTestUtils;
@@ -51,6 +52,7 @@ public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCas
     protected List<Class<? extends Plugin>> pluginTypes() {
         List<Class<? extends Plugin>> types = super.pluginTypes();
         types.add(GroovyPlugin.class);
+        types.add(MustachePlugin.class);
         return types;
     }
 

elasticsearch/qa/messy-test-xpack-with-mustache/build.gradle

@@ -0,0 +1,11 @@
+
+/*
+ * Messy tests that depend on mustache directly. Fix these!
+ */
+
+apply plugin: 'elasticsearch.messy-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'testArtifacts')
+  testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
+}

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/BasicWatcherTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.test.integration;
+package org.elasticsearch.messy.tests;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.SearchRequest;
@@ -13,12 +13,11 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.Template;
-import org.elasticsearch.script.mustache.MustacheScriptEngineService;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.watcher.client.WatcherClient;
 import org.elasticsearch.watcher.condition.compare.CompareCondition;
@@ -35,6 +34,9 @@ import org.elasticsearch.watcher.trigger.schedule.support.MonthTimes;
 import org.elasticsearch.watcher.trigger.schedule.support.WeekTimes;
 import org.elasticsearch.watcher.watch.WatchStore;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.*;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
@@ -57,6 +59,14 @@ import static org.hamcrest.Matchers.*;
  */
 public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
 
+    @Override
+    protected List<Class<? extends Plugin>> pluginTypes() {
+        List<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.pluginTypes());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+    
     public void testIndexWatch() throws Exception {
         WatcherClient watcherClient = watcherClient();
         createIndex("idx");

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/EmailActionIntegrationTests.java

@@ -3,11 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.actions.email;
+package org.elasticsearch.messy.tests;
 
 import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.watcher.actions.email.service.EmailTemplate;
@@ -18,6 +20,8 @@ import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.watcher.trigger.schedule.IntervalSchedule;
 import org.junit.After;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -48,6 +52,14 @@ public class EmailActionIntegrationTests extends AbstractWatcherIntegrationTestC
         server.stop();
     }
     
+    @Override
+    protected List<Class<? extends Plugin>> pluginTypes() {
+        List<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.pluginTypes());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         if(server == null) {

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateSearchInputMappingsTests.java

@@ -3,17 +3,23 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.history;
+package org.elasticsearch.messy.tests;
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.watcher.execution.ExecutionState;
+import org.elasticsearch.watcher.history.HistoryStore;
 import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
@@ -31,6 +37,15 @@ import static org.hamcrest.Matchers.notNullValue;
  * not analyzed so they can be used in aggregations
  */
 public class HistoryTemplateSearchInputMappingsTests extends AbstractWatcherIntegrationTestCase {
+    
+    @Override
+    protected List<Class<? extends Plugin>> pluginTypes() {
+        List<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.pluginTypes());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+    
     @Override
     protected boolean timeWarped() {
         return true; // just to have better control over the triggers

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.input.search;
+package org.elasticsearch.messy.tests;
 
 import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -17,7 +17,9 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -27,6 +29,9 @@ import org.elasticsearch.watcher.actions.ExecutableActions;
 import org.elasticsearch.watcher.condition.always.ExecutableAlwaysCondition;
 import org.elasticsearch.watcher.execution.TriggeredExecutionContext;
 import org.elasticsearch.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.watcher.input.search.ExecutableSearchInput;
+import org.elasticsearch.watcher.input.search.SearchInput;
+import org.elasticsearch.watcher.input.search.SearchInputFactory;
 import org.elasticsearch.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.watcher.input.simple.SimpleInput;
 import org.elasticsearch.watcher.support.init.proxy.ClientProxy;
@@ -46,6 +51,7 @@ import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -68,6 +74,15 @@ import static org.joda.time.DateTimeZone.UTC;
  */
 @ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 1)
 public class SearchInputTests extends ESIntegTestCase {
+    
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.nodePlugins());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+    
     private final static String TEMPLATE_QUERY = "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"event_type\":{\"query\":\"a\"," +
             "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" +
             "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," +

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.transform.search;
+package org.elasticsearch.messy.tests;
 
 import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -23,7 +23,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -39,6 +41,9 @@ import org.elasticsearch.watcher.support.init.proxy.ClientProxy;
 import org.elasticsearch.watcher.support.text.TextTemplate;
 import org.elasticsearch.watcher.transform.Transform;
 import org.elasticsearch.watcher.transform.TransformBuilders;
+import org.elasticsearch.watcher.transform.search.ExecutableSearchTransform;
+import org.elasticsearch.watcher.transform.search.SearchTransform;
+import org.elasticsearch.watcher.transform.search.SearchTransformFactory;
 import org.elasticsearch.watcher.trigger.schedule.IntervalSchedule;
 import org.elasticsearch.watcher.trigger.schedule.ScheduleTrigger;
 import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent;
@@ -55,6 +60,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -88,6 +94,15 @@ import static org.joda.time.DateTimeZone.UTC;
  */
 @ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 1)
 public class SearchTransformTests extends ESIntegTestCase {
+    
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.nodePlugins());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+    
     @Override
     public Settings nodeSettings(int nodeOrdinal) {
         final Path tempDir = createTempDir();

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.integration;
+package org.elasticsearch.messy.tests;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
@@ -11,8 +11,10 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.cache.query.terms.TermsLookup;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.Template;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
@@ -20,15 +22,26 @@ import org.elasticsearch.test.ShieldSettingsSource;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 
 import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
+@ShieldIntegTestCase.AwaitsFix(bugUrl = "clean up test to not use mustache templates, otherwise needs many resources here")
 public class ShieldCachePermissionTests extends ShieldIntegTestCase {
     static final String READ_ONE_IDX_USER = "read_user";
     
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.nodePlugins());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+
     @Override
     public String configUsers() {
         return super.configUsers()

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/TimeThrottleIntegrationTests.java

@@ -3,11 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.actions;
+package org.elasticsearch.messy.tests;
 
 
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.watcher.client.WatcherClient;
 import org.elasticsearch.watcher.condition.compare.CompareCondition;
 import org.elasticsearch.watcher.execution.ExecutionState;
@@ -18,6 +20,8 @@ import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -36,6 +40,15 @@ import static org.hamcrest.Matchers.is;
 /**
  */
 public class TimeThrottleIntegrationTests extends AbstractWatcherIntegrationTestCase {
+    
+    @Override
+    protected List<Class<? extends Plugin>> pluginTypes() {
+        List<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.pluginTypes());
+        types.add(MustachePlugin.class);
+        return types;
+    }
+    
     private IndexResponse indexTestDoc() {
         createIndex("actions", "events");
         ensureGreen("actions", "events");

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * This package contains tests that use mustache to test what looks
+ * to be unrelated functionality, or functionality that should be 
+ * tested with a mock instead. Instead of doing an epic battle
+ * with these tests, they are temporarily moved here to the mustache
+ * module's tests, but that is likely not where they belong. Please 
+ * help by cleaning them up and we can remove this package!
+ *
+ * <ul>
+ *   <li>If the test is testing templating integration with another core subsystem,
+ *       fix it to use a mock instead, so it can be in the core tests again</li>
+ *   <li>If the test is just being lazy, and does not really need templating to test
+ *       something, clean it up!</li>
+ * </ul>
+ */
+
+// renames that took place:
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/BasicWatcherTests.java
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/EmailActionIntegrationTests.java
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateSearchInputMappingsTests.java
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputTests.java
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformTests.java
+//  renamed:    x-pack/shield/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionTests.java
+//  renamed:    x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java -> qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/TimeThrottleIntegrationTests.java
+
+package org.elasticsearch.messy.tests;

elasticsearch/qa/shield-audit-tests/build.gradle

@@ -13,7 +13,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_user',
               password: 'changeme',

elasticsearch/qa/shield-client-tests/build.gradle

@@ -13,7 +13,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'transport', '-p', 'changeme', '-r', 'transport_client'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_user',
               password: 'changeme',

elasticsearch/qa/shield-core-rest-tests/build.gradle

@@ -37,7 +37,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_user',
               password: 'changeme',

elasticsearch/qa/shield-example-realm/build.gradle

@@ -26,7 +26,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_user',
               password: 'changeme',

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -18,6 +18,23 @@ task createKey(type: LoggedExec) {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
+  String subjectAlternateNames = 'san=dns:localhost,ip:127.0.0.1'
+  // some machines have a different name for ipv6 loopback,
+  // at least on ubuntu its ip6-localhost. other machines, like windows,
+  // won't resolve it back to any hostname at all. Try to setup ipv6 to
+  // work in all cases.
+  try {
+      String localhost6 = InetAddress.getByName("::1").getCanonicalHostName()
+      if (!localhost6.equals("localhost")) {
+          if (localhost6.startsWith("0")) {
+              subjectAlternateNames += ",ip:" + localhost6
+          } else {
+              subjectAlternateNames += ",dns:" + localhost6
+          }
+      }
+  } catch (UnknownHostException ok) {
+      // e.g. no ipv6 support
+  }
   executable = 'keytool'
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
@@ -26,7 +43,7 @@ task createKey(type: LoggedExec) {
        '-keyalg', 'RSA',
        '-keysize', '2048',
        '-validity', '712',
-       '-ext', 'san=dns:localhost,ip:127.0.0.1',
+       '-ext', subjectAlternateNames,
        '-storepass', 'keypass'
 }
 
@@ -43,8 +60,6 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each
 
 integTest {
   cluster {
-    // TODO: use some variable here for port number
-    systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400'
     systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.path', keystore.name
     systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.password', 'keypass'
     systemProperty 'es.shield.transport.ssl', 'true'

elasticsearch/qa/smoke-test-plugins/build.gradle

@@ -21,7 +21,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_user',
               password: 'changeme',

elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.test.rest;
+package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.test.rest;
+package org.elasticsearch.smoketest;
 
 import org.elasticsearch.test.rest.RestTestCandidate;
 

elasticsearch/qa/smoke-test-watcher-with-shield/build.gradle

@@ -15,7 +15,8 @@ integTest {
   dependsOn copyWatcherRestTests
   systemProperty 'tests.rest.blacklist',
     ['hijack/10_basic/*',
-     'array_compare_watch/10_basic/Basic array_compare watch'].join(',')
+     'array_compare_watch/10_basic/Basic array_compare watch',
+     'getting_started/10_monitor_cluster_health/Getting started - Monitor cluster health'].join(',')
 
   cluster {
     plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
@@ -28,7 +29,7 @@ integTest {
                  'bin/x-pack/esusers', 'useradd', 'powerless_user', '-p', 'changeme', '-r', 'crapy_role'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://localhost:${node.httpPort()}",
+      ant.get(src: "http://${node.httpUri()}",
               dest: tmpFile.toString(),
               username: 'test_admin',
               password: 'changeme',

elasticsearch/x-pack/build.gradle

@@ -33,10 +33,12 @@ dependencies {
   compile 'com.google.guava:guava:16.0.1' // needed by watcher and shield tests for jimfs
   compile 'com.google.code.findbugs:jsr305:3.0.1' // TODO: remove this
   compile 'com.sun.mail:javax.mail:1.5.3'
+  // fork of mustache: https://github.com/elastic/x-plugins/issues/1116 
+  compile 'com.github.spullara.mustache.java:compiler:0.9.1' // TODO: remove this
   testCompile 'org.subethamail:subethasmtp:3.1.7'
 
   // common test deps
-  testCompile 'org.elasticsearch:securemock:1.1'
+  testCompile 'org.elasticsearch:securemock:1.2'
   testCompile 'org.slf4j:slf4j-log4j12:1.6.2'
   testCompile 'org.slf4j:slf4j-api:1.6.2'
 
@@ -68,7 +70,6 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
 
 ext.expansions = [
   'project.version': version,
-  'integ.http.port': integTest.cluster.baseHttpPort
 ]
 
 processResources {
@@ -115,13 +116,15 @@ bundlePlugin {
   from('watcher/bin/watcher') {
     into 'bin'
   }
+}
 
-  integTest {
+integTest {
+  // TODO: fix this rest test to not depend on a hardcoded port!
+  systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*'
   cluster {
     // TODO set up tests so that shield can be enabled or disabled
     systemProperty 'es.shield.enabled', 'false'
   }
-  }
 }
 
 // TODO: don't publish test artifacts just to run messy tests, fix the tests!

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java

@@ -42,7 +42,7 @@ public class PutLicenseResponseTests extends ESTestCase {
         response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
         contentBuilder.endObject();
 
-        Map<String, Object> map = XContentHelper.convertToMap(contentBuilder.bytesStream().bytes(), false).v2();
+        Map<String, Object> map = XContentHelper.convertToMap(contentBuilder.bytes(), false).v2();
         assertThat(map.containsKey("acknowledged"), equalTo(true));
         boolean actualAcknowledged = (boolean) map.get("acknowledged");
         assertThat(actualAcknowledged, equalTo(acknowledged));

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java

@@ -199,18 +199,15 @@ public class HttpExporter extends Exporter {
             }
             builder.endObject();
             builder.endObject();
-
+        }
         // Adds action metadata line bulk separator
-            builder.flush(); // Flush is needed here because the separator is written directly in the builder's stream
+        out.write(xContentType.xContent().streamSeparator());
-            builder.stream().write(builder.contentType().xContent().streamSeparator());
 
         // Render the MarvelDoc
         renderer.render(marvelDoc, xContentType, out);
 
         // Adds final bulk separator
-            builder.flush();
+        out.write(xContentType.xContent().streamSeparator());
-            builder.stream().write(builder.contentType().xContent().streamSeparator());
-        }
     }
 
     @SuppressWarnings("unchecked")

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java

@@ -61,7 +61,7 @@ public class ShieldActionMapperTests extends ESTestCase {
         }
         clearScrollRequest.addScrollId("_all");
         //make sure that wherever the _all is among the scroll ids the action name gets translated
-        Collections.shuffle(clearScrollRequest.getScrollIds(), getRandom());
+        Collections.shuffle(clearScrollRequest.getScrollIds(), random());
 
         assertThat(shieldActionMapper.action(ClearScrollAction.NAME, clearScrollRequest), equalTo(ShieldActionMapper.CLUSTER_PERMISSION_SCROLL_CLEAR_ALL_NAME));
     }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java

@@ -60,7 +60,7 @@ public class RealmsTests extends ESTestCase {
         for (int i = 0; i < factories.size() - 1; i++) {
             orders.add(i);
         }
-        Collections.shuffle(orders, getRandom());
+        Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
         for (int i = 0; i < factories.size() - 1; i++) {
             builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i);
@@ -118,7 +118,7 @@ public class RealmsTests extends ESTestCase {
         for (int i = 0; i < factories.size() - 1; i++) {
             orders.add(i);
         }
-        Collections.shuffle(orders, getRandom());
+        Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
         for (int i = 0; i < factories.size() - 1; i++) {
             builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i);
@@ -186,7 +186,7 @@ public class RealmsTests extends ESTestCase {
         for (int i = 0; i < factories.size() - 1; i++) {
             orders.add(i);
         }
-        Collections.shuffle(orders, getRandom());
+        Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
         for (int i = 0; i < factories.size() - 1; i++) {
             builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i);

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java

@@ -8,11 +8,10 @@ package org.elasticsearch.shield.tribe;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Arrays;
 
@@ -42,7 +41,7 @@ public class TribeShieldLoadedTests extends ESTestCase {
         Settings.Builder builder = defaultSettings();
 
         try {
-            NodeBuilder.nodeBuilder().settings(builder.build()).build();
+            new Node(builder.build());
             fail("node initialization should have failed due to missing shield plugin");
         } catch(Throwable t) {
             assertThat(t.getMessage(), containsString("Missing mandatory plugins [shield]"));
@@ -55,7 +54,7 @@ public class TribeShieldLoadedTests extends ESTestCase {
         Settings.Builder builder = addTribeSettings(defaultSettings(), "t2");
 
         try {
-            NodeBuilder.nodeBuilder().settings(builder.build()).build();
+            new Node(builder.build());
             fail("node initialization should have failed due to missing shield plugin");
         } catch(Throwable t) {
             assertThat(t.getMessage(), containsString("Missing mandatory plugins [shield]"));

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java

@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.watcher.test.bench;
 
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.watcher.support.clock.SystemClock;
 import org.elasticsearch.watcher.trigger.Trigger;
 import org.elasticsearch.watcher.trigger.TriggerEngine;
@@ -22,7 +22,6 @@ import org.elasticsearch.watcher.trigger.schedule.engine.TickerScheduleTriggerEn
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -65,7 +64,7 @@ public class ScheduleEngineTriggerBenchmark {
         }
         ScheduleRegistry scheduleRegistry = new ScheduleRegistry(emptyMap());
         List<String> impls = new ArrayList<>(Arrays.asList(new String[]{"schedule", "ticker"}));
-        Collections.shuffle(impls);
+        Randomness.shuffle(impls);
 
         List<Stats> results = new ArrayList<>();
         for (String impl : impls) {

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java

@@ -11,21 +11,22 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.watcher.WatcherPlugin;
-import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.xpack.XPackPlugin;
+import org.elasticsearch.watcher.WatcherPlugin;
 import org.elasticsearch.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.watcher.client.WatcherClient;
 import org.elasticsearch.watcher.support.http.HttpRequestTemplate;
 import org.elasticsearch.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.watcher.trigger.ScheduleTriggerEngineMock;
 import org.elasticsearch.watcher.trigger.TriggerModule;
+import org.elasticsearch.xpack.XPackPlugin;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
 
 import static org.elasticsearch.watcher.actions.ActionBuilders.indexAction;
 import static org.elasticsearch.watcher.condition.ConditionBuilders.scriptCondition;

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java

@@ -19,18 +19,15 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPoolStats;
-import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.watcher.WatcherState;
 import org.elasticsearch.watcher.actions.ActionBuilders;
 import org.elasticsearch.watcher.actions.logging.LoggingLevel;
@@ -39,6 +36,7 @@ import org.elasticsearch.watcher.client.WatcherClient;
 import org.elasticsearch.watcher.history.HistoryStore;
 import org.elasticsearch.watcher.support.clock.Clock;
 import org.elasticsearch.watcher.watch.WatchStore;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -47,7 +45,9 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.search.aggregations.AggregationBuilders.*;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.watcher.condition.ConditionBuilders.scriptCondition;
 import static org.elasticsearch.watcher.input.InputBuilders.searchInput;
 import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule;
@@ -94,7 +94,7 @@ public class WatcherScheduleEngineBenchmark {
 
 
         // First clean everything and index the watcher (but not via put alert api!)
-        try (Node node = NodeBuilder.nodeBuilder().settings(SETTINGS).data(false).node()) {
+        try (Node node = new Node(Settings.builder().put(SETTINGS).put("node.data", false).build()).start()) {
             try (Client client = node.client()) {
                 ClusterHealthResponse response = client.admin().cluster().prepareHealth().setWaitForNodes("2").get();
                 if (response.getNumberOfNodes() != 2 && response.getNumberOfDataNodes() != 1) {

elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/test/watcher/array_compare_watch/10_basic.yaml

@@ -1,5 +1,10 @@
 ---
 "Basic array_compare watch":
+
+  - skip:
+      version:     " - "
+      reason:      Remove direct dependency on mustache (or at least it should be using xmustache!!!!)
+
   - do:
       cluster.health:
           wait_for_status: yellow