diff --git a/qa/smoke-test-watcher-with-shield/build.gradle b/qa/smoke-test-watcher-with-shield/build.gradle
new file mode 100644
index 00000000000..eb1568c4ce5
--- /dev/null
+++ b/qa/smoke-test-watcher-with-shield/build.gradle
@@ -0,0 +1,45 @@
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:shield', configuration: 'runtime')
+  testCompile project(path: ':x-plugins:watcher', configuration: 'runtime')
+}
+
+// bring in watcher rest test suite
+task copyWatcherRestTests(type: Copy) {
+  into project.sourceSets.test.output.resourcesDir
+  from project(':x-plugins:watcher').sourceSets.test.resources.srcDirs
+  include 'rest-api-spec/test/**'
+}
+
+integTest.dependsOn(copyWatcherRestTests)
+
+integTest {
+  systemProperty 'tests.rest.blacklist',
+    ['hijack/10_basic/*',
+     'array_compare_watch/10_basic/Basic array_compare watch'].join(',')
+
+  cluster {
+    plugin 'license', project(':x-plugins:license:plugin')
+    plugin 'shield', project(':x-plugins:shield')
+    plugin 'watcher', project(':x-plugins:watcher')
+    extraConfigFile 'shield', 'roles.yml'
+    setupCommand 'setupTestAdminUser',
+                 'bin/shield/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin'
+    setupCommand 'setupWatcherManagerUser',
+                 'bin/shield/esusers', 'useradd', 'watcher_manager', '-p', 'changeme', '-r', 'watcher_manager'
+    setupCommand 'setupPowerlessUser',
+                 'bin/shield/esusers', 'useradd', 'powerless_user', '-p', 'changeme', '-r', 'crapy_role'
+    waitCondition = { node, ant ->
+      File tmpFile = new File(node.cwd, 'wait.success')
+      ant.get(src: "http://localhost:${node.httpPort()}",
+              dest: tmpFile.toString(),
+              username: 'test_admin',
+              password: 'changeme',
+              ignoreerrors: true,
+              retries: 10)
+      return tmpFile.exists()
+    }
+  }
+}
+
diff --git a/qa/smoke-test-watcher-with-shield/integration-tests.xml b/qa/smoke-test-watcher-with-shield/integration-tests.xml
deleted file mode 100644
index 8b0d6aedf22..00000000000
--- a/qa/smoke-test-watcher-with-shield/integration-tests.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  ~ ELASTICSEARCH CONFIDENTIAL
-  ~ __________________
-  ~
-  ~  [2014] Elasticsearch Incorporated. All Rights Reserved.
-  ~
-  ~ NOTICE:  All information contained herein is, and remains
-  ~ the property of Elasticsearch Incorporated and its suppliers,
-  ~ if any.  The intellectual and technical concepts contained
-  ~ herein are proprietary to Elasticsearch Incorporated
-  ~ and its suppliers and may be covered by U.S. and Foreign Patents,
-  ~ patents in process, and are protected by trade secret or copyright law.
-  ~ Dissemination of this information or reproduction of this material
-  ~ is strictly forbidden unless prior written permission is obtained
-  ~ from Elasticsearch Incorporated.
-  -->
-
-<project name="smoke-test-watcher-and-shield"
-         xmlns:ac="antlib:net.sf.antcontrib">
-
-  <import file="${elasticsearch.integ.antfile.default}"/>
-
-  <!-- redefined to work with auth -->
-  <macrodef name="waitfor-elasticsearch">
-      <attribute name="port"/>
-      <attribute name="timeoutproperty"/>
-    <sequential>
-      <echo>Waiting for elasticsearch to become available on port @{port}...</echo>
-      <waitfor maxwait="30" maxwaitunit="second"
-               checkevery="500" checkeveryunit="millisecond"
-               timeoutproperty="@{timeoutproperty}">
-        <socket server="127.0.0.1" port="@{port}"/>
-      </waitfor>
-    </sequential>
-  </macrodef>
-
-  <target name="start-external-cluster-with-watcher-and-shield" depends="setup-workspace">
-    <ac:for list="${xplugins.list}" param="xplugin.name">
-      <sequential>
-        <fail message="Expected @{xplugin.name}-${version}.zip as a dependency, but could not be found in ${integ.deps}/plugins}">
-          <condition>
-            <not>
-              <available file="${integ.deps}/plugins/@{xplugin.name}-${elasticsearch.version}.zip" />
-            </not>
-          </condition>
-        </fail>
-      </sequential>
-    </ac:for>
-
-    <ac:for param="file">
-      <path>
-        <fileset dir="${integ.deps}/plugins"/>
-      </path>
-      <sequential>
-        <local name="plugin.name"/>
-        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
-        <install-plugin name="${plugin.name}" file="@{file}"/>
-      </sequential>
-    </ac:for>
-
-    <local name="home"/>
-    <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
-
-    <echo>Adding roles.yml with watcher roles</echo>
-    <copy file="watcher-with-shield-roles.yml" tofile="${home}/config/shield/roles.yml" overwrite="true"/>
-
-    <echo>Adding shield users...</echo>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="test_admin"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="admin"/>
-      </nested>
-    </run-script>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="watcher_manager"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="watcher_manager"/>
-      </nested>
-    </run-script>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="powerless_user"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="crapy_role"/>
-      </nested>
-    </run-script>
-
-    <startup-elasticsearch>
-      <!-- Useful for when debugging -->
-      <!--<additional-args>-->
-        <!--<arg value="-Des.http.cors.enabled=true"/>-->
-        <!--<arg value="-Des.http.cors.allow-origin=*"/>-->
-      <!--</additional-args>-->
-    </startup-elasticsearch>
-
-    <echo>Checking we can connect with basic auth on port ${integ.http.port}...</echo>
-    <local name="temp.file"/>
-    <tempfile property="temp.file" destdir="${java.io.tmpdir}"/>
-    <get src="http://127.0.0.1:${integ.http.port}" dest="${temp.file}" 
-         username="test_admin" password="changeme" verbose="true" retries="10"/>
-  </target>
-</project>
diff --git a/qa/smoke-test-watcher-with-shield/watcher-with-shield-roles.yml b/qa/smoke-test-watcher-with-shield/roles.yml
similarity index 100%
rename from qa/smoke-test-watcher-with-shield/watcher-with-shield-roles.yml
rename to qa/smoke-test-watcher-with-shield/roles.yml