diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
new file mode 100644
index 00000000000..a3deb67646c
--- /dev/null
+++ b/qa/smoke-test-plugins/build.gradle
@@ -0,0 +1,50 @@
+import org.elasticsearch.gradle.MavenFilteringHack
+
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:shield', configuration: 'runtime')
+}
+
+ext.pluginCount = 0
+// this loop must be outside of a configuration closure, otherwise it may get executed multiple times
+for (Project subproj : project.rootProject.subprojects) {
+  if (subproj.path.startsWith(':plugins:')) {
+    // need to get a non-decorated project object, so must re-lookup the project by path
+    integTest.clusterConfig.plugin(subproj.name, project(subproj.path))
+    project.pluginCount += 1
+  }
+}
+
+project.pluginCount += 4
+integTest {
+  cluster {
+    plugin 'licence', project(':x-plugins:license:plugin')
+    plugin 'shield', project(':x-plugins:shield')
+    plugin 'watcher', project(':x-plugins:watcher')
+    plugin 'marvel-agent', project(':x-plugins:marvel')
+
+    setupCommand 'setupDummyUser',
+                 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+    waitCondition = { node, ant ->
+      File tmpFile = new File(node.cwd, 'wait.success')
+      ant.get(src: "http://localhost:${node.httpPort()}",
+              dest: tmpFile.toString(),
+              username: 'test_user',
+              password: 'changeme',
+              ignoreerrors: true,
+              retries: 10)
+      return tmpFile.exists()
+    }
+  }
+} 
+
+ext.expansions = [
+  'expected.plugin.count': pluginCount
+]
+
+processTestResources {
+  inputs.properties(expansions)
+  MavenFilteringHack.filter(it, expansions)
+}
+
diff --git a/qa/smoke-test-plugins/integration-tests.xml b/qa/smoke-test-plugins/integration-tests.xml
deleted file mode 100644
index 4da32222a8f..00000000000
--- a/qa/smoke-test-plugins/integration-tests.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-<?xml version="1.0"?>
-<project name="smoke-test-x-plugins"
-         xmlns:ac="antlib:net.sf.antcontrib">
-
-  <import file="${elasticsearch.integ.antfile.default}"/>
-  <import file="${elasticsearch.tools.directory}/ant/shield-overrides.xml"/>
-
-</project>
diff --git a/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
deleted file mode 100644
index c1454200368..00000000000
--- a/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Integration tests for smoke testing plugins
-#
-"Plugins are actually installed":
-    - do:
-        cluster.state: {}
-
-    # Get master node id
-    - set: { master_node: master }
-
-    - do:
-        nodes.info: {}
-
-    - match:  { nodes.$master.plugins.15.name: license }
-    - match:  { nodes.$master.plugins.18.name: marvel-agent }
-    - match:  { nodes.$master.plugins.21.name: shield }
-    - match:  { nodes.$master.plugins.24.name: watcher }
diff --git a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
new file mode 100644
index 00000000000..f87af6475a4
--- /dev/null
+++ b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
@@ -0,0 +1,14 @@
+# Integration tests for smoke testing plugins
+#
+"Plugins are actually installed":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - length:  { nodes.$master.plugins: ${expected.plugin.count}  }
+    # TODO: check that every plugin is installed