diff --git a/dev-tools/smoke_test_plugins.py b/dev-tools/smoke_test_plugins.py
deleted file mode 100644
index da6c2c95209..00000000000
--- a/dev-tools/smoke_test_plugins.py
+++ /dev/null
@@ -1,172 +0,0 @@
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance  with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on 
-# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-# either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-import datetime
-import traceback
-import json
-import os
-import shutil
-import signal
-import socket
-import subprocess
-import tempfile
-import threading
-import time
-
-from http.client import HTTPConnection
-
-LOG = os.environ.get('ES_SMOKE_TEST_PLUGINS_LOG', '/tmp/elasticsearch_smoke_test_plugins.log')
-
-print('Logging to %s' % LOG)
-
-if os.path.exists(LOG):
-  raise RuntimeError('please remove old log %s first' % LOG)
-
-try:
-  JAVA_HOME = os.environ['JAVA7_HOME']
-except KeyError:
-  try:
-    JAVA_HOME = os.environ['JAVA_HOME']
-  except KeyError:
-    raise RuntimeError("""
-    Please set JAVA_HOME in the env before running release tool
-    On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.7*'`""")
-
-JAVA_ENV = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (JAVA_HOME, JAVA_HOME, JAVA_HOME)
-
-try:
-  # make sure mvn3 is used if mvn3 is available
-  # some systems use maven 2 as default
-  subprocess.check_output('mvn3 --version', shell=True, stderr=subprocess.STDOUT)
-  MVN = 'mvn3'
-except subprocess.CalledProcessError:
-  MVN = 'mvn'
-
-def log(msg):
-  f = open(LOG, mode='ab')
-  f.write(('\n'+msg).encode('utf-8'))
-  f.close()
-
-def run(command, quiet=False):
-  log('%s: RUN: %s\n' % (datetime.datetime.now(), command))
-  if os.system('%s >> %s 2>&1' % (command, LOG)):
-    msg = '    FAILED: %s [see log %s]' % (command, LOG)
-    if not quiet:
-      print(msg)
-    raise RuntimeError(msg)
-
-def readServerOutput(p, startupEvent, failureEvent):
-  try:
-    while True:
-      line = p.stdout.readline()
-      if len(line) == 0:
-        p.poll()
-        if not startupEvent.isSet():
-          failureEvent.set()
-          startupEvent.set()
-        print('ES: **process exit**\n')
-        break
-      line = line.decode('utf-8').rstrip()
-      if line.endswith('started') and not startupEvent.isSet():
-        startupEvent.set()
-      print('ES: %s' % line)
-  except:
-    print()
-    print('Exception reading Elasticsearch output:')
-    traceback.print_exc()
-    failureEvent.set()
-    startupEvent.set()
-
-if __name__ == '__main__':
-  print('Build release bits...')
-
-  run('%s; %s clean package -DskipTests' % (JAVA_ENV, MVN))
-
-  for f in os.listdir('distribution/tar/target/releases/'):
-    if f.endswith('.tar.gz'):
-      artifact = f
-      break
-  else:
-    raise RuntimeError('could not find elasticsearch release under distribution/tar/target/releases/')
-  
-  tmp_dir = tempfile.mkdtemp()
-  p = None
-  try:
-    # Extract artifact:
-    run('tar -xzf distribution/tar/target/releases/%s -C %s' % (artifact, tmp_dir))
-    es_install_dir = os.path.join(tmp_dir, artifact[:-7])
-    es_plugin_path = os.path.join(es_install_dir, 'bin/plugin')
-    installed_plugin_names = set()
-    print('Find plugins:')
-    for name in os.listdir('plugins'):
-      if name not in ('target', 'pom.xml'):
-        url = 'file://%s/plugins/%s/target/releases/elasticsearch-%s-2.0.0-beta1-SNAPSHOT.zip' % (os.path.abspath('.'), name, name)
-        print('  install plugin %s...' % name)
-        run('%s; %s install %s --url %s' % (JAVA_ENV, es_plugin_path, name, url))
-        installed_plugin_names.add(name)
-
-    print('Start Elasticsearch')
-
-    env = os.environ.copy()
-    env['JAVA_HOME'] = JAVA_HOME
-    env['PATH'] = '%s/bin:%s' % (JAVA_HOME, env['PATH'])
-    env['JAVA_CMD'] = '%s/bin/java' % JAVA_HOME
-    
-    startupEvent = threading.Event()
-    failureEvent = threading.Event()
-    p = subprocess.Popen(('%s/bin/elasticsearch' % es_install_dir,
-                          '-Des.node.name=smoke_tester',
-                          '-Des.cluster.name=smoke_tester_cluster'
-                          '-Des.discovery.zen.ping.multicast.enabled=false',
-                          '-Des.logger.level=debug',
-                          '-Des.script.inline=on',
-                          '-Des.script.indexed=on'),
-                         stdout = subprocess.PIPE,
-                         stderr = subprocess.STDOUT,
-                         env = env)
-    thread = threading.Thread(target=readServerOutput, args=(p, startupEvent, failureEvent))
-    thread.setDaemon(True)
-    thread.start()
-
-    startupEvent.wait(1200)
-    if failureEvent.isSet():
-      raise RuntimeError('ES failed to start')
-
-    print('Confirm plugins are installed')
-    conn = HTTPConnection('127.0.0.1', 9200, 20);
-    conn.request('GET', '/_nodes?plugin=true&pretty=true')
-    res = conn.getresponse()
-    if res.status == 200:
-      nodes = json.loads(res.read().decode("utf-8"))['nodes']
-      for _, node in nodes.items():
-        node_plugins = node['plugins']
-        for node_plugin in node_plugins:
-          plugin_name = node_plugin['name']
-          if plugin_name not in installed_plugin_names:
-            raise RuntimeError('Unexpeced plugin %s' % plugin_name)
-          installed_plugin_names.remove(plugin_name)
-      if len(installed_plugin_names) > 0:
-        raise RuntimeError('Plugins not loaded %s' % installed_plugin_names)
-    else:
-      raise RuntimeError('Expected HTTP 200 but got %s' % res.status)
-  finally:
-    if p is not None:
-      try:
-        os.kill(p.pid, signal.SIGKILL)
-      except ProcessLookupError:
-        pass
-    shutil.rmtree(tmp_dir)
-
diff --git a/dev-tools/src/main/resources/ant/integration-tests.xml b/dev-tools/src/main/resources/ant/integration-tests.xml
index 4cd6b4728f8..8378fff5012 100644
--- a/dev-tools/src/main/resources/ant/integration-tests.xml
+++ b/dev-tools/src/main/resources/ant/integration-tests.xml
@@ -15,13 +15,6 @@
   <!-- name of our cluster, maybe needs changing -->
   <property name="integ.cluster.name" value="prepare_release"/>
 
-  <!-- arguments passed to elasticsearch when running -->
-  <property name="integ.args"
-            value="-Des.node.name=smoke_tester -Des.cluster.name=${integ.cluster.name}
-                   -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on
-                   -Des.http.port=${integ.http.port} -Des.transport.tcp.port=${integ.transport.port}
-                   -Des.script.indexed=on -Des.pidfile=${integ.pidfile} -Des.repositories.url.allowed_urls=http://snapshot.test*"/>
-
   <!-- runs an OS script -->
   <macrodef name="run-script">
       <attribute name="script"/>
@@ -37,14 +30,19 @@
       <!-- create a temp CWD, to enforce that commands don't rely on CWD -->
       <mkdir dir="${integ.temp}"/>
 
-      <exec executable="cmd" osfamily="winnt" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}">
+      <!-- print commands we run -->
+      <local name="script.base"/>
+      <basename file="@{script}" property="script.base"/>
+      <echo>execute: ${script.base} @{args}</echo>
+
+      <exec executable="cmd" osfamily="winnt" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}" taskname="${script.base}">
         <arg value="/c"/>
         <arg value="@{script}.bat"/>
         <arg line="@{args}"/>
         <nested/>
       </exec>
 
-      <exec executable="sh" osfamily="unix" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}">
+      <exec executable="sh" osfamily="unix" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}" taskname="${script.base}">
         <arg value="@{script}"/>
         <arg line="@{args}"/>
         <nested/>
@@ -119,31 +117,60 @@
   <!-- start elasticsearch and wait until its ready -->
   <macrodef name="startup-elasticsearch">
       <attribute name="home" default="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
+      <attribute name="spawn" default="true"/>
       <attribute name="args" default="${integ.args}"/>
+      <attribute name="es.cluster.name" default="${integ.cluster.name}"/>
+      <attribute name="es.http.port" default="${integ.http.port}"/>
+      <attribute name="es.transport.tcp.port" default="${integ.transport.port}"/>
+      <attribute name="es.pidfile" default="${integ.pidfile}"/>
+      <attribute name="additional.args" default=""/>
+      <element name="nested.args" optional="true"/>
     <sequential>
+
+      <!-- build args to pass to es -->
+      <local name="integ.args"/>
+      <property name="integ.args" value="
+-Des.cluster.name=@{es.cluster.name}
+-Des.http.port=@{es.http.port}
+-Des.transport.tcp.port=@{es.transport.tcp.port}
+-Des.pidfile=@{es.pidfile}
+-Des.path.repo=@{home}/repo
+-Des.discovery.zen.ping.multicast.enabled=false
+-Des.script.inline=on
+-Des.script.indexed=on
+-Des.repositories.url.allowed_urls=http://snapshot.test*
+@{additional.args}"
+      />
+
+      <!-- run bin/elasticsearch with args -->
       <echo>Starting up external cluster...</echo>
-      <run-script script="@{home}/bin/elasticsearch" spawn="true"
-                  args="@{args} -Des.path.repo=@{home}/repo"/>
+      <run-script script="@{home}/bin/elasticsearch" 
+                  spawn="@{spawn}"
+                  args="${integ.args}">
+        <nested>
+          <nested.args/>
+        </nested>
+      </run-script>
 
       <local name="failed.to.start"/>
       <waitfor maxwait="30" maxwaitunit="second"
                checkevery="500" checkeveryunit="millisecond"
                timeoutproperty="failed.to.start">
-        <http url="http://127.0.0.1:${integ.http.port}"/>
+        <http url="http://127.0.0.1:@{es.http.port}"/>
       </waitfor>
 
       <!-- best effort, print console log. useful if it fails especially -->
       <local name="log.contents"/>
-      <loadfile srcFile="@{home}/logs/${integ.cluster.name}.log"
+      <loadfile srcFile="@{home}/logs/@{es.cluster.name}.log"
                 property="log.contents"
                 failonerror="false"/>
-      <echo message="${log.contents}"/>
+      <echo message="${log.contents}" taskname="elasticsearch"/>
 
       <fail message="ES instance did not start" if="failed.to.start"/>
 
       <local name="integ.pid"/>
       <extract-pid property="integ.pid"/>
-      <echo>External cluster started PID ${integ.pid}</echo>
+      <echo>External cluster started PID @{es.pidfile}</echo>
     </sequential>
   </macrodef>
 
@@ -162,12 +189,11 @@
       <unzip src="${project.build.directory}/releases/${project.artifactId}-${project.version}.zip" dest="${integ.scratch}"/>
       <local name="home"/>
       <property name="home" location="${integ.scratch}/${project.artifactId}-${elasticsearch.version}"/>
-      <run-script script="${home}/bin/elasticsearch" spawn="false"
-                  args="${integ.args} -Des.path.repo=${home}/repo">
-        <nested>
+      <startup-elasticsearch spawn="false" home="${home}">
+        <nested.args>
           <env key="JAVA_OPTS" value="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000"/>
-        </nested>
-      </run-script>
+        </nested.args>
+      </startup-elasticsearch>
   </target>
 
   <!-- unzip core release artifact, install plugin, then start ES -->
diff --git a/pom.xml b/pom.xml
index 188ed85015e..1e7ca1a6c75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1463,5 +1463,6 @@ org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UT
         <module>core</module>
         <module>distribution</module>
         <module>plugins</module>
+        <module>qa</module>
     </modules>
 </project>
diff --git a/qa/pom.xml b/qa/pom.xml
new file mode 100644
index 00000000000..a5d68c1beaf
--- /dev/null
+++ b/qa/pom.xml
@@ -0,0 +1,320 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.elasticsearch.qa</groupId>
+    <artifactId>elasticsearch-qa</artifactId>
+    <version>2.0.0-beta1-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>QA: Parent POM</name>
+    <inceptionYear>2015</inceptionYear>
+
+    <parent>
+        <groupId>org.elasticsearch</groupId>
+        <artifactId>elasticsearch-parent</artifactId>
+        <version>2.0.0-beta1-SNAPSHOT</version>
+    </parent>
+
+    <properties>
+    </properties>
+
+    <dependencies>
+        <!-- elasticsearch and its test framework -->
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Provided dependencies by elasticsearch itself  -->
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-backward-codecs</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-analyzers-common</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-queries</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-memory</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-highlighter</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-queryparser</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-suggest</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-join</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-spatial</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-expressions</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.spatial4j</groupId>
+            <artifactId>spatial4j</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.vividsolutions</groupId>
+            <artifactId>jts</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.spullara.mustache.java</groupId>
+            <artifactId>compiler</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.carrotsearch</groupId>
+            <artifactId>hppc</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.joda</groupId>
+            <artifactId>joda-convert</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-smile</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-cbor</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.ning</groupId>
+            <artifactId>compress-lzf</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.tdunning</groupId>
+            <artifactId>t-digest</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.codehaus.groovy</groupId>
+            <artifactId>groovy-all</artifactId>
+            <classifier>indy</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>apache-log4j-extras</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.java.dev.jna</groupId>
+            <artifactId>jna</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- Required by the REST test framework -->
+        <!-- TODO: remove this dependency when we will have a REST Test module -->
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <!-- typical layout -->
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>**/*.properties</include>
+                </includes>
+            </resource>
+        </resources>
+
+        <testResources>
+            <testResource>
+                <directory>src/test/java</directory>
+                <includes>
+                    <include>**/*.json</include>
+                    <include>**/*.txt</include>
+                </includes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <excludes>
+                    <exclude>elasticsearch.yml</exclude>
+                    <exclude>**/*.properties</exclude>
+                </excludes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>elasticsearch.yml</include>
+                    <include>**/*.properties</include>
+                </includes>
+            </testResource>
+            <!-- REST API specification and test suites -->
+            <testResource>
+                <directory>${project.basedir}/rest-api-spec</directory>
+                <filtering>true</filtering>
+                <targetPath>rest-api-spec</targetPath>
+                <includes>
+                    <include>api/*.json</include>
+                    <include>test/**/*.yaml</include>
+                </includes>
+            </testResource>
+            <!-- REST API specifications copied from main Elasticsearch specs
+                 because they are required to execute the REST tests in here  -->
+            <testResource>
+                <directory>${elasticsearch.tools.directory}/rest-api-spec</directory>
+                <targetPath>rest-api-spec</targetPath>
+                <includes>
+                    <!-- required by the test framework -->
+                    <include>api/info.json</include>
+                    <include>api/cluster.health.json</include>
+                    <include>api/cluster.state.json</include>
+                    <!-- used in plugin REST tests -->
+                    <include>api/index.json</include>
+                    <include>api/get.json</include>
+                    <include>api/update.json</include>
+                    <include>api/search.json</include>
+                    <include>api/indices.analyze.json</include>
+                    <include>api/indices.create.json</include>
+                    <include>api/indices.refresh.json</include>
+                    <include>api/nodes.info.json</include>
+                    <include>api/count.json</include>
+                </includes>
+            </testResource>
+            <!-- shared test resources like log4j.properties -->
+            <testResource>
+                <directory>${elasticsearch.tools.directory}/shared-test-resources</directory>
+                <filtering>false</filtering>
+            </testResource>
+        </testResources>
+
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>com.carrotsearch.randomizedtesting</groupId>
+                    <artifactId>junit4-maven-plugin</artifactId>
+                    <executions>
+                        <execution>
+                            <id>integ-tests</id>
+                            <configuration>
+                                <!-- currently only 1 cpu works, because integ tests don't make "unique" test directories? -->
+                                <parallelism>1</parallelism>
+                                <systemProperties>
+                                    <!-- use external cluster -->
+                                    <tests.cluster>127.0.0.1:${integ.transport.port}</tests.cluster>
+                                </systemProperties>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
+    <modules>
+        <module>smoke-test-plugins</module>
+    </modules>
+</project>
diff --git a/qa/smoke-test-plugins/integration-tests.xml b/qa/smoke-test-plugins/integration-tests.xml
new file mode 100644
index 00000000000..d00d8c37bab
--- /dev/null
+++ b/qa/smoke-test-plugins/integration-tests.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<project name="smoke-test-plugins"
+         xmlns:ac="antlib:net.sf.antcontrib">
+
+  <import file="${elasticsearch.integ.antfile.default}"/>
+
+  <macrodef name="convert-plugin-name">
+      <attribute name="file"/>
+      <attribute name="outputproperty"/>
+    <sequential>
+      <local name="file.base"/>
+      <basename file="@{file}" property="file.base"/>
+      <filter-property src="file.base" dest="@{outputproperty}">
+        <chain>
+          <replaceregex pattern="^elasticsearch-" replace=""/>
+          <replacestring from="-${elasticsearch.version}.zip" to=""/>
+        </chain>
+      </filter-property>
+    </sequential>
+  </macrodef>
+
+  <target name="start-external-cluster-with-plugins" depends="setup-workspace" unless="${shouldskip}">
+    <fail message="Expected ${expected.plugin.count} dependencies, are plugins missing from this pom.xml?">
+      <condition>
+        <resourcecount count="${expected.plugin.count}" when="ne">
+          <fileset dir="${integ.deps}/plugins"/>
+        </resourcecount>
+      </condition>
+    </fail>
+    <ac:for param="file">
+      <path>
+        <fileset dir="${integ.deps}/plugins"/>
+      </path>
+      <sequential>
+        <local name="plugin.name"/>
+        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
+        <install-plugin name="${plugin.name}" file="@{file}"/>
+      </sequential>
+    </ac:for>
+    <startup-elasticsearch/>
+  </target>
+</project>
diff --git a/qa/smoke-test-plugins/pom.xml b/qa/smoke-test-plugins/pom.xml
new file mode 100644
index 00000000000..417dcdc767a
--- /dev/null
+++ b/qa/smoke-test-plugins/pom.xml
@@ -0,0 +1,238 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.elasticsearch.qa</groupId>
+    <artifactId>elasticsearch-qa</artifactId>
+    <version>2.0.0-beta1-SNAPSHOT</version>
+  </parent>
+
+  <!-- 
+    This test unzips elasticsearch, installs each plugin,
+    starts elasticsearch, verifies loaded plugin count.
+
+    "expected plugin count" is computed from plugins/, 
+    currently any folder having a pom.xml file.
+ 
+    our yaml file uses property filtering to populate it.
+  -->
+
+  <artifactId>smoke-test-plugins</artifactId>
+  <name>QA: Smoke Test Plugins</name>
+  <description>Loads up all of our plugins</description>
+
+  <properties>
+    <skip.unit.tests>true</skip.unit.tests>
+    <elasticsearch.integ.antfile>${project.basedir}/integration-tests.xml</elasticsearch.integ.antfile>
+    <tests.rest.suite>smoke_test_plugins</tests.rest.suite>
+    <tests.rest.load_packaged>false</tests.rest.load_packaged>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+         <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-dependency-plugin</artifactId>
+         <executions>
+           <execution>
+             <id>integ-setup-dependencies</id>
+             <phase>pre-integration-test</phase>
+             <goals>
+               <goal>copy</goal>
+             </goals>
+             <configuration>
+               <!-- <skip>${skip.integ.tests}</skip> -->
+               <useBaseVersion>true</useBaseVersion>
+               <outputDirectory>${integ.deps}/plugins</outputDirectory>
+
+               <artifactItems>
+                 <!-- elasticsearch distribution -->
+                 <artifactItem>
+                   <groupId>org.elasticsearch.distribution.zip</groupId>
+                   <artifactId>elasticsearch</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                   <outputDirectory>${integ.deps}</outputDirectory>
+                 </artifactItem>
+
+                 <!-- plugins -->
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-kuromoji</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-smartcn</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-stempel</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-phonetic</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-icu</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-gce</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-azure</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-aws</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-site-example</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-lang-python</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-lang-javascript</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-delete-by-query</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+               </artifactItems>
+             </configuration>
+           </execution>
+         </executions>
+      </plugin>
+      <!-- integration tests -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>count-expected-plugins</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <property name="plugins.dir" location="${project.basedir}/../../plugins"/>
+                <resourcecount property="expected.plugin.count">
+                  <fileset dir="${plugins.dir}" includes="*/pom.xml"/>
+                </resourcecount>
+                <echo>Found ${expected.plugin.count} plugins in ${plugins.dir}</echo>
+              </target>
+              <exportAntProperties>true</exportAntProperties>
+            </configuration>
+          </execution>
+          <!-- start up external cluster -->
+          <execution>
+            <id>integ-setup</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster-with-plugins">
+                  <property name="plugins.dir" value="${plugins.dir}"/>
+                  <property name="expected.plugin.count" value="${expected.plugin.count}"/>
+                </ant>
+              </target>
+            </configuration>
+          </execution>
+          <!-- shut down external cluster -->
+          <execution>
+            <id>integ-teardown</id>
+            <phase>post-integration-test</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+        <dependencies>
+          <dependency>
+            <groupId>ant-contrib</groupId>
+            <artifactId>ant-contrib</artifactId>
+            <version>1.0b3</version>
+            <exclusions>
+              <exclusion>
+                <groupId>ant</groupId>
+                <artifactId>ant</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+          <dependency>
+            <groupId>org.apache.ant</groupId>
+            <artifactId>ant-nodeps</artifactId>
+            <version>1.8.1</version>
+          </dependency>
+        </dependencies>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
diff --git a/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
new file mode 100644
index 00000000000..dbb09225fce
--- /dev/null
+++ b/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for smoke testing plugins
+#
+"Correct Plugin Count":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - length:  { nodes.$master.plugins: ${expected.plugin.count}  }
diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java
new file mode 100644
index 00000000000..6e0243b2a04
--- /dev/null
+++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+public class SmokeTestPluginsIT extends ESRestTestCase {
+
+    public SmokeTestPluginsIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
+