Get plugin smoketester running in jenkins.

We have a smoke_test_plugins.py, but its a bit slow, not integrated
into our build, etc.

I converted this into an integration test. It is definitely uglier
but more robust and fast (e.g. 20 seconds time to verify).

Also there is refactoring of existing integ tests logic, like printing
out commands we execute and stuff

dev-tools/smoke_test_plugins.py

@@ -1,172 +0,0 @@
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance  with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on 
-# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-# either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-import datetime
-import traceback
-import json
-import os
-import shutil
-import signal
-import socket
-import subprocess
-import tempfile
-import threading
-import time
-
-from http.client import HTTPConnection
-
-LOG = os.environ.get('ES_SMOKE_TEST_PLUGINS_LOG', '/tmp/elasticsearch_smoke_test_plugins.log')
-
-print('Logging to %s' % LOG)
-
-if os.path.exists(LOG):
-  raise RuntimeError('please remove old log %s first' % LOG)
-
-try:
-  JAVA_HOME = os.environ['JAVA7_HOME']
-except KeyError:
-  try:
-    JAVA_HOME = os.environ['JAVA_HOME']
-  except KeyError:
-    raise RuntimeError("""
-    Please set JAVA_HOME in the env before running release tool
-    On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.7*'`""")
-
-JAVA_ENV = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (JAVA_HOME, JAVA_HOME, JAVA_HOME)
-
-try:
-  # make sure mvn3 is used if mvn3 is available
-  # some systems use maven 2 as default
-  subprocess.check_output('mvn3 --version', shell=True, stderr=subprocess.STDOUT)
-  MVN = 'mvn3'
-except subprocess.CalledProcessError:
-  MVN = 'mvn'
-
-def log(msg):
-  f = open(LOG, mode='ab')
-  f.write(('\n'+msg).encode('utf-8'))
-  f.close()
-
-def run(command, quiet=False):
-  log('%s: RUN: %s\n' % (datetime.datetime.now(), command))
-  if os.system('%s >> %s 2>&1' % (command, LOG)):
-    msg = '    FAILED: %s [see log %s]' % (command, LOG)
-    if not quiet:
-      print(msg)
-    raise RuntimeError(msg)
-
-def readServerOutput(p, startupEvent, failureEvent):
-  try:
-    while True:
-      line = p.stdout.readline()
-      if len(line) == 0:
-        p.poll()
-        if not startupEvent.isSet():
-          failureEvent.set()
-          startupEvent.set()
-        print('ES: **process exit**\n')
-        break
-      line = line.decode('utf-8').rstrip()
-      if line.endswith('started') and not startupEvent.isSet():
-        startupEvent.set()
-      print('ES: %s' % line)
-  except:
-    print()
-    print('Exception reading Elasticsearch output:')
-    traceback.print_exc()
-    failureEvent.set()
-    startupEvent.set()
-
-if __name__ == '__main__':
-  print('Build release bits...')
-
-  run('%s; %s clean package -DskipTests' % (JAVA_ENV, MVN))
-
-  for f in os.listdir('distribution/tar/target/releases/'):
-    if f.endswith('.tar.gz'):
-      artifact = f
-      break
-  else:
-    raise RuntimeError('could not find elasticsearch release under distribution/tar/target/releases/')
-  
-  tmp_dir = tempfile.mkdtemp()
-  p = None
-  try:
-    # Extract artifact:
-    run('tar -xzf distribution/tar/target/releases/%s -C %s' % (artifact, tmp_dir))
-    es_install_dir = os.path.join(tmp_dir, artifact[:-7])
-    es_plugin_path = os.path.join(es_install_dir, 'bin/plugin')
-    installed_plugin_names = set()
-    print('Find plugins:')
-    for name in os.listdir('plugins'):
-      if name not in ('target', 'pom.xml'):
-        url = 'file://%s/plugins/%s/target/releases/elasticsearch-%s-2.0.0-beta1-SNAPSHOT.zip' % (os.path.abspath('.'), name, name)
-        print('  install plugin %s...' % name)
-        run('%s; %s install %s --url %s' % (JAVA_ENV, es_plugin_path, name, url))
-        installed_plugin_names.add(name)
-
-    print('Start Elasticsearch')
-
-    env = os.environ.copy()
-    env['JAVA_HOME'] = JAVA_HOME
-    env['PATH'] = '%s/bin:%s' % (JAVA_HOME, env['PATH'])
-    env['JAVA_CMD'] = '%s/bin/java' % JAVA_HOME
-    
-    startupEvent = threading.Event()
-    failureEvent = threading.Event()
-    p = subprocess.Popen(('%s/bin/elasticsearch' % es_install_dir,
-                          '-Des.node.name=smoke_tester',
-                          '-Des.cluster.name=smoke_tester_cluster'
-                          '-Des.discovery.zen.ping.multicast.enabled=false',
-                          '-Des.logger.level=debug',
-                          '-Des.script.inline=on',
-                          '-Des.script.indexed=on'),
-                         stdout = subprocess.PIPE,
-                         stderr = subprocess.STDOUT,
-                         env = env)
-    thread = threading.Thread(target=readServerOutput, args=(p, startupEvent, failureEvent))
-    thread.setDaemon(True)
-    thread.start()
-
-    startupEvent.wait(1200)
-    if failureEvent.isSet():
-      raise RuntimeError('ES failed to start')
-
-    print('Confirm plugins are installed')
-    conn = HTTPConnection('127.0.0.1', 9200, 20);
-    conn.request('GET', '/_nodes?plugin=true&pretty=true')
-    res = conn.getresponse()
-    if res.status == 200:
-      nodes = json.loads(res.read().decode("utf-8"))['nodes']
-      for _, node in nodes.items():
-        node_plugins = node['plugins']
-        for node_plugin in node_plugins:
-          plugin_name = node_plugin['name']
-          if plugin_name not in installed_plugin_names:
-            raise RuntimeError('Unexpeced plugin %s' % plugin_name)
-          installed_plugin_names.remove(plugin_name)
-      if len(installed_plugin_names) > 0:
-        raise RuntimeError('Plugins not loaded %s' % installed_plugin_names)
-    else:
-      raise RuntimeError('Expected HTTP 200 but got %s' % res.status)
-  finally:
-    if p is not None:
-      try:
-        os.kill(p.pid, signal.SIGKILL)
-      except ProcessLookupError:
-        pass
-    shutil.rmtree(tmp_dir)
-

dev-tools/src/main/resources/ant/integration-tests.xml

@@ -15,13 +15,6 @@
   <!-- name of our cluster, maybe needs changing -->
   <property name="integ.cluster.name" value="prepare_release"/>
 
-  <!-- arguments passed to elasticsearch when running -->
-  <property name="integ.args"
-            value="-Des.node.name=smoke_tester -Des.cluster.name=${integ.cluster.name}
-                   -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on
-                   -Des.http.port=${integ.http.port} -Des.transport.tcp.port=${integ.transport.port}
-                   -Des.script.indexed=on -Des.pidfile=${integ.pidfile} -Des.repositories.url.allowed_urls=http://snapshot.test*"/>
-
   <!-- runs an OS script -->
   <macrodef name="run-script">
       <attribute name="script"/>
@@ -37,14 +30,19 @@
       <!-- create a temp CWD, to enforce that commands don't rely on CWD -->
       <mkdir dir="${integ.temp}"/>
 
-      <exec executable="cmd" osfamily="winnt" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}">
+      <!-- print commands we run -->
+      <local name="script.base"/>
+      <basename file="@{script}" property="script.base"/>
+      <echo>execute: ${script.base} @{args}</echo>
+
+      <exec executable="cmd" osfamily="winnt" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}" taskname="${script.base}">
         <arg value="/c"/>
         <arg value="@{script}.bat"/>
         <arg line="@{args}"/>
         <nested/>
       </exec>
 
-      <exec executable="sh" osfamily="unix" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}">
+      <exec executable="sh" osfamily="unix" dir="${integ.temp}" failonerror="${failonerror}" spawn="@{spawn}" taskname="${script.base}">
         <arg value="@{script}"/>
         <arg line="@{args}"/>
         <nested/>
@@ -119,31 +117,60 @@
   <!-- start elasticsearch and wait until its ready -->
   <macrodef name="startup-elasticsearch">
       <attribute name="home" default="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
+      <attribute name="spawn" default="true"/>
       <attribute name="args" default="${integ.args}"/>
+      <attribute name="es.cluster.name" default="${integ.cluster.name}"/>
+      <attribute name="es.http.port" default="${integ.http.port}"/>
+      <attribute name="es.transport.tcp.port" default="${integ.transport.port}"/>
+      <attribute name="es.pidfile" default="${integ.pidfile}"/>
+      <attribute name="additional.args" default=""/>
+      <element name="nested.args" optional="true"/>
     <sequential>
+
+      <!-- build args to pass to es -->
+      <local name="integ.args"/>
+      <property name="integ.args" value="
+-Des.cluster.name=@{es.cluster.name}
+-Des.http.port=@{es.http.port}
+-Des.transport.tcp.port=@{es.transport.tcp.port}
+-Des.pidfile=@{es.pidfile}
+-Des.path.repo=@{home}/repo
+-Des.discovery.zen.ping.multicast.enabled=false
+-Des.script.inline=on
+-Des.script.indexed=on
+-Des.repositories.url.allowed_urls=http://snapshot.test*
+@{additional.args}"
+      />
+
+      <!-- run bin/elasticsearch with args -->
       <echo>Starting up external cluster...</echo>
-      <run-script script="@{home}/bin/elasticsearch" spawn="true"
-                  args="@{args} -Des.path.repo=@{home}/repo"/>
+      <run-script script="@{home}/bin/elasticsearch" 
+                  spawn="@{spawn}"
+                  args="${integ.args}">
+        <nested>
+          <nested.args/>
+        </nested>
+      </run-script>
 
       <local name="failed.to.start"/>
       <waitfor maxwait="30" maxwaitunit="second"
                checkevery="500" checkeveryunit="millisecond"
                timeoutproperty="failed.to.start">
-        <http url="http://127.0.0.1:${integ.http.port}"/>
+        <http url="http://127.0.0.1:@{es.http.port}"/>
       </waitfor>
 
       <!-- best effort, print console log. useful if it fails especially -->
       <local name="log.contents"/>
-      <loadfile srcFile="@{home}/logs/${integ.cluster.name}.log"
+      <loadfile srcFile="@{home}/logs/@{es.cluster.name}.log"
                 property="log.contents"
                 failonerror="false"/>
-      <echo message="${log.contents}"/>
+      <echo message="${log.contents}" taskname="elasticsearch"/>
 
       <fail message="ES instance did not start" if="failed.to.start"/>
 
       <local name="integ.pid"/>
       <extract-pid property="integ.pid"/>
-      <echo>External cluster started PID ${integ.pid}</echo>
+      <echo>External cluster started PID @{es.pidfile}</echo>
     </sequential>
   </macrodef>
 
@@ -162,12 +189,11 @@
       <unzip src="${project.build.directory}/releases/${project.artifactId}-${project.version}.zip" dest="${integ.scratch}"/>
       <local name="home"/>
       <property name="home" location="${integ.scratch}/${project.artifactId}-${elasticsearch.version}"/>
-      <run-script script="${home}/bin/elasticsearch" spawn="false"
-                  args="${integ.args} -Des.path.repo=${home}/repo">
-        <nested>
+      <startup-elasticsearch spawn="false" home="${home}">
+        <nested.args>
           <env key="JAVA_OPTS" value="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000"/>
-        </nested>
-      </run-script>
+        </nested.args>
+      </startup-elasticsearch>
   </target>
 
   <!-- unzip core release artifact, install plugin, then start ES -->

pom.xml

@@ -1463,5 +1463,6 @@ org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UT
         <module>core</module>
         <module>distribution</module>
         <module>plugins</module>
+        <module>qa</module>
     </modules>
 </project>

qa/pom.xml

@@ -0,0 +1,320 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.elasticsearch.qa</groupId>
+    <artifactId>elasticsearch-qa</artifactId>
+    <version>2.0.0-beta1-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>QA: Parent POM</name>
+    <inceptionYear>2015</inceptionYear>
+
+    <parent>
+        <groupId>org.elasticsearch</groupId>
+        <artifactId>elasticsearch-parent</artifactId>
+        <version>2.0.0-beta1-SNAPSHOT</version>
+    </parent>
+
+    <properties>
+    </properties>
+
+    <dependencies>
+        <!-- elasticsearch and its test framework -->
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Provided dependencies by elasticsearch itself  -->
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-backward-codecs</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-analyzers-common</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-queries</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-memory</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-highlighter</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-queryparser</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-suggest</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-join</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-spatial</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-expressions</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.spatial4j</groupId>
+            <artifactId>spatial4j</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.vividsolutions</groupId>
+            <artifactId>jts</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.spullara.mustache.java</groupId>
+            <artifactId>compiler</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.carrotsearch</groupId>
+            <artifactId>hppc</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.joda</groupId>
+            <artifactId>joda-convert</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-smile</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-cbor</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.ning</groupId>
+            <artifactId>compress-lzf</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.tdunning</groupId>
+            <artifactId>t-digest</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.codehaus.groovy</groupId>
+            <artifactId>groovy-all</artifactId>
+            <classifier>indy</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>apache-log4j-extras</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.java.dev.jna</groupId>
+            <artifactId>jna</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- Required by the REST test framework -->
+        <!-- TODO: remove this dependency when we will have a REST Test module -->
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <!-- typical layout -->
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>**/*.properties</include>
+                </includes>
+            </resource>
+        </resources>
+
+        <testResources>
+            <testResource>
+                <directory>src/test/java</directory>
+                <includes>
+                    <include>**/*.json</include>
+                    <include>**/*.txt</include>
+                </includes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <excludes>
+                    <exclude>elasticsearch.yml</exclude>
+                    <exclude>**/*.properties</exclude>
+                </excludes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>elasticsearch.yml</include>
+                    <include>**/*.properties</include>
+                </includes>
+            </testResource>
+            <!-- REST API specification and test suites -->
+            <testResource>
+                <directory>${project.basedir}/rest-api-spec</directory>
+                <filtering>true</filtering>
+                <targetPath>rest-api-spec</targetPath>
+                <includes>
+                    <include>api/*.json</include>
+                    <include>test/**/*.yaml</include>
+                </includes>
+            </testResource>
+            <!-- REST API specifications copied from main Elasticsearch specs
+                 because they are required to execute the REST tests in here  -->
+            <testResource>
+                <directory>${elasticsearch.tools.directory}/rest-api-spec</directory>
+                <targetPath>rest-api-spec</targetPath>
+                <includes>
+                    <!-- required by the test framework -->
+                    <include>api/info.json</include>
+                    <include>api/cluster.health.json</include>
+                    <include>api/cluster.state.json</include>
+                    <!-- used in plugin REST tests -->
+                    <include>api/index.json</include>
+                    <include>api/get.json</include>
+                    <include>api/update.json</include>
+                    <include>api/search.json</include>
+                    <include>api/indices.analyze.json</include>
+                    <include>api/indices.create.json</include>
+                    <include>api/indices.refresh.json</include>
+                    <include>api/nodes.info.json</include>
+                    <include>api/count.json</include>
+                </includes>
+            </testResource>
+            <!-- shared test resources like log4j.properties -->
+            <testResource>
+                <directory>${elasticsearch.tools.directory}/shared-test-resources</directory>
+                <filtering>false</filtering>
+            </testResource>
+        </testResources>
+
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>com.carrotsearch.randomizedtesting</groupId>
+                    <artifactId>junit4-maven-plugin</artifactId>
+                    <executions>
+                        <execution>
+                            <id>integ-tests</id>
+                            <configuration>
+                                <!-- currently only 1 cpu works, because integ tests don't make "unique" test directories? -->
+                                <parallelism>1</parallelism>
+                                <systemProperties>
+                                    <!-- use external cluster -->
+                                    <tests.cluster>127.0.0.1:${integ.transport.port}</tests.cluster>
+                                </systemProperties>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
+    <modules>
+        <module>smoke-test-plugins</module>
+    </modules>
+</project>

qa/smoke-test-plugins/integration-tests.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<project name="smoke-test-plugins"
+         xmlns:ac="antlib:net.sf.antcontrib">
+
+  <import file="${elasticsearch.integ.antfile.default}"/>
+
+  <macrodef name="convert-plugin-name">
+      <attribute name="file"/>
+      <attribute name="outputproperty"/>
+    <sequential>
+      <local name="file.base"/>
+      <basename file="@{file}" property="file.base"/>
+      <filter-property src="file.base" dest="@{outputproperty}">
+        <chain>
+          <replaceregex pattern="^elasticsearch-" replace=""/>
+          <replacestring from="-${elasticsearch.version}.zip" to=""/>
+        </chain>
+      </filter-property>
+    </sequential>
+  </macrodef>
+
+  <target name="start-external-cluster-with-plugins" depends="setup-workspace" unless="${shouldskip}">
+    <fail message="Expected ${expected.plugin.count} dependencies, are plugins missing from this pom.xml?">
+      <condition>
+        <resourcecount count="${expected.plugin.count}" when="ne">
+          <fileset dir="${integ.deps}/plugins"/>
+        </resourcecount>
+      </condition>
+    </fail>
+    <ac:for param="file">
+      <path>
+        <fileset dir="${integ.deps}/plugins"/>
+      </path>
+      <sequential>
+        <local name="plugin.name"/>
+        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
+        <install-plugin name="${plugin.name}" file="@{file}"/>
+      </sequential>
+    </ac:for>
+    <startup-elasticsearch/>
+  </target>
+</project>

qa/smoke-test-plugins/pom.xml

@@ -0,0 +1,238 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.elasticsearch.qa</groupId>
+    <artifactId>elasticsearch-qa</artifactId>
+    <version>2.0.0-beta1-SNAPSHOT</version>
+  </parent>
+
+  <!-- 
+    This test unzips elasticsearch, installs each plugin,
+    starts elasticsearch, verifies loaded plugin count.
+
+    "expected plugin count" is computed from plugins/, 
+    currently any folder having a pom.xml file.
+ 
+    our yaml file uses property filtering to populate it.
+  -->
+
+  <artifactId>smoke-test-plugins</artifactId>
+  <name>QA: Smoke Test Plugins</name>
+  <description>Loads up all of our plugins</description>
+
+  <properties>
+    <skip.unit.tests>true</skip.unit.tests>
+    <elasticsearch.integ.antfile>${project.basedir}/integration-tests.xml</elasticsearch.integ.antfile>
+    <tests.rest.suite>smoke_test_plugins</tests.rest.suite>
+    <tests.rest.load_packaged>false</tests.rest.load_packaged>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+         <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-dependency-plugin</artifactId>
+         <executions>
+           <execution>
+             <id>integ-setup-dependencies</id>
+             <phase>pre-integration-test</phase>
+             <goals>
+               <goal>copy</goal>
+             </goals>
+             <configuration>
+               <!-- <skip>${skip.integ.tests}</skip> -->
+               <useBaseVersion>true</useBaseVersion>
+               <outputDirectory>${integ.deps}/plugins</outputDirectory>
+
+               <artifactItems>
+                 <!-- elasticsearch distribution -->
+                 <artifactItem>
+                   <groupId>org.elasticsearch.distribution.zip</groupId>
+                   <artifactId>elasticsearch</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                   <outputDirectory>${integ.deps}</outputDirectory>
+                 </artifactItem>
+
+                 <!-- plugins -->
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-kuromoji</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-smartcn</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-stempel</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-phonetic</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-analysis-icu</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-gce</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-azure</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-cloud-aws</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-site-example</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-lang-python</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-lang-javascript</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+
+                 <artifactItem>
+                   <groupId>org.elasticsearch.plugin</groupId>
+                   <artifactId>elasticsearch-delete-by-query</artifactId>
+                   <version>${elasticsearch.version}</version>
+                   <type>zip</type>
+                   <overWrite>true</overWrite>
+                 </artifactItem>
+               </artifactItems>
+             </configuration>
+           </execution>
+         </executions>
+      </plugin>
+      <!-- integration tests -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>count-expected-plugins</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <property name="plugins.dir" location="${project.basedir}/../../plugins"/>
+                <resourcecount property="expected.plugin.count">
+                  <fileset dir="${plugins.dir}" includes="*/pom.xml"/>
+                </resourcecount>
+                <echo>Found ${expected.plugin.count} plugins in ${plugins.dir}</echo>
+              </target>
+              <exportAntProperties>true</exportAntProperties>
+            </configuration>
+          </execution>
+          <!-- start up external cluster -->
+          <execution>
+            <id>integ-setup</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster-with-plugins">
+                  <property name="plugins.dir" value="${plugins.dir}"/>
+                  <property name="expected.plugin.count" value="${expected.plugin.count}"/>
+                </ant>
+              </target>
+            </configuration>
+          </execution>
+          <!-- shut down external cluster -->
+          <execution>
+            <id>integ-teardown</id>
+            <phase>post-integration-test</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+        <dependencies>
+          <dependency>
+            <groupId>ant-contrib</groupId>
+            <artifactId>ant-contrib</artifactId>
+            <version>1.0b3</version>
+            <exclusions>
+              <exclusion>
+                <groupId>ant</groupId>
+                <artifactId>ant</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+          <dependency>
+            <groupId>org.apache.ant</groupId>
+            <artifactId>ant-nodeps</artifactId>
+            <version>1.8.1</version>
+          </dependency>
+        </dependencies>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml

@@ -0,0 +1,13 @@
+# Integration tests for smoke testing plugins
+#
+"Correct Plugin Count":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - length:  { nodes.$master.plugins: ${expected.plugin.count}  }

qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+public class SmokeTestPluginsIT extends ESRestTestCase {
+
+    public SmokeTestPluginsIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
+