Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389)

* Remove deprecation warnings to prepare for Gradle 5

Gradle replaced `project.sourceSets.main.output.classesDir` of type
`File` with `project.sourceSets.main.output.classesDirs` of type
`FileCollection`
(see [SourceSetOutput](https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/java/org/gradle/api/tasks/SourceSetOutput.java))
Build output is now stored on a per language folder.

There are a few places where we use that, here's these and how it's
fixed:

- Randomized Test execution
    - look in all test folders ( pass the multi dir configuration to the
    ant runner )
    - DRY the task configuration by introducing `basedOn` for
      `RandomizedTestingTask` DSL
- Extend the naming convention test to support passing in multiple
  directories
- Fix the standalon test plugin, the dires were not passed trough,
  checked with a debuger and the statement had no affect due to a
  missing `=`.

Closes #30354

* Only check Java tests, PR feedback

- Name checker was ran for Groovy tests that don't adhere to the same
  convections causing the check to fail
- implement PR feedback

* Replace `add` with `addAll`

This worked because the list is passed to `project.files` that does the
right thing.

* Revert "Only check Java tests, PR feedback"

This reverts commit 9bd9389875d8b88aadb50df57a45cd0d2b073241.

* Remove `basedOn` helper

* Bring some changes back

Previus revert accidentally reverted too much

* Fix negation

* add back public

* revert name check changes

* Revert "revert name check changes"

This reverts commit a2800c0b363168339ea65e2a79ec8256e5883e6d.

* Pass all dirs to name check

Only run on Java for build-tools, this is safe because it's a self test.
It needs more work before we could pass in the Groovy classes as well as
these inherit from `GroovyTestCase`

* remove self tests from name check

The self complicates the task setup and disable real checks on
build-tools.
With this change there are no more self tests, and the build-tools tests
adhere to the conventions.
The self test will be replaced by gradle test kit, thus the addition of
the Gradle plugin builder plugin.

* First test to run a Gradle build

* Add tests that replace the name check self test

* Clean up integ test base class

* Always run tests

* Align with test naming conventions

* Make integ. test case inherit from unit test case

The check requires this

* Remove `import static org.junit.Assert.*`

buildSrc/build.gradle

@@ -17,11 +17,13 @@
  * under the License.
  */
 
+
 import java.nio.file.Files
 
-import org.gradle.util.GradleVersion
+plugins {
-
+  id 'java-gradle-plugin'
-apply plugin: 'groovy'
+  id 'groovy'
+}
 
 group = 'org.elasticsearch.gradle'
 
@@ -83,9 +85,10 @@ repositories {
 }
 
 dependencies {
-  compile gradleApi()
   compile localGroovy()
   compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+
   compile("junit:junit:${props.getProperty('junit')}") {
     transitive = false
   }
@@ -97,8 +100,10 @@ dependencies {
   compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
+  testCompile "junit:junit:${props.getProperty('junit')}"
 }
 
+
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
 // Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
@@ -113,14 +118,12 @@ dependencies {
  *****************************************************************************/
 // this will only happen when buildSrc is built on its own during build init
 if (project == rootProject) {
-
   repositories {
     if (System.getProperty("repos.mavenLocal") != null) {
       mavenLocal()
     }
     mavenCentral()
   }
-  test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
 }
 
 /*****************************************************************************
@@ -145,9 +148,6 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
-  // test for elasticsearch.build tries to run with ES...
-  test.enabled = false
-
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 
@@ -158,14 +158,7 @@ if (project != rootProject) {
   }
 
   namingConventions {
-    testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
+    testClass = 'org.elasticsearch.gradle.test.GradleUnitTestCase'
-    integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
+    integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
-
-  task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
-    checkForTestsInMain = true
-    testClass = namingConventions.testClass
-    integTestClass = namingConventions.integTestClass
-  }
-  precommit.dependsOn namingConventionsMain
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -89,7 +89,7 @@ class RandomizedTestingPlugin implements Plugin<Project> {
             description = 'Runs unit tests with the randomized testing framework'
             dependsOn oldTestTask.dependsOn, 'testClasses'
             classpath = oldTestTask.classpath
-            testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
+            testClassesDirs = oldTestTask.project.sourceSets.test.output.classesDirs
         }
 
         // hack so check task depends on custom test

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -6,18 +6,20 @@ import groovy.xml.NamespaceBuilder
 import groovy.xml.NamespaceBuilderSupport
 import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
+import org.apache.tools.ant.Project
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
+import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
-import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.specs.Spec
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
@@ -43,8 +45,8 @@ class RandomizedTestingTask extends DefaultTask {
     @Input
     String parallelism = '1'
 
-    @InputDirectory
+    @Input
-    File testClassesDir
+    FileCollection testClassesDirs
 
     @Optional
     @Input
@@ -220,7 +222,7 @@ class RandomizedTestingTask extends DefaultTask {
                 listener = new DefaultLogger(
                         errorPrintStream: System.err,
                         outputPrintStream: System.out,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             } else {
                 // we want to buffer the info, and emit it if the test fails
                 antLoggingBuffer = new ByteArrayOutputStream()
@@ -228,7 +230,7 @@ class RandomizedTestingTask extends DefaultTask {
                 listener = new DefaultLogger(
                         errorPrintStream: stream,
                         outputPrintStream: stream,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             }
             project.ant.project.addBuildListener(listener)
         }
@@ -251,12 +253,10 @@ class RandomizedTestingTask extends DefaultTask {
                 if (argLine != null) {
                     jvmarg(line: argLine)
                 }
-                fileset(dir: testClassesDir) {
+                testClassesDirs.each { testClassDir ->
-                    for (String includePattern : patternSet.getIncludes()) {
+                    fileset(dir: testClassDir) {
-                        include(name: includePattern)
+                        patternSet.getIncludes().each { include(name: it) }
-                    }
+                        patternSet.getExcludes().each { exclude(name: it) }
-                    for (String excludePattern : patternSet.getExcludes()) {
-                        exclude(name: excludePattern)
                     }
                 }
                 for (Map.Entry<String, Object> prop : systemProperties) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -751,7 +751,7 @@ class BuildPlugin implements Plugin<Project> {
         project.extensions.add('additionalTest', { String name, Closure config ->
             RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
             additionalTest.classpath = test.classpath
-            additionalTest.testClassesDir = test.testClassesDir
+            additionalTest.testClassesDirs = test.testClassesDirs
             additionalTest.configure(commonTestConfig(project))
             additionalTest.configure(config)
             additionalTest.dependsOn(project.tasks.testClasses)

buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy

@@ -1,45 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle
-
-import org.gradle.api.GradleException
-import org.gradle.api.tasks.Exec
-
-/**
- * A wrapper around gradle's Exec task to capture output and log on error.
- */
-class LoggedExec extends Exec {
-
-    protected ByteArrayOutputStream output = new ByteArrayOutputStream()
-
-    LoggedExec() {
-        if (logger.isInfoEnabled() == false) {
-            standardOutput = output
-            errorOutput = output
-            ignoreExitValue = true
-            doLast {
-                if (execResult.exitValue != 0) {
-                    output.toString('UTF-8').eachLine { line -> logger.error(line) }
-                    throw new GradleException("Process '${executable} ${args.join(' ')}' finished with non-zero exit value ${execResult.exitValue}")
-                }
-            }
-        }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.java

@@ -0,0 +1,41 @@
+package org.elasticsearch.gradle;
+
+import groovy.lang.Closure;
+import org.gradle.api.GradleException;
+import org.gradle.api.Task;
+import org.gradle.api.tasks.Exec;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;
+
+/**
+ * A wrapper around gradle's Exec task to capture output and log on error.
+ */
+public class LoggedExec extends Exec {
+
+    protected ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+    public LoggedExec() {
+        if (getLogger().isInfoEnabled() == false) {
+            setStandardOutput(output);
+            setErrorOutput(output);
+            setIgnoreExitValue(true);
+            doLast(new Closure<Void>(this, this) {
+                public void doCall(Task it) throws IOException {
+                    if (getExecResult().getExitValue() != 0) {
+                        for (String line : output.toString("UTF-8").split("\\R")) {
+                            getLogger().error(line);
+                        }
+                        throw new GradleException(
+                                "Process \'" + getExecutable() + " " +
+                                        getArgs().stream().collect(Collectors.joining(" "))+
+                                        "\' finished with non-zero exit value " +
+                                        String.valueOf(getExecResult().getExitValue())
+                        );
+                    }
+                }
+            });
+        }
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy

@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle
-
-/**
- * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
- */
-class VersionProperties {
-    static final Version elasticsearch
-    static final String lucene
-    static final Map<String, String> versions = new HashMap<>()
-    static {
-        Properties props = new Properties()
-        InputStream propsStream = VersionProperties.class.getResourceAsStream('/version.properties')
-        if (propsStream == null) {
-            throw new RuntimeException('/version.properties resource missing')
-        }
-        props.load(propsStream)
-        elasticsearch = Version.fromString(props.getProperty('elasticsearch'))
-        lucene = props.getProperty('lucene')
-        for (String property : props.stringPropertyNames()) {
-            versions.put(property, props.getProperty(property))
-        }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.java

@@ -0,0 +1,50 @@
+package org.elasticsearch.gradle;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
+ */
+public class VersionProperties {
+    public static Version getElasticsearch() {
+        return elasticsearch;
+    }
+
+    public static String getLucene() {
+        return lucene;
+    }
+
+    public static Map<String, String> getVersions() {
+        return versions;
+    }
+
+    private static final Version elasticsearch;
+    private static final String lucene;
+    private static final Map<String, String> versions = new HashMap<String, String>();
+    static {
+        Properties props = getVersionProperties();
+        elasticsearch = Version.fromString(props.getProperty("elasticsearch"));
+        lucene = props.getProperty("lucene");
+        for (String property : props.stringPropertyNames()) {
+            versions.put(property, props.getProperty(property));
+        }
+    }
+
+    private static Properties getVersionProperties() {
+        Properties props = new Properties();
+        InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties");
+        if (propsStream == null) {
+            throw new RuntimeException("/version.properties resource missing");
+        }
+        try {
+            props.load(propsStream);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return props;
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy

@@ -50,11 +50,11 @@ public class LoggerUsageTask extends LoggedExec {
                 List files = []
                 // But only if the source sets that will make them exist
                 if (project.sourceSets.findByName("main")) {
-                    files.add(project.sourceSets.main.output.classesDir)
+                    files.addAll(project.sourceSets.main.output.classesDirs.getFiles())
                     dependsOn project.tasks.classes
                 }
                 if (project.sourceSets.findByName("test")) {
-                    files.add(project.sourceSets.test.output.classesDir)
+                    files.addAll(project.sourceSets.test.output.classesDirs.getFiles())
                     dependsOn project.tasks.testClasses
                 }
                 /* In an extra twist, it isn't good enough that the source set

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy

@@ -1,126 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle.precommit
-
-import org.elasticsearch.gradle.LoggedExec
-import org.elasticsearch.gradle.VersionProperties
-import org.gradle.api.artifacts.Dependency
-import org.gradle.api.file.FileCollection
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.OutputFile
-/**
- * Runs NamingConventionsCheck on a classpath/directory combo to verify that
- * tests are named according to our conventions so they'll be picked up by
- * gradle. Read the Javadoc for NamingConventionsCheck to learn more.
- */
-public class NamingConventionsTask extends LoggedExec {
-    /**
-     * We use a simple "marker" file that we touch when the task succeeds
-     * as the task output. This is compared against the modified time of the
-     * inputs (ie the jars/class files).
-     */
-    @OutputFile
-    File successMarker = new File(project.buildDir, "markers/${this.name}")
-
-    /**
-     * Should we skip the integ tests in disguise tests? Defaults to true because only core names its
-     * integ tests correctly.
-     */
-    @Input
-    boolean skipIntegTestInDisguise = false
-
-    /**
-     * Superclass for all tests.
-     */
-    @Input
-    String testClass = 'org.apache.lucene.util.LuceneTestCase'
-
-    /**
-     * Superclass for all integration tests.
-     */
-    @Input
-    String integTestClass = 'org.elasticsearch.test.ESIntegTestCase'
-
-    /**
-     * Should the test also check the main classpath for test classes instead of
-     * doing the usual checks to the test classpath.
-     */
-    @Input
-    boolean checkForTestsInMain = false;
-
-    public NamingConventionsTask() {
-        // Extra classpath contains the actual test
-        if (false == project.configurations.names.contains('namingConventions')) {
-            project.configurations.create('namingConventions')
-            Dependency buildToolsDep = project.dependencies.add('namingConventions',
-                    "org.elasticsearch.gradle:build-tools:${VersionProperties.elasticsearch}")
-            buildToolsDep.transitive = false // We don't need gradle in the classpath. It conflicts.
-        }
-        FileCollection classpath = project.files(project.configurations.namingConventions,
-                                                 project.sourceSets.test.compileClasspath,
-                                                 project.sourceSets.test.output)
-        dependsOn(classpath)
-        inputs.files(classpath)
-        description = "Tests that test classes aren't misnamed or misplaced"
-        executable = new File(project.runtimeJavaHome, 'bin/java')
-        if (false == checkForTestsInMain) {
-            /* This task is created by default for all subprojects with this
-             * setting and there is no point in running it if the files don't
-             * exist. */
-            onlyIf { project.sourceSets.test.output.classesDir.exists() }
-        }
-
-        /*
-         * We build the arguments in a funny afterEvaluate/doFirst closure so that we can wait for the classpath to be
-         * ready for us. Strangely neither one on their own are good enough.
-         */
-        project.afterEvaluate {
-            doFirst {
-                args('-Djna.nosys=true')
-                args('-cp', classpath.asPath, 'org.elasticsearch.test.NamingConventionsCheck')
-                args('--test-class', testClass)
-                if (skipIntegTestInDisguise) {
-                    args('--skip-integ-tests-in-disguise')
-                } else {
-                    args('--integ-test-class', integTestClass)
-                }
-                /*
-                 * The test framework has classes that fail the checks to validate that the checks fail properly.
-                 * Since these would cause the build to fail we have to ignore them with this parameter. The
-                 * process of ignoring them lets us validate that they were found so this ignore parameter acts
-                 * as the test for the NamingConventionsCheck.
-                 */
-                if (':build-tools'.equals(project.path)) {
-                    args('--self-test')
-                }
-                if (checkForTestsInMain) {
-                    args('--main')
-                    args('--')
-                    args(project.sourceSets.main.output.classesDir.absolutePath)
-                } else {
-                    args('--')
-                    args(project.sourceSets.test.output.classesDir.absolutePath)
-                }
-            }
-        }
-        doLast { successMarker.setText("", 'UTF-8') }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.java

@@ -0,0 +1,185 @@
+package org.elasticsearch.gradle.precommit;
+
+import groovy.lang.Closure;
+import org.codehaus.groovy.runtime.ResourceGroovyMethods;
+import org.elasticsearch.gradle.LoggedExec;
+import org.elasticsearch.test.NamingConventionsCheck;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.tasks.AbstractExecTask;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.SourceSetContainer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Runs NamingConventionsCheck on a classpath/directory combo to verify that
+ * tests are named according to our conventions so they'll be picked up by
+ * gradle. Read the Javadoc for NamingConventionsCheck to learn more.
+ */
+public class NamingConventionsTask extends LoggedExec {
+    public NamingConventionsTask() {
+        setDescription("Tests that test classes aren't misnamed or misplaced");
+        final Project project = getProject();
+
+        SourceSetContainer sourceSets = getJavaSourceSets();
+        final FileCollection classpath = project.files(
+                // This works because the class only depends on one class from junit that will be available from the
+                // tests compile classpath. It's the most straight forward way of telling Java where to find the main
+                // class.
+                NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation().getPath(),
+                // the tests to be loaded
+                checkForTestsInMain ? sourceSets.getByName("main").getRuntimeClasspath() : project.files(),
+                sourceSets.getByName("test").getCompileClasspath(),
+                sourceSets.getByName("test").getOutput()
+        );
+        dependsOn(project.getTasks().matching(it -> "testCompileClasspath".equals(it.getName())));
+        getInputs().files(classpath);
+
+        setExecutable(new File(
+                Objects.requireNonNull(
+                        project.getExtensions().getByType(ExtraPropertiesExtension.class).get("runtimeJavaHome")
+                ).toString(),
+                "bin/java")
+        );
+
+        if (checkForTestsInMain == false) {
+            /* This task is created by default for all subprojects with this
+             * setting and there is no point in running it if the files don't
+             * exist. */
+            onlyIf((unused) -> getExistingClassesDirs().isEmpty() == false);
+        }
+
+        /*
+         * We build the arguments in a funny afterEvaluate/doFirst closure so that we can wait for the classpath to be
+         * ready for us. Strangely neither one on their own are good enough.
+         */
+        project.afterEvaluate(new Closure<Task>(this, this) {
+            public Task doCall(Project it) {
+                return doFirst(new Closure<AbstractExecTask>(NamingConventionsTask.this, NamingConventionsTask.this) {
+                    public AbstractExecTask doCall(Task it) {
+                        args("-Djna.nosys=true");
+                        args("-cp", classpath.getAsPath(), "org.elasticsearch.test.NamingConventionsCheck");
+                        args("--test-class", getTestClass());
+                        if (skipIntegTestInDisguise) {
+                            args("--skip-integ-tests-in-disguise");
+                        } else {
+                            args("--integ-test-class", getIntegTestClass());
+                        }
+                        if (getCheckForTestsInMain()) {
+                            args("--main");
+                            args("--");
+                        } else {
+                            args("--");
+                        }
+                        return args(getExistingClassesDirs().getAsPath());
+                    }
+                });
+            }
+        });
+        doLast(new Closure<Object>(this, this) {
+            public void doCall(Task it) {
+                try {
+                    ResourceGroovyMethods.setText(getSuccessMarker(), "", "UTF-8");
+                } catch (IOException e) {
+                    throw new GradleException("io exception", e);
+                }
+            }
+        });
+    }
+
+    private SourceSetContainer getJavaSourceSets() {
+        return getProject().getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
+    }
+
+    public FileCollection getExistingClassesDirs() {
+        FileCollection classesDirs = getJavaSourceSets().getByName(checkForTestsInMain ? "main" : "test")
+                .getOutput().getClassesDirs();
+        return classesDirs.filter(it -> it.exists());
+    }
+
+    public File getSuccessMarker() {
+        return successMarker;
+    }
+
+    public void setSuccessMarker(File successMarker) {
+        this.successMarker = successMarker;
+    }
+
+    public boolean getSkipIntegTestInDisguise() {
+        return skipIntegTestInDisguise;
+    }
+
+    public boolean isSkipIntegTestInDisguise() {
+        return skipIntegTestInDisguise;
+    }
+
+    public void setSkipIntegTestInDisguise(boolean skipIntegTestInDisguise) {
+        this.skipIntegTestInDisguise = skipIntegTestInDisguise;
+    }
+
+    public String getTestClass() {
+        return testClass;
+    }
+
+    public void setTestClass(String testClass) {
+        this.testClass = testClass;
+    }
+
+    public String getIntegTestClass() {
+        return integTestClass;
+    }
+
+    public void setIntegTestClass(String integTestClass) {
+        this.integTestClass = integTestClass;
+    }
+
+    public boolean getCheckForTestsInMain() {
+        return checkForTestsInMain;
+    }
+
+    public boolean isCheckForTestsInMain() {
+        return checkForTestsInMain;
+    }
+
+    public void setCheckForTestsInMain(boolean checkForTestsInMain) {
+        this.checkForTestsInMain = checkForTestsInMain;
+    }
+
+    /**
+     * We use a simple "marker" file that we touch when the task succeeds
+     * as the task output. This is compared against the modified time of the
+     * inputs (ie the jars/class files).
+     */
+    @OutputFile
+    private File successMarker = new File(getProject().getBuildDir(), "markers/" + this.getName());
+    /**
+     * Should we skip the integ tests in disguise tests? Defaults to true because only core names its
+     * integ tests correctly.
+     */
+    @Input
+    private boolean skipIntegTestInDisguise = false;
+    /**
+     * Superclass for all tests.
+     */
+    @Input
+    private String testClass = "org.apache.lucene.util.LuceneTestCase";
+    /**
+     * Superclass for all integration tests.
+     */
+    @Input
+    private String integTestClass = "org.elasticsearch.test.ESIntegTestCase";
+    /**
+     * Should the test also check the main classpath for test classes instead of
+     * doing the usual checks to the test classpath.
+     */
+    @Input
+    private boolean checkForTestsInMain = false;
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -61,7 +61,7 @@ public class RestIntegTestTask extends DefaultTask {
         clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
         runner.dependsOn(clusterInit)
         runner.classpath = project.sourceSets.test.runtimeClasspath
-        runner.testClassesDir = project.sourceSets.test.output.classesDir
+        runner.testClassesDirs = project.sourceSets.test.output.classesDirs
         clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
 
         // start with the common test configuration

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy

@@ -47,7 +47,7 @@ public class StandaloneTestPlugin implements Plugin<Project> {
         test.configure(BuildPlugin.commonTestConfig(project))
         BuildPlugin.configureCompile(project)
         test.classpath = project.sourceSets.test.runtimeClasspath
-        test.testClassesDir project.sourceSets.test.output.classesDir
+        test.testClassesDirs = project.sourceSets.test.output.classesDirs
         test.mustRunAfter(project.precommit)
         project.check.dependsOn(test)
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy

@@ -22,14 +22,9 @@ import org.apache.commons.io.output.TeeOutputStream
 import org.elasticsearch.gradle.LoggedExec
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Optional
-import org.gradle.api.tasks.TaskAction
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
 
 import javax.inject.Inject
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.locks.Lock
-import java.util.concurrent.locks.ReadWriteLock
-import java.util.concurrent.locks.ReentrantLock
 
 /**
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output

buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test;
 
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Modifier;
 import java.nio.file.FileVisitResult;
@@ -30,6 +31,7 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 /**
  * Checks that all tests in a directory are named according to our naming conventions. This is important because tests that do not follow
@@ -37,19 +39,13 @@ import java.util.Set;
  * a class with a main method so gradle can call it for each project. This has the advantage of allowing gradle to calculate when it is
  * {@code UP-TO-DATE} so it can be skipped if the compiled classes haven't changed. This is useful on large modules for which checking all
  * the modules can be slow.
- *
- * Annoyingly, this cannot be tested using standard unit tests because to do so you'd have to declare classes that violate the rules. That
- * would cause the test fail which would prevent the build from passing. So we have to make a mechanism for removing those test classes. Now
- * that we have such a mechanism it isn't much work to fail the process if we don't detect the offending classes. Thus, the funky
- * {@code --self-test} that is only run in the test:framework project.
  */
 public class NamingConventionsCheck {
     public static void main(String[] args) throws IOException {
         Class<?> testClass = null;
         Class<?> integTestClass = null;
-        Path rootPath = null;
+        String rootPathList = null;
         boolean skipIntegTestsInDisguise = false;
-        boolean selfTest = false;
         boolean checkMainClasses = false;
         for (int i = 0; i < args.length; i++) {
             String arg = args[i];
@@ -63,14 +59,11 @@ public class NamingConventionsCheck {
                 case "--skip-integ-tests-in-disguise":
                     skipIntegTestsInDisguise = true;
                     break;
-                case "--self-test":
-                    selfTest = true;
-                    break;
                 case "--main":
                     checkMainClasses = true;
                     break;
                 case "--":
-                    rootPath = Paths.get(args[++i]);
+                    rootPathList = args[++i];
                     break;
                 default:
                     fail("unsupported argument '" + arg + "'");
@@ -78,44 +71,49 @@ public class NamingConventionsCheck {
         }
 
         NamingConventionsCheck check = new NamingConventionsCheck(testClass, integTestClass);
-        if (checkMainClasses) {
+        for (String rootDir : rootPathList.split(Pattern.quote(File.pathSeparator))) {
-            check.checkMain(rootPath);
+            Path rootPath = Paths.get(rootDir);
-        } else {
-            check.checkTests(rootPath, skipIntegTestsInDisguise);
-        }
-
-        if (selfTest) {
             if (checkMainClasses) {
-                assertViolation(NamingConventionsCheckInMainTests.class.getName(), check.testsInMain);
+                check.checkMain(rootPath);
-                assertViolation(NamingConventionsCheckInMainIT.class.getName(), check.testsInMain);
             } else {
-                assertViolation("WrongName", check.missingSuffix);
+                check.checkTests(rootPath, skipIntegTestsInDisguise);
-                assertViolation("WrongNameTheSecond", check.missingSuffix);
-                assertViolation("DummyAbstractTests", check.notRunnable);
-                assertViolation("DummyInterfaceTests", check.notRunnable);
-                assertViolation("InnerTests", check.innerClasses);
-                assertViolation("NotImplementingTests", check.notImplementing);
-                assertViolation("PlainUnit", check.pureUnitTest);
             }
         }
 
         // Now we should have no violations
-        assertNoViolations(
+        int exitCode  = 0 ;
+        exitCode += countAndPrintViolations(
                 "Not all subclasses of " + check.testClass.getSimpleName()
                     + " match the naming convention. Concrete classes must end with [Tests]",
-                check.missingSuffix);
+                check.missingSuffix) ;
-        assertNoViolations("Classes ending with [Tests] are abstract or interfaces", check.notRunnable);
+        exitCode += countAndPrintViolations(
-        assertNoViolations("Found inner classes that are tests, which are excluded from the test runner", check.innerClasses);
+            "Classes ending with [Tests] are abstract or interfaces",
-        assertNoViolations("Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]", check.pureUnitTest);
+            check.notRunnable
-        assertNoViolations("Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]", check.notImplementing);
+        );
-        assertNoViolations(
+        exitCode += countAndPrintViolations(
-                "Classes ending with [Tests] or [IT] or extending [" + check.testClass.getSimpleName() + "] must be in src/test/java",
+            "Found inner classes that are tests, which are excluded from the test runner",
-                check.testsInMain);
+            check.innerClasses
+        );
+        exitCode += countAndPrintViolations(
+            "Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]",
+            check.pureUnitTest
+        );
+        exitCode += countAndPrintViolations(
+            "Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]",
+            check.notImplementing
+        );
+        exitCode += countAndPrintViolations(
+                "Classes ending with [Tests] or [IT] or extending [" +
+                    check.testClass.getSimpleName() + "] must be in src/test/java",
+                check.testsInMain
+        );
         if (skipIntegTestsInDisguise == false) {
-            assertNoViolations(
+            exitCode += countAndPrintViolations("Subclasses of " + check.integTestClass.getSimpleName() +
-                    "Subclasses of " + check.integTestClass.getSimpleName() + " should end with IT as they are integration tests",
+                        " should end with IT as they are integration tests",
-                    check.integTestsInDisguise);
+                    check.integTestsInDisguise
+                );
         }
+        System.exit(exitCode);
     }
 
     private final Set<Class<?>> notImplementing = new HashSet<>();
@@ -138,7 +136,9 @@ public class NamingConventionsCheck {
         Files.walkFileTree(rootPath, new TestClassVisitor() {
             @Override
             protected void visitTestClass(Class<?> clazz) {
-                if (skipTestsInDisguised == false && integTestClass.isAssignableFrom(clazz)) {
+                if (skipTestsInDisguised == false &&
+                    integTestClass.isAssignableFrom(clazz) &&
+                    clazz != integTestClass) {
                     integTestsInDisguise.add(clazz);
                 }
                 if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
@@ -196,18 +196,15 @@ public class NamingConventionsCheck {
 
     }
 
-    /**
+    private static int countAndPrintViolations(String message, Set<Class<?>> set) {
-     * Fail the process if there are any violations in the set. Named to look like a junit assertion even though it isn't because it is
-     * similar enough.
-     */
-    private static void assertNoViolations(String message, Set<Class<?>> set) {
         if (false == set.isEmpty()) {
             System.err.println(message + ":");
             for (Class<?> bad : set) {
                 System.err.println(" * " + bad.getName());
             }
-            System.exit(1);
+            return 1;
         }
+        return 0;
     }
 
     /**
@@ -254,15 +251,16 @@ public class NamingConventionsCheck {
          * Visit classes named like a test.
          */
         protected abstract void visitTestClass(Class<?> clazz);
+
         /**
          * Visit classes named like an integration test.
          */
         protected abstract void visitIntegrationTestClass(Class<?> clazz);
+
         /**
          * Visit classes not named like a test at all.
          */
         protected abstract void visitOtherClass(Class<?> clazz);
-
         @Override
         public final FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
             // First we visit the root directory
@@ -310,5 +308,7 @@ public class NamingConventionsCheck {
         public final FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
             throw exc;
         }
+
     }
+
 }

buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTests.groovy

@@ -1,6 +1,9 @@
 package org.elasticsearch.gradle
 
-class VersionCollectionTest extends GroovyTestCase {
+import org.elasticsearch.gradle.test.GradleUnitTestCase
+import org.junit.Test
+
+class VersionCollectionTests extends GradleUnitTestCase {
 
   String formatVersion(String version) {
     return " public static final Version V_${version.replaceAll("\\.", "_")} "
@@ -16,6 +19,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * branched from Major-1.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
    * from 7.0 perspective, or master at the time of this writing.
    */
+  @Test
   void testAgainstMajorUnreleasedWithExistingStagedMinorRelease() {
     VersionCollection vc = new VersionCollection(allVersions)
     assertNotNull(vc)
@@ -51,6 +55,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
    * simulates the behavior from 7.0 perspective, or master at the time of this writing.
    */
+  @Test
   void testAgainstMajorUnreleasedWithoutStagedMinorRelease() {
     List localVersion = allVersions.clone()
     localVersion.add(formatVersion('6.2.1')) // release 6.2
@@ -89,6 +94,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * branched from Major.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
    * from 6.3 perspective.
    */
+  @Test
   void testAgainstMinorReleasedBranch() {
     List localVersion = allVersions.clone()
     localVersion.removeAll { it.toString().contains('7_0_0')} // remove all the 7.x so that the actual version is 6.3 (6.x)
@@ -126,6 +132,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
    * simulates the behavior from 6.3 perspective.
    */
+  @Test
   void testAgainstMinorReleasedBranchNoStagedMinor() {
     List localVersion = allVersions.clone()
     // remove all the 7.x and add a 6.2.1 which means 6.2 was released
@@ -162,6 +169,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * This validates the logic of being on a released minor branch. At the time of writing, 6.2 is unreleased, so this is equivalent of being
    * on 6.1.
    */
+  @Test
   void testAgainstOldMinor() {
 
     List localVersion = allVersions.clone()
@@ -195,6 +203,7 @@ class VersionCollectionTest extends GroovyTestCase {
    * This validates the lower bound of wire compat, which is 5.0. It also validates that the span of 2.x to 5.x if it is decided to port
    * this fix all the way to the maint 5.6 release.
    */
+  @Test
   void testFloorOfWireCompatVersions() {
     List localVersion = [formatVersion('2.0.0'), formatVersion('2.0.1'), formatVersion('2.1.0'), formatVersion('2.1.1'),
                           formatVersion('5.0.0'), formatVersion('5.0.1'), formatVersion('5.1.0'), formatVersion('5.1.1'),

buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestFromSnippetsTaskTests.groovy

@@ -19,31 +19,41 @@
 
 package org.elasticsearch.gradle.doc
 
-import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.shouldAddShardFailureCheck
+import org.elasticsearch.gradle.test.GradleUnitTestCase
-import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
+import org.gradle.api.InvalidUserDataException
+import org.junit.Rule
+import org.junit.rules.ExpectedException
+
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.shouldAddShardFailureCheck
+
+class RestTestFromSnippetsTaskTests extends GradleUnitTestCase {
+
+    @Rule
+    public ExpectedException expectedEx = ExpectedException.none()
 
-class RestTestFromSnippetsTaskTest extends GroovyTestCase {
     void testInvalidBlockQuote() {
-        String input = "\"foo\": \"\"\"bar\"";
+        String input = "\"foo\": \"\"\"bar\""
-        String message = shouldFail({ replaceBlockQuote(input) });
+        expectedEx.expect(InvalidUserDataException.class)
-        assertEquals("Invalid block quote starting at 7 in:\n$input", message);
+        expectedEx.expectMessage("Invalid block quote starting at 7 in:\n$input")
+        replaceBlockQuote(input)
     }
 
     void testSimpleBlockQuote() {
         assertEquals("\"foo\": \"bort baz\"",
-            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\""));
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\""))
     }
 
     void testMultipleBlockQuotes() {
         assertEquals("\"foo\": \"bort baz\", \"bar\": \"other\"",
-            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\""));
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\""))
     }
 
     void testEscapingInBlockQuote() {
         assertEquals("\"foo\": \"bort\\\" baz\"",
-            replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\""));
+            replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\""))
         assertEquals("\"foo\": \"bort\\n baz\"",
-            replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""));
+            replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""))
     }
 
     void testIsDocWriteRequest() {

buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java

@@ -0,0 +1,72 @@
+package org.elasticsearch.gradle.precommit;
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.gradle.testkit.runner.TaskOutcome;
+
+import java.util.Arrays;
+
+public class NamingConventionsTaskIT extends GradleIntegrationTestCase {
+
+    public void testPluginCanBeApplied() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
+            .withArguments("hello", "-s", "-PcheckForTestsInMain=false")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SUCCESS, result.task(":hello").getOutcome());
+        assertTrue(result.getOutput().contains("build plugin can be applied"));
+    }
+
+    public void testNameCheckFailsAsItShould() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
+            .withArguments("namingConventions", "-s", "-PcheckForTestsInMain=false")
+            .withPluginClasspath()
+            .buildAndFail();
+
+        assertNotNull("task did not run", result.task(":namingConventions"));
+        assertEquals(TaskOutcome.FAILED, result.task(":namingConventions").getOutcome());
+        for (String line : Arrays.asList(
+            "Found inner classes that are tests, which are excluded from the test runner:",
+            "* org.elasticsearch.test.NamingConventionsCheckInMainIT$InternalInvalidTests",
+            "Classes ending with [Tests] must subclass [UnitTestCase]:",
+            "* org.elasticsearch.test.NamingConventionsCheckInMainTests",
+            "* org.elasticsearch.test.NamingConventionsCheckInMainIT",
+            "Not all subclasses of UnitTestCase match the naming convention. Concrete classes must end with [Tests]:",
+            "* org.elasticsearch.test.WrongName")) {
+            assertTrue(
+                "expected:  '" + line + "' but it was not found in the output",
+                result.getOutput().contains(line)
+            );
+        }
+    }
+
+    public void testNameCheckFailsAsItShouldWithMain() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
+            .withArguments("namingConventions", "-s", "-PcheckForTestsInMain=true")
+            .withPluginClasspath()
+            .buildAndFail();
+
+        assertNotNull("task did not run", result.task(":namingConventions"));
+        assertEquals(TaskOutcome.FAILED, result.task(":namingConventions").getOutcome());
+
+        for (String line : Arrays.asList(
+            "Classes ending with [Tests] or [IT] or extending [UnitTestCase] must be in src/test/java:",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyInterfaceTests",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyAbstractTests",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$InnerTests",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$NotImplementingTests",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongNameTheSecond",
+            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongName")) {
+            assertTrue(
+                "expected:  '" + line + "' but it was not found in the output",
+                result.getOutput().contains(line)
+            );
+        }
+    }
+
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/BaseTestCase.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test;
+
+import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
+import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import org.junit.Assert;
+import org.junit.runner.RunWith;
+
+@RunWith(RandomizedRunner.class)
+@TestMethodProviders({
+    JUnit4MethodProvider.class,
+    JUnit3MethodProvider.class
+})
+public abstract class BaseTestCase extends Assert {
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java

@@ -0,0 +1,16 @@
+package org.elasticsearch.gradle.test;
+
+import java.io.File;
+
+public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
+
+    protected File getProjectDir(String name) {
+        File root = new File("src/testKit/");
+        if (root.exists() == false) {
+            throw new RuntimeException("Could not find resources dir for integration tests. " +
+                "Note that these tests can only be ran by Gradle and are not currently supported by the IDE");
+        }
+        return new File(root, name);
+    }
+
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleUnitTestCase.java

@@ -0,0 +1,14 @@
+package org.elasticsearch.gradle.test;
+
+import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
+import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import org.junit.runner.RunWith;
+
+@RunWith(RandomizedRunner.class)
+@TestMethodProviders({
+    JUnit4MethodProvider.class,
+    JUnit3MethodProvider.class
+})
+public abstract class GradleUnitTestCase extends BaseTestCase  {
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/JUnit3MethodProvider.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test;
+
+import com.carrotsearch.randomizedtesting.ClassModel;
+import com.carrotsearch.randomizedtesting.ClassModel.MethodModel;
+import com.carrotsearch.randomizedtesting.TestMethodProvider;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Backwards compatible test* method provider (public, non-static).
+ *
+ * copy of org.apache.lucene.util.LuceneJUnit3MethodProvider to avoid a dependency between build and test fw.
+ */
+public final class JUnit3MethodProvider implements TestMethodProvider {
+    @Override
+    public Collection<Method> getTestMethods(Class<?> suiteClass, ClassModel classModel) {
+        Map<Method,MethodModel> methods = classModel.getMethods();
+        ArrayList<Method> result = new ArrayList<>();
+        for (MethodModel mm : methods.values()) {
+            // Skip any methods that have overrieds/ shadows.
+            if (mm.getDown() != null) continue;
+
+            Method m = mm.element;
+            if (m.getName().startsWith("test") &&
+                Modifier.isPublic(m.getModifiers()) &&
+                !Modifier.isStatic(m.getModifiers()) &&
+                m.getParameterTypes().length == 0) {
+                result.add(m);
+            }
+        }
+        return result;
+    }
+}

buildSrc/src/testKit/namingConventionsSelfTest/build.gradle

@@ -0,0 +1,30 @@
+plugins {
+    id 'java'
+    id 'elasticsearch.build'
+}
+
+dependencyLicenses.enabled = false
+dependenciesInfo.enabled = false
+forbiddenApisMain.enabled = false
+forbiddenApisTest.enabled = false
+jarHell.enabled = false
+thirdPartyAudit.enabled = false
+
+ext.licenseFile = file("$buildDir/dummy/license")
+ext.noticeFile = file("$buildDir/dummy/notice")
+
+task hello {
+    doFirst {
+        println "build plugin can be applied"
+    }
+}
+
+dependencies {
+    compile "junit:junit:${versions.junit}"
+}
+
+namingConventions  {
+    checkForTestsInMain = project.property("checkForTestsInMain") == "true"
+    testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
+    integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
+}

buildSrc/src/testKit/namingConventionsSelfTest/src/test/java/org/elasticsearch/test/NamingConventionsCheckInMainIT.java

@@ -23,4 +23,9 @@ package org.elasticsearch.test;
  * This class should fail the naming conventions self test.
  */
 public class NamingConventionsCheckInMainIT {
+
+    public static class InternalInvalidTests extends NamingConventionsCheckBadClasses.UnitTestCase {
+
+    }
+
 }

buildSrc/src/testKit/namingConventionsSelfTest/src/test/java/org/elasticsearch/test/WrongName.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+/**
+ * This class should fail the naming conventions self test.
+ */
+public class WrongName extends NamingConventionsCheckBadClasses.UnitTestCase {
+}

server/build.gradle

@@ -332,7 +332,7 @@ if (isEclipse == false || project.path == ":server-tests") {
                  dependsOn: test.dependsOn) {
     configure(BuildPlugin.commonTestConfig(project))
     classpath = project.test.classpath
-    testClassesDir = project.test.testClassesDir
+    testClassesDirs = project.test.testClassesDirs
     include '**/*IT.class'
   }
   check.dependsOn integTest

x-pack/plugin/ml/build.gradle

@@ -99,7 +99,7 @@ task internalClusterTest(type: RandomizedTestingTask,
                          dependsOn: test.dependsOn) {
   configure(BuildPlugin.commonTestConfig(project))
   classpath = project.test.classpath
-  testClassesDir = project.test.testClassesDir
+  testClassesDirs = project.test.testClassesDirs
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }

x-pack/plugin/monitoring/build.gradle

@@ -62,7 +62,7 @@ task internalClusterTest(type: RandomizedTestingTask,
                          dependsOn: test.dependsOn) {
   configure(BuildPlugin.commonTestConfig(project))
   classpath = project.test.classpath
-  testClassesDir = project.test.testClassesDir
+  testClassesDirs = project.test.testClassesDirs
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }

x-pack/plugin/rollup/build.gradle

@@ -42,7 +42,7 @@ task internalClusterTest(type: RandomizedTestingTask,
         dependsOn: test.dependsOn) {
     configure(BuildPlugin.commonTestConfig(project))
     classpath = project.test.classpath
-    testClassesDir = project.test.testClassesDir
+    testClassesDirs = project.test.testClassesDirs
     include '**/*IT.class'
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }

x-pack/plugin/upgrade/build.gradle

@@ -35,7 +35,7 @@ task internalClusterTest(type: RandomizedTestingTask,
                          dependsOn: test.dependsOn) {
   configure(BuildPlugin.commonTestConfig(project))
   classpath = project.test.classpath
-  testClassesDir = project.test.testClassesDir
+  testClassesDirs = project.test.testClassesDirs
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }