Testing conventions: add support for checking base classes (#36650)

buildSrc/build.gradle

@@ -223,6 +223,18 @@ if (project != rootProject) {
     integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
 
+  testingConventions {
+    naming.clear()
+    naming {
+      Tests {
+        baseClass 'org.elasticsearch.gradle.test.GradleUnitTestCase'
+      }
+      IT {
+        baseClass 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
+      }
+    }
+  }
+
   /*
    * We alread configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -78,6 +78,19 @@ public class PluginBuildPlugin extends BuildPlugin {
                 skipIntegTestInDisguise = true
             }
         }
+        project.testingConventions {
+            naming.clear()
+            naming {
+                Tests {
+                    baseClass 'org.apache.lucene.util.LuceneTestCase'
+                }
+                IT {
+                    baseClass 'org.elasticsearch.test.ESIntegTestCase'
+                    baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+                    baseClass 'org.elasticsearch.test.ESSingleNodeTestCase'
+                }
+            }
+        }
         createIntegTestTask(project)
         createBundleTask(project)
         project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -91,7 +91,17 @@ class PrecommitTasks {
     }
 
     static Task configureTestingConventions(Project project) {
-        project.getTasks().create("testingConventions", TestingConventionsTasks.class)
+        TestingConventionsTasks task = project.getTasks().create("testingConventions", TestingConventionsTasks.class)
+        task.naming {
+            Tests {
+                baseClass "org.apache.lucene.util.LuceneTestCase"
+            }
+            IT {
+                baseClass "org.elasticsearch.test.ESIntegTestCase"
+                baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+            }
+        }
+        return task
     }
 
     private static Task configureJarHell(Project project) {

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionRule.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Represent rules for tests enforced by the @{link {@link TestingConventionsTasks}}
+ *
+ * Rules are identified by name, tests must have this name as a suffix and implement one of the base classes
+ * and be part of all the specified tasks.
+ */
+public class TestingConventionRule implements Serializable {
+
+    private final String suffix;
+
+    private Set<String> baseClasses = new HashSet<>();
+
+    private Set<Pattern> taskNames = new HashSet<>();
+
+    public TestingConventionRule(String suffix) {
+        this.suffix = suffix;
+    }
+
+    public String getSuffix() {
+        return suffix;
+    }
+
+    /**
+     * Alias for @{link getSuffix} as Gradle requires a name property
+     *
+     */
+    public String getName() {
+        return suffix;
+    }
+
+    public void baseClass(String clazz) {
+        baseClasses.add(clazz);
+    }
+
+    public void setBaseClasses(Collection<String> baseClasses) {
+        this.baseClasses.clear();
+        this.baseClasses.addAll(baseClasses);
+    }
+
+    public void taskName(Pattern expression) {
+        taskNames.add(expression);
+    }
+    public void taskName(String expression) {
+        taskNames.add(Pattern.compile(expression));
+    }
+
+    public void setTaskNames(Collection<Pattern> expressions) {
+        taskNames.clear();
+        taskNames.addAll(expressions);
+    }
+
+    public Set<String> getBaseClasses() {
+        return baseClasses;
+    }
+
+    public Set<Pattern> getTaskNames() {
+        return taskNames;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TestingConventionRule that = (TestingConventionRule) o;
+        return Objects.equals(suffix, that.suffix);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(suffix);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -18,8 +18,10 @@
  */
 package org.elasticsearch.gradle.precommit;
 
+import groovy.lang.Closure;
 import org.elasticsearch.gradle.tool.Boilerplate;
 import org.gradle.api.DefaultTask;
+import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Task;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
@@ -54,50 +56,37 @@ import java.util.stream.Stream;
 
 public class TestingConventionsTasks extends DefaultTask {
 
-    private static final String TEST_CLASS_SUFIX = "Tests";
-    private static final String INTEG_TEST_CLASS_SUFIX = "IT";
     private static final String TEST_METHOD_PREFIX = "test";
 
-    /**
-     * Are there tests to execute ? Accounts for @Ignore and @AwaitsFix
-     */
-    private Boolean activeTestsExists;
-
     private Map<String, File> testClassNames;
 
+    private final NamedDomainObjectContainer<TestingConventionRule> naming;
+
     public TestingConventionsTasks() {
         setDescription("Tests various testing conventions");
         // Run only after everything is compiled
         Boilerplate.getJavaSourceSets(getProject()).all(sourceSet -> dependsOn(sourceSet.getClassesTaskName()));
+        naming = getProject().container(TestingConventionRule.class);
     }
-    
+
     @Input
-    public Map<String, Set<File>> classFilesPerTask(FileTree testClassFiles) {
+    public Map<String, Set<File>> classFilesPerEnabledTask(FileTree testClassFiles) {
         Map<String, Set<File>> collector = new HashMap<>();
+
         // RandomizedTestingTask
         collector.putAll(
-            Stream.concat(
+            getProject().getTasks().withType(getRandomizedTestingTask()).stream()
-                getProject().getTasks().withType(getRandomizedTestingTask()).stream(),
-                // Look at sub-projects too. As sometimes tests are implemented in parent but ran in sub-projects against
-                // different configurations
-                getProject().getSubprojects().stream().flatMap(subproject ->
-                    subproject.getTasks().withType(getRandomizedTestingTask()).stream()
-                )
-            )
                 .filter(Task::getEnabled)
                 .collect(Collectors.toMap(
                     Task::getPath,
                     task -> testClassFiles.matching(getRandomizedTestingPatternSet(task)).getFiles()
-            ))
+                    )
+                )
         );
+
         // Gradle Test
         collector.putAll(
-            Stream.concat(
+            getProject().getTasks().withType(Test.class).stream()
-                getProject().getTasks().withType(Test.class).stream(),
-                getProject().getSubprojects().stream().flatMap(subproject ->
-                        subproject.getTasks().withType(Test.class).stream()
-                )
-            )
                 .filter(Task::getEnabled)
                 .collect(Collectors.toMap(
                     Task::getPath,
@@ -119,14 +108,22 @@ public class TestingConventionsTasks extends DefaultTask {
         return testClassNames;
     }
 
+    @Input
+    public NamedDomainObjectContainer<TestingConventionRule> getNaming() {
+        return naming;
+    }
+
     @OutputFile
     public File getSuccessMarker() {
         return new File(getProject().getBuildDir(), "markers/" + getName());
     }
 
+    public void naming(Closure<TestingConventionRule> action) {
+        naming.configure(action);
+    }
+
     @TaskAction
     public void doCheck() throws IOException {
-        activeTestsExists = false;
         final String problems;
 
         try (URLClassLoader isolatedClassLoader = new URLClassLoader(
@@ -134,62 +131,83 @@ public class TestingConventionsTasks extends DefaultTask {
         )) {
             Predicate<Class<?>> isStaticClass = clazz -> Modifier.isStatic(clazz.getModifiers());
             Predicate<Class<?>> isPublicClass = clazz -> Modifier.isPublic(clazz.getModifiers());
-            Predicate<Class<?>> implementsNamingConvention = clazz ->
+            Predicate<Class<?>> isAbstractClass = clazz -> Modifier.isAbstract(clazz.getModifiers());
-                clazz.getName().endsWith(TEST_CLASS_SUFIX) ||
-                    clazz.getName().endsWith(INTEG_TEST_CLASS_SUFIX);
 
-            Map<File, ? extends Class<?>> classes = getTestClassNames().entrySet().stream()
+            final Map<File, ? extends Class<?>> classes = getTestClassNames().entrySet().stream()
                 .collect(Collectors.toMap(
                     Map.Entry::getValue,
                     entry -> loadClassWithoutInitializing(entry.getKey(), isolatedClassLoader))
                 );
 
-            FileTree allTestClassFiles = getProject().files(
+            final FileTree allTestClassFiles = getProject().files(
                 classes.values().stream()
                     .filter(isStaticClass.negate())
                     .filter(isPublicClass)
-                    .filter(implementsNamingConvention)
+                    .filter((Predicate<Class<?>>) this::implementsNamingConvention)
                     .map(clazz -> testClassNames.get(clazz.getName()))
                     .collect(Collectors.toList())
             ).getAsFileTree();
 
-            final Map<String, Set<File>> classFilesPerTask = classFilesPerTask(allTestClassFiles);
+            final Map<String, Set<File>> classFilesPerTask = classFilesPerEnabledTask(allTestClassFiles);
 
-            Map<String, Set<Class<?>>> testClassesPerTask = classFilesPerTask.entrySet().stream()
+            final Map<String, Set<Class<?>>> testClassesPerTask = classFilesPerTask.entrySet().stream()
                 .collect(
                     Collectors.toMap(
                         Map.Entry::getKey,
                         entry -> entry.getValue().stream()
                             .map(classes::get)
-                            .filter(implementsNamingConvention)
+                            .filter(this::implementsNamingConvention)
                             .collect(Collectors.toSet())
                     )
                 );
 
+            final Map<String, Set<Class<?>>> suffixToBaseClass;
+            if (classes.isEmpty()) {
+                // Don't load base classes if we don't have any tests.
+                // This allows defaults to be configured for projects that don't have any tests
+                //
+                suffixToBaseClass = Collections.emptyMap();
+            } else {
+                suffixToBaseClass = naming.stream()
+                    .collect(
+                        Collectors.toMap(
+                            TestingConventionRule::getSuffix,
+                            rule -> rule.getBaseClasses().stream()
+                                .map(each -> loadClassWithoutInitializing(each, isolatedClassLoader))
+                                .collect(Collectors.toSet())
+                        ));
+            }
+
             problems = collectProblems(
                 checkNoneExists(
                     "Test classes implemented by inner classes will not run",
                     classes.values().stream()
                         .filter(isStaticClass)
-                        .filter(implementsNamingConvention.or(this::seemsLikeATest))
+                        .filter(isPublicClass)
+                        .filter(((Predicate<Class<?>>) this::implementsNamingConvention).or(this::seemsLikeATest))
                 ),
                 checkNoneExists(
                     "Seem like test classes but don't match naming convention",
                     classes.values().stream()
                         .filter(isStaticClass.negate())
                         .filter(isPublicClass)
-                        .filter(this::seemsLikeATest)
+                        .filter(isAbstractClass.negate())
-                        .filter(implementsNamingConvention.negate())
+                        .filter(this::seemsLikeATest) // TODO when base classes are set, check for classes that extend them
+                        .filter(((Predicate<Class<?>>) this::implementsNamingConvention).negate())
                 ),
+                // TODO: check for non public classes that seem like tests
+                // TODO: check for abstract classes that implement the naming conventions
+                // No empty enabled tasks
                 collectProblems(
                     testClassesPerTask.entrySet().stream()
-                        .map( entry ->
+                        .map(entry ->
                             checkAtLeastOneExists(
-                                "test class in " + entry.getKey(),
+                                "test class included in task " + entry.getKey(),
                                 entry.getValue().stream()
                             )
                         )
-                        .collect(Collectors.joining())
+                        .sorted()
+                        .collect(Collectors.joining("\n"))
                 ),
                 checkNoneExists(
                     "Test classes are not included in any enabled task (" +
@@ -201,25 +219,43 @@ public class TestingConventionsTasks extends DefaultTask {
                                 .anyMatch(fileSet -> fileSet.contains(testFile)) == false
                         )
                         .map(classes::get)
+                ),
+                collectProblems(
+                    suffixToBaseClass.entrySet().stream()
+                        .filter(entry -> entry.getValue().isEmpty() == false)
+                        .map(entry -> {
+                            return checkNoneExists(
+                                "Tests classes with suffix `" + entry.getKey() + "` should extend " +
+                                    entry.getValue().stream().map(Class::getName).collect(Collectors.joining(" or ")) +
+                                    " but the following classes do not",
+                                classes.values().stream()
+                                    .filter(clazz -> clazz.getName().endsWith(entry.getKey()))
+                                    .filter(clazz -> entry.getValue().stream()
+                                        .anyMatch(test -> test.isAssignableFrom(clazz)) == false)
+                            );
+                        }).sorted()
+                        .collect(Collectors.joining("\n"))
                 )
+                // TODO: check that the testing tasks are included in the right task based on the name ( from the rule )
+                // TODO: check to make sure that the main source set doesn't have classes that match
+                //         the naming convention (just the names, don't load classes)
             );
         }
 
         if (problems.isEmpty()) {
-            getLogger().error(problems);
-            throw new IllegalStateException("Testing conventions are not honored");
-        } else {
             getSuccessMarker().getParentFile().mkdirs();
             Files.write(getSuccessMarker().toPath(), new byte[]{}, StandardOpenOption.CREATE);
+        } else {
+            getLogger().error(problems);
+            throw new IllegalStateException("Testing conventions are not honored");
         }
     }
 
     private String collectProblems(String... problems) {
         return Stream.of(problems)
             .map(String::trim)
-            .filter(String::isEmpty)
+            .filter(s -> s.isEmpty() == false)
-            .map(each -> each + "\n")
+            .collect(Collectors.joining("\n"));
-            .collect(Collectors.joining());
     }
 
     @SuppressWarnings("unchecked")
@@ -251,10 +287,11 @@ public class TestingConventionsTasks extends DefaultTask {
     private String checkNoneExists(String message, Stream<? extends Class<?>> stream) {
         String problem = stream
             .map(each -> "  * " + each.getName())
+            .sorted()
             .collect(Collectors.joining("\n"));
         if (problem.isEmpty() == false) {
             return message + ":\n" + problem;
-        } else{
+        } else {
             return "";
         }
     }
@@ -263,28 +300,33 @@ public class TestingConventionsTasks extends DefaultTask {
         if (stream.findAny().isPresent()) {
             return "";
         } else {
-            return "Expected at least one " + message + ", but found none.\n";
+            return "Expected at least one " + message + ", but found none.";
         }
     }
 
     private boolean seemsLikeATest(Class<?> clazz) {
         try {
             ClassLoader classLoader = clazz.getClassLoader();
-            Class<?> junitTest;
+
-            try {
+            Class<?> junitTest = loadClassWithoutInitializing("org.junit.Assert", classLoader);
-                junitTest = classLoader.loadClass("junit.framework.Test");
-            } catch (ClassNotFoundException e) {
-                throw new IllegalStateException("Could not load junit.framework.Test. It's expected that this class is " +
-                    "available on the tests classpath");
-            }
             if (junitTest.isAssignableFrom(clazz)) {
-                getLogger().info("{} is a test because it extends junit.framework.Test", clazz.getName());
+                getLogger().info("{} is a test because it extends {}", clazz.getName(), junitTest.getName());
                 return true;
             }
+
+            Class<?> junitAnnotation = loadClassWithoutInitializing("org.junit.Test", classLoader);
             for (Method method : clazz.getMethods()) {
-                if (matchesTestMethodNamingConvention(clazz, method)) return true;
+                if (matchesTestMethodNamingConvention(method)) {
-                if (isAnnotated(clazz, method, junitTest)) return true;
+                    getLogger().info("{} is a test because it has method named '{}'", clazz.getName(), method.getName());
+                    return true;
+                }
+                if (isAnnotated(method, junitAnnotation)) {
+                    getLogger().info("{} is a test because it has method '{}' annotated with '{}'",
+                        clazz.getName(), method.getName(), junitAnnotation.getName());
+                    return true;
+                }
             }
+
             return false;
         } catch (NoClassDefFoundError e) {
             // Include the message to get more info to get more a more useful message when running Gradle without -s
@@ -294,23 +336,25 @@ public class TestingConventionsTasks extends DefaultTask {
         }
     }
 
-    private boolean matchesTestMethodNamingConvention(Class<?> clazz, Method method) {
+    private boolean implementsNamingConvention(Class<?> clazz) {
-        if (method.getName().startsWith(TEST_METHOD_PREFIX) &&
+        if (naming.stream()
-            Modifier.isStatic(method.getModifiers()) == false &&
+            .map(TestingConventionRule::getSuffix)
-            method.getReturnType().equals(Void.class)
+            .anyMatch(suffix -> clazz.getName().endsWith(suffix))) {
-        ) {
+            getLogger().info("{} is a test because it matches the naming convention", clazz.getName());
-            getLogger().info("{} is a test because it has method: {}", clazz.getName(), method.getName());
             return true;
         }
         return false;
     }
 
-    private boolean isAnnotated(Class<?> clazz, Method method, Class<?> annotation) {
+    private boolean matchesTestMethodNamingConvention(Method method) {
+        return method.getName().startsWith(TEST_METHOD_PREFIX) &&
+            Modifier.isStatic(method.getModifiers()) == false
+            ;
+    }
+
+    private boolean isAnnotated(Method method, Class<?> annotation) {
         for (Annotation presentAnnotation : method.getAnnotations()) {
             if (annotation.isAssignableFrom(presentAnnotation.getClass())) {
-                getLogger().info("{} is a test because {} is annotated with junit.framework.Test",
-                    clazz.getName(), method.getName()
-                );
                 return true;
             }
         }
@@ -380,14 +424,14 @@ public class TestingConventionsTasks extends DefaultTask {
 
     private Class<?> loadClassWithoutInitializing(String name, ClassLoader isolatedClassLoader) {
         try {
-            return Class.forName(name,
+            return Class.forName(
+                name,
                 // Don't initialize the class to save time. Not needed for this test and this doesn't share a VM with any other tests.
                 false,
                 isolatedClassLoader
             );
         } catch (ClassNotFoundException e) {
-            // Will not get here as the exception will be loaded by isolatedClassLoader
+            throw new RuntimeException("Failed to load class " + name + ". Incorrect test runtime classpath?", e);
-            throw new RuntimeException("Failed to load class " + name, e);
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -21,6 +21,7 @@ package org.elasticsearch.gradle.testfixtures;
 import com.avast.gradle.dockercompose.ComposeExtension;
 import com.avast.gradle.dockercompose.DockerComposePlugin;
 import org.elasticsearch.gradle.precommit.JarHellTask;
+import org.elasticsearch.gradle.precommit.TestingConventionsTasks;
 import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Plugin;
@@ -100,6 +101,10 @@ public class TestFixturesPlugin implements Plugin<Project> {
                 tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
                     task.setEnabled(false)
                 );
+                // conventions are not honored when the tasks are disabled
+                tasks.withType(TestingConventionsTasks.class, task ->
+                    task.setEnabled(false)
+                );
                 return;
             }
             tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -67,5 +67,8 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]ScriptServiceTests.java" checks="LineLength" />
 
  <!-- Temporarily contains extra-long lines as examples for tests to be written, see https://github.com/elastic/elasticsearch/issues/34829 -->
-  <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
+ <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
+
+ <!-- Gradle requires inputs to be seriablizable -->
+ <suppress files="buildSrc[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />
 </suppressions>

buildSrc/src/test/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit;
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.Before;
+
+public class TestingConventionsTasksIT extends GradleIntegrationTestCase {
+
+    @Before
+    public void setUp() {
+    }
+
+    public void testInnerClasses() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":no_tests_in_inner_classes:testingConventions", "-i", "-s");
+        BuildResult result = runner.buildAndFail();
+        assertOutputContains(result.getOutput(),
+            "Test classes implemented by inner classes will not run:",
+            "  * org.elasticsearch.gradle.testkit.NastyInnerClasses$LooksLikeATestWithoutNamingConvention1",
+            "  * org.elasticsearch.gradle.testkit.NastyInnerClasses$LooksLikeATestWithoutNamingConvention2",
+            "  * org.elasticsearch.gradle.testkit.NastyInnerClasses$LooksLikeATestWithoutNamingConvention3",
+            "  * org.elasticsearch.gradle.testkit.NastyInnerClasses$NamingConventionIT",
+            "  * org.elasticsearch.gradle.testkit.NastyInnerClasses$NamingConventionTests"
+        );
+    }
+
+    public void testNamingConvention() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":incorrect_naming_conventions:testingConventions", "-i", "-s");
+        BuildResult result = runner.buildAndFail();
+        assertOutputContains(result.getOutput(),
+            "Seem like test classes but don't match naming convention:",
+            "  * org.elasticsearch.gradle.testkit.LooksLikeATestWithoutNamingConvention1",
+            "  * org.elasticsearch.gradle.testkit.LooksLikeATestWithoutNamingConvention2",
+            "  * org.elasticsearch.gradle.testkit.LooksLikeATestWithoutNamingConvention3"
+        );
+        assertOutputDoesNotContain(result.getOutput(), "LooksLikeTestsButAbstract");
+    }
+
+    public void testNoEmptyTasks() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":empty_test_task:testingConventions", "-i", "-s");
+        BuildResult result = runner.buildAndFail();
+        assertOutputContains(result.getOutput(),
+            "Expected at least one test class included in task :empty_test_task:emptyTest, but found none.",
+            "Expected at least one test class included in task :empty_test_task:emptyTestRandomized, but found none."
+        );
+    }
+
+    public void testAllTestTasksIncluded() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":all_classes_in_tasks:testingConventions", "-i", "-s");
+        BuildResult result = runner.buildAndFail();
+        assertOutputContains(result.getOutput(),
+            "Test classes are not included in any enabled task (:all_classes_in_tasks:emptyTestRandomized):",
+            "  * org.elasticsearch.gradle.testkit.NamingConventionIT",
+            "  * org.elasticsearch.gradle.testkit.NamingConventionTests"
+        );
+    }
+
+    public void testTaskNotImplementBaseClass() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":not_implementing_base:testingConventions", "-i", "-s");
+        BuildResult result = runner.buildAndFail();
+        assertOutputContains(result.getOutput(),
+            "Tests classes with suffix `IT` should extend org.elasticsearch.gradle.testkit.Integration but the following classes do not:",
+                "  * org.elasticsearch.gradle.testkit.NamingConventionIT",
+                "  * org.elasticsearch.gradle.testkit.NamingConventionMissmatchIT",
+                "Tests classes with suffix `Tests` should extend org.elasticsearch.gradle.testkit.Unit but the following classes do not:",
+                "  * org.elasticsearch.gradle.testkit.NamingConventionMissmatchTests",
+                "  * org.elasticsearch.gradle.testkit.NamingConventionTests"
+        );
+    }
+
+    public void testValidSetupWithoutBaseClass() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":valid_setup_no_base:testingConventions", "-i", "-s");
+        BuildResult result = runner.build();
+        assertTaskSuccessful(result, ":valid_setup_no_base:testingConventions");
+    }
+
+    public void testValidSetupWithBaseClass() {
+        GradleRunner runner = getGradleRunner("testingConventions")
+            .withArguments("clean", ":valid_setup_with_base:testingConventions", "-i", "-s");
+        BuildResult result = runner.build();
+        assertTaskSuccessful(result, ":valid_setup_with_base:testingConventions");
+    }
+
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java

@@ -43,7 +43,7 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
         if (index.equals(index.stream().sorted().collect(Collectors.toList())) == false) {
             fail("Expected the following lines to appear in this order:\n" +
                 Stream.of(lines).map(line -> "   - `" + line + "`").collect(Collectors.joining("\n")) +
-                "\nBut they did not. Output is:\n\n```" + output + "\n```\n"
+                "\nTBut the order was different. Output is:\n\n```" + output + "\n```\n"
             );
         }
     }

buildSrc/src/testKit/testingConventions/all_classes_in_tasks/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionIT {
+
+}

buildSrc/src/testKit/testingConventions/all_classes_in_tasks/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionTests {
+
+}

buildSrc/src/testKit/testingConventions/build.gradle

@@ -0,0 +1,86 @@
+plugins {
+    id 'elasticsearch.build' apply false
+}
+
+allprojects {
+    apply plugin: 'java'
+    apply plugin: 'elasticsearch.build'
+
+    repositories {
+        jcenter()
+    }
+    dependencies {
+        testCompile "junit:junit:4.12"
+    }
+
+    ext.licenseFile = file("$buildDir/dummy/license")
+    ext.noticeFile = file("$buildDir/dummy/notice")
+
+    testingConventions.naming {
+        // Reset default to no baseClass checks
+        Tests {
+            baseClasses = []
+        }
+        IT {
+            baseClasses = []
+        }
+    }
+
+    unitTest.enabled = false
+}
+
+project(':empty_test_task') {
+    task emptyTest(type: Test) {
+
+    }
+
+    task emptyTestRandomized(type: com.carrotsearch.gradle.junit4.RandomizedTestingTask) {
+
+    }
+}
+
+project(':all_classes_in_tasks') {
+    task emptyTestRandomized(type: com.carrotsearch.gradle.junit4.RandomizedTestingTask) {
+        include "**/Convention*"
+    }
+}
+
+project(':not_implementing_base') {
+    testingConventions.naming {
+        Tests {
+            baseClass 'org.elasticsearch.gradle.testkit.Unit'
+        }
+        IT {
+            baseClass 'org.elasticsearch.gradle.testkit.Integration'
+        }
+    }
+    task randomized(type: com.carrotsearch.gradle.junit4.RandomizedTestingTask) {
+        include "**/*IT.class"
+        include "**/*Tests.class"
+    }
+}
+
+project(':valid_setup_no_base') {
+    task randomized(type: com.carrotsearch.gradle.junit4.RandomizedTestingTask) {
+        include "**/*IT.class"
+        include "**/*Tests.class"
+    }
+}
+
+project (':valid_setup_with_base') {
+    task randomized(type: com.carrotsearch.gradle.junit4.RandomizedTestingTask) {
+        include "**/*IT.class"
+        include "**/*Tests.class"
+    }
+    testingConventions.naming {
+        Tests {
+            baseClass 'org.elasticsearch.gradle.testkit.Unit'
+        }
+        IT {
+            baseClass 'org.elasticsearch.gradle.testkit.Integration'
+        }
+    }
+}
+
+
+

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/LooksLikeATestWithoutNamingConvention1.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+import org.junit.Test;
+
+public class LooksLikeATestWithoutNamingConvention1 {
+
+    @Test
+    public void annotatedTestMethod() {
+
+    }
+
+}

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/LooksLikeATestWithoutNamingConvention2.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+import org.junit.Assert;
+
+public class LooksLikeATestWithoutNamingConvention2 extends Assert {
+
+}

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/LooksLikeATestWithoutNamingConvention3.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LooksLikeATestWithoutNamingConvention3 {
+
+    public void testMethod() {
+
+    }
+
+}

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/LooksLikeTestsButAbstract.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public abstract class LooksLikeTestsButAbstract {
+
+    public void testMethod() {
+
+    }
+
+}

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionIT {
+
+}

buildSrc/src/testKit/testingConventions/incorrect_naming_conventions/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionTests {
+
+}

buildSrc/src/testKit/testingConventions/no_tests_in_inner_classes/src/test/java/org/elasticsearch/gradle/testkit/NastyInnerClasses.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class NastyInnerClasses {
+
+    public static class NamingConventionTests {
+
+    }
+
+    public static class NamingConventionIT {
+
+    }
+
+    public static class LooksLikeATestWithoutNamingConvention1 {
+        @Test
+        public void annotatedTestMethod() {
+
+        }
+    }
+
+    public static class LooksLikeATestWithoutNamingConvention2 extends Assert {
+
+    }
+
+    public static class LooksLikeATestWithoutNamingConvention3 {
+
+        public void testMethod() {
+
+        }
+
+    }
+
+    static abstract public class NonOffendingAbstractTests {
+
+    }
+
+    private static class NonOffendingPrivateTests {
+
+    }
+
+    static class NonOffendingPackageTests {
+
+    }
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/AbstractIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public abstract class AbstractIT {
+
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/Integration.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class Integration {
+    
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionIT {
+
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionMissmatchIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionMissmatchIT extends Unit {
+
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionMissmatchTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionMissmatchTests extends Integration {
+
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionTests {
+
+}

buildSrc/src/testKit/testingConventions/not_implementing_base/src/test/java/org/elasticsearch/gradle/testkit/Unit.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class Unit {
+
+}

buildSrc/src/testKit/testingConventions/settings.gradle

@@ -0,0 +1,7 @@
+include 'no_tests_in_inner_classes'
+include 'incorrect_naming_conventions'
+include 'empty_test_task'
+include 'all_classes_in_tasks'
+include 'not_implementing_base'
+include 'valid_setup_no_base'
+include 'valid_setup_with_base'

buildSrc/src/testKit/testingConventions/valid_setup_no_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionIT {
+
+}

buildSrc/src/testKit/testingConventions/valid_setup_no_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionTests {
+
+}

buildSrc/src/testKit/testingConventions/valid_setup_with_base/src/test/java/org/elasticsearch/gradle/testkit/Integration.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class Integration {
+
+}

buildSrc/src/testKit/testingConventions/valid_setup_with_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionIT.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionIT extends Integration {
+
+}

buildSrc/src/testKit/testingConventions/valid_setup_with_base/src/test/java/org/elasticsearch/gradle/testkit/NamingConventionTests.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class NamingConventionTests extends Unit {
+
+}

buildSrc/src/testKit/testingConventions/valid_setup_with_base/src/test/java/org/elasticsearch/gradle/testkit/Unit.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.testkit;
+
+public class Unit {
+
+}

client/rest/build.gradle

@@ -77,6 +77,15 @@ namingConventions {
   skipIntegTestInDisguise = true
 }
 
+testingConventions {
+  naming.clear()
+  naming {
+    Tests {
+      baseClass 'org.elasticsearch.client.RestClientTestCase'
+    }
+  }
+}
+
 thirdPartyAudit.ignoreMissingClasses (
   //commons-logging optional dependencies
   'org.apache.avalon.framework.logger.Logger',

client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java

@@ -91,7 +91,7 @@ public class RestClientDocumentation {
     // end::rest-client-options-singleton
 
     @SuppressWarnings("unused")
-    public void testUsage() throws IOException, InterruptedException {
+    public void usage() throws IOException, InterruptedException {
 
         //tag::rest-client-init
         RestClient restClient = RestClient.builder(
@@ -291,7 +291,7 @@ public class RestClientDocumentation {
     }
 
     @SuppressWarnings("unused")
-    public void testCommonConfiguration() throws Exception {
+    public void commonConfiguration() throws Exception {
         {
             //tag::rest-client-config-timeouts
             RestClientBuilder builder = RestClient.builder(

client/sniffer/build.gradle

@@ -78,6 +78,16 @@ namingConventions {
   skipIntegTestInDisguise = true
 }
 
+testingConventions {
+  naming.clear()
+  naming {
+    Tests {
+      baseClass 'org.elasticsearch.client.RestClientTestCase'
+    }
+  }
+}
+
+
 dependencyLicenses {
   dependencies = project.configurations.runtime.fileCollection {
     it.group.startsWith('org.elasticsearch') == false

client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java

@@ -52,7 +52,7 @@ import java.util.concurrent.TimeUnit;
 public class SnifferDocumentation {
 
     @SuppressWarnings("unused")
-    public void testUsage() throws IOException {
+    public void usage() throws IOException {
         {
             //tag::sniffer-init
             RestClient restClient = RestClient.builder(

client/transport/build.gradle

@@ -52,3 +52,12 @@ namingConventions {
   //we don't have integration tests
   skipIntegTestInDisguise = true
 }
+
+testingConventions {
+  naming.clear()
+  naming {
+    Tests {
+      baseClass 'com.carrotsearch.randomizedtesting.RandomizedTest'
+    }
+  }
+}

distribution/tools/launchers/build.gradle

@@ -38,6 +38,15 @@ namingConventions {
     skipIntegTestInDisguise = true
 }
 
+testingConventions {
+    naming.clear()
+    naming {
+      Tests {
+        baseClass 'org.elasticsearch.tools.launchers.LaunchersTestCase'
+      }
+    }
+}
+
 javadoc.enabled = false
 loggerUsageCheck.enabled = false
 jarHell.enabled = false

libs/secure-sm/build.gradle

@@ -66,3 +66,12 @@ jarHell.enabled = false
 namingConventions {
     testClass = 'junit.framework.TestCase'
 }
+
+testingConventions {
+    naming.clear()
+    naming {
+        Tests {
+            baseClass 'junit.framework.TestCase'
+        }
+    }
+}

plugins/examples/rest-handler/build.gradle

@@ -41,4 +41,10 @@ integTestCluster {
 }
 integTestRunner {
   systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
+}
+
+testingConventions.naming {
+  IT {
+    baseClass 'org.elasticsearch.test.ESTestCase'
+  }
 }

qa/smoke-test-client/build.gradle

@@ -40,3 +40,13 @@ singleNodeIntegTestCluster {
 integTestCluster.dependsOn(singleNodeIntegTestRunner, 'singleNodeIntegTestCluster#stop')
 
 check.dependsOn(integTest)
+
+
+testingConventions {
+    naming.clear()
+    naming {
+        IT {
+            baseClass 'org.elasticsearch.smoketest.ESSmokeClientTestCase'
+        }
+    }
+}

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -27,3 +27,11 @@ dependencies {
     testCompile project(path: ':modules:lang-painless', configuration: 'runtime')
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
+
+testingConventions {
+    naming {
+        IT {
+            baseClass 'org.elasticsearch.ingest.AbstractScriptTestCase'
+        }
+    }
+}

qa/wildfly/build.gradle

@@ -217,3 +217,14 @@ dependencyLicenses.enabled = false
 dependenciesInfo.enabled = false
 
 thirdPartyAudit.enabled = false
+
+
+testingConventions {
+    naming.clear()
+    // We only have one "special" integration test here to connect to wildfly
+    naming {
+        IT {
+            baseClass 'org.apache.lucene.util.LuceneTestCase'
+        }
+    }
+}

server/build.gradle

@@ -161,6 +161,19 @@ forbiddenPatterns {
   exclude '**/*.st'
 }
 
+testingConventions {
+  naming.clear()
+  naming {
+    Tests {
+      baseClass "org.apache.lucene.util.LuceneTestCase"
+    }
+    IT {
+      baseClass "org.elasticsearch.test.ESIntegTestCase"
+      baseClass "org.elasticsearch.test.ESSingleNodeTestCase"
+    }
+  }
+}
+
 task generateModulesList {
   List<String> modules = project(':modules').subprojects.collect { it.name }
   modules.add('x-pack')

x-pack/plugin/ccr/build.gradle

@@ -54,3 +54,9 @@ dependencyLicenses {
 run {
     plugin xpackModule('core')
 }
+
+testingConventions.naming {
+    IT {
+        baseClass "org.elasticsearch.xpack.CcrIntegTestCase"
+    }
+}

x-pack/plugin/sql/build.gradle

@@ -1,3 +1,5 @@
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
+
 evaluationDependsOn(xpackModule('core'))
 
 apply plugin: 'elasticsearch.esplugin'
@@ -18,6 +20,12 @@ archivesBaseName = 'x-pack-sql'
 // All integration tests live in qa modules
 integTest.enabled = false
 
+task internalClusterTest(type: RandomizedTestingTask,
+        group: JavaBasePlugin.VERIFICATION_GROUP
+) {
+    include '**/*IT.class'
+}
+
 dependencies {
     // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')

x-pack/plugin/sql/qa/security/build.gradle

@@ -6,6 +6,9 @@ Project mainProject = project
 
 group = "${group}.x-pack.qa.sql.security"
 
+// Tests are pushed down to subprojects and will be checked there.
+testingConventions.enabled = false 
+
 subprojects {
   // Use resources from the parent project in subprojects
   sourceSets {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java

@@ -18,6 +18,7 @@ import static org.hamcrest.Matchers.hasSize;
 
 public class SqlActionIT extends AbstractSqlIntegTestCase {
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37191")
     public void testSqlAction() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test").get());
         client().prepareBulk()

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlClearCursorActionIT.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
+import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
@@ -16,6 +17,7 @@ import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
+@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37191")
 public class SqlClearCursorActionIT extends AbstractSqlIntegTestCase {
 
     public void testSqlClearCursorAction() throws Exception {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlDisabledIT.java

@@ -29,6 +29,7 @@ public class SqlDisabledIT extends AbstractSqlIntegTestCase {
                 .build();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37191")
     public void testSqlAction() throws Exception {
         Throwable throwable = expectThrows(Throwable.class,
                 () -> new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE).query("SHOW tables").get());

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
+import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
@@ -34,6 +35,7 @@ import static org.elasticsearch.license.XPackLicenseStateTests.randomTrialBasicS
 import static org.elasticsearch.license.XPackLicenseStateTests.randomTrialOrPlatinumMode;
 import static org.hamcrest.Matchers.equalTo;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37191")
 public class SqlLicenseIT extends AbstractLicensesIntegrationTestCase {
     @Override
     protected boolean ignoreExternalCluster() {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java

@@ -17,6 +17,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 
 public class SqlTranslateActionIT extends AbstractSqlIntegTestCase {
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37191")
     public void testSqlTranslateAction() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test").get());
         client().prepareBulk()

x-pack/qa/full-cluster-restart/build.gradle

@@ -88,6 +88,10 @@ licenseHeaders {
 forbiddenPatterns {
     exclude '**/system_key'
 }
+
+// tests are pushed down to subprojects
+testingConventions.enabled = false 
+
 /**
  * Subdirectories of this project are test rolling upgrades with various
  * configuration options based on their name.

x-pack/qa/rolling-upgrade/build.gradle

@@ -76,6 +76,9 @@ forbiddenPatterns {
   exclude '**/system_key'
 }
 
+// Tests are pushed down to subprojects 
+testingConventions.enabled = false 
+
 /**
  * Subdirectories of this project are test rolling upgrades with various
  * configuration options based on their name.

x-pack/qa/security-migrate-tests/build.gradle

@@ -31,3 +31,12 @@ integTestCluster {
     return tmpFile.exists()
   }
 }
+
+testingConventions {
+  naming.clear()
+  naming {
+    IT {
+      baseClass 'org.elasticsearch.xpack.security.MigrateToolTestCase'
+    }
+  }
+}

x-pack/qa/smoke-test-watcher/build.gradle

@@ -16,3 +16,7 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
 }
+
+integTestRunner {
+  include "**/*Tests.class"
+}

x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java

@@ -30,7 +30,7 @@ import java.util.Map;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class WatcherTemplateIT extends ESTestCase {
+public class WatcherTemplateTests extends ESTestCase {
 
     private TextTemplateEngine textTemplateEngine;
 

x-pack/qa/transport-client-tests/build.gradle

@@ -10,3 +10,13 @@ integTestCluster {
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }
+
+
+testingConventions {
+  naming.clear()
+  naming {
+    IT {
+      baseClass 'org.elasticsearch.xpack.ml.client.ESXPackSmokeClientTestCase'
+    }
+  }
+}

x-pack/transport-client/build.gradle

@@ -29,6 +29,15 @@ namingConventions {
     skipIntegTestInDisguise = true
 }
 
+testingConventions {
+    naming.clear()
+    naming {
+        Tests {
+            baseClass 'com.carrotsearch.randomizedtesting.RandomizedTest'
+        }
+    }
+}
+
 publishing {
     publications {
         nebula(MavenPublication) {