Detecting tests which was not added to TestSuiteOrdering

Even test is skipped by `@Disabled` annotation
newer surefire schedule it for execution.

When tests are missing in TestSuiteOrdering
are executed as one of first tests in unpredictable order.

its/core-it-suite/pom.xml

@@ -498,6 +498,8 @@ under the License.
           <includes>
             <include>**/MavenIT*.java</include>
           </includes>
+          <!-- test annotated by @Tag("disabled") will be skipped from executions -->
+          <excludedGroups>disabled</excludedGroups>
           <forkCount>0</forkCount>
           <reuseForks>true</reuseForks>
           <skip>true</skip>

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng4356NoNeedlessRelookupFromActiveCollectionTest.java

@@ -25,7 +25,7 @@ import org.apache.maven.shared.verifier.Verifier;
 import java.io.File;
 import java.util.Properties;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -33,7 +33,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author Benjamin Bentmann
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng4356NoNeedlessRelookupFromActiveCollectionTest
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng4565MultiConditionProfileActivationTest.java

@@ -24,7 +24,7 @@ import org.apache.maven.shared.verifier.Verifier;
 
 import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -33,7 +33,7 @@ import org.junit.jupiter.api.Test;
  * When multiple activators are present in a profile they should be AND'd. All activator must
  * conditions must be satisfied in order for the profile to be activated.
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng4565MultiConditionProfileActivationTest
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng4761PluginLevelDependencyScopesTest.java

@@ -24,7 +24,7 @@ import org.apache.maven.shared.verifier.Verifier;
 
 import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author jdcasey
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng4761PluginLevelDependencyScopesTest
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5206PlexusLifecycleHonoured.java

@@ -24,7 +24,7 @@ import org.apache.maven.shared.verifier.Verifier;
 
 import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author Olivier Lamy
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng5206PlexusLifecycleHonoured
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5418FileProjectPropertiesActivatorTest.java

@@ -24,7 +24,7 @@ import org.apache.maven.shared.verifier.Verifier;
 
 import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author Olivier Lamy
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng5418FileProjectPropertiesActivatorTest
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5503ZipInReactorTest.java

@@ -25,6 +25,7 @@ import org.apache.maven.shared.verifier.Verifier;
 import java.io.File;
 
 import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -32,6 +33,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author jvz
  */
+@Tag( "disabled" )
 @Disabled
 public class MavenITmng5503ZipInReactorTest
     extends AbstractMavenIntegrationTestCase

its/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5557ProperlyRestrictedReactor.java

@@ -25,7 +25,7 @@ import org.apache.maven.shared.verifier.VerificationException;
 
 import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test;
  *
  * @author jvanzyl
  */
-@Disabled
+@Tag("disabled")
 public class MavenITmng5557ProperlyRestrictedReactor
     extends AbstractMavenIntegrationTestCase
 {

its/core-it-suite/src/test/java/org/apache/maven/it/TestSuiteOrdering.java

@@ -22,14 +22,17 @@ package org.apache.maven.it;
 import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.maven.shared.verifier.Verifier;
 import org.junit.jupiter.api.ClassDescriptor;
 import org.junit.jupiter.api.ClassOrderer;
 import org.junit.jupiter.api.ClassOrdererContext;
+import org.junit.jupiter.api.Tag;
 
 /**
  * The Core IT suite.
@@ -39,6 +42,9 @@ public class TestSuiteOrdering implements ClassOrderer
 
     private static PrintStream out = System.out;
 
+    // store missed test to show info only once
+    private final static List<Class<?>> MISSED_TESTS = new ArrayList<>();
+
     final Map<Class<?>, Integer> tests = new HashMap<>();
 
     private static void infoProperty( PrintStream info, String property )
@@ -746,21 +752,36 @@ public class TestSuiteOrdering implements ClassOrderer
          */
     }
 
-    void addTestSuite( Class<?> clazz ) {
+    void addTestSuite( Class<?> clazz )
+    {
         addTestSuite( clazz, tests.size() );
     }
 
-    void addTestSuite( Class<?> clazz, int order ) {
+    void addTestSuite( Class<?> clazz, int order )
+    {
         tests.put( clazz, order );
     }
 
-    int getIndex( ClassDescriptor cd ) {
+    int getIndex( ClassDescriptor cd )
+    {
         Integer i = tests.get( cd.getTestClass() );
         return i != null ? i : -1;
     }
 
     public void orderClasses( ClassOrdererContext context )
     {
+        context.getClassDescriptors().stream()
+            .filter( cd -> !MISSED_TESTS.contains( cd.getTestClass() ) )
+            .filter( cd -> getIndex( cd ) == -1 )
+            .filter( cd -> cd.findRepeatableAnnotations( Tag.class ).stream()
+                .noneMatch( t -> "disabled".equals( t.value() ) ) )
+            .forEach( cd -> {
+                out.println( "Test " + cd.getTestClass()
+                                 + " is not present in TestSuiteOrdering " + System.lineSeparator()
+                                 + "\t- please add it or annotate with @Tag(\"disabled\")" + System.lineSeparator() );
+                MISSED_TESTS.add( cd.getTestClass() );
+            } );
+
         context.getClassDescriptors().sort( Comparator.comparing( this::getIndex ) );
     }