New Test annotation @NotImplementedYet

Signed-off-by: Jan Schatteman <jschatte@redhat.com>

hibernate-core/src/main/java/org/hibernate/NotImplementedYetException.java

@@ -0,0 +1,15 @@
+/*
+ * Hibernate, Relational Persistence for Idiomatic Java
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later.
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate;
+
+/**
+ * Marker interface for exceptions that indicate that something hasn't been implemented yet for a certain version
+ *
+ * @author Jan Schatteman
+ */
+public interface NotImplementedYetException {
+}

hibernate-core/src/main/java/org/hibernate/NotYetImplementedFor6Exception.java

@@ -13,7 +13,8 @@ import org.hibernate.metamodel.mapping.NonTransientException;
  *
  * todo (6.0) : prior going final, we need to find all usages of this and implement all methods (or throw a different exception)
  */
-public class NotYetImplementedFor6Exception extends RuntimeException implements NonTransientException {
+public class NotYetImplementedFor6Exception extends RuntimeException implements NonTransientException,
+		NotImplementedYetException {
 	public NotYetImplementedFor6Exception(String message) {
 		super( message );
 	}

hibernate-testing/src/main/java/org/hibernate/testing/junit5/StandardTags.java

@@ -11,6 +11,7 @@ package org.hibernate.testing.junit5;
  */
 @SuppressWarnings("unused")
 public final class StandardTags {
+	public static final String NOT_IMPLEMENTED_YET = "not-implemented-yet";
 	public static final String FAILURE_EXPECTED = "failure-expected";
 	public static final String PERF = "perf";
 	public static final String QUERY = "query";

hibernate-testing/src/main/java/org/hibernate/testing/orm/junit/NotImplementedYet.java

@@ -0,0 +1,46 @@
+/*
+ * Hibernate, Relational Persistence for Idiomatic Java
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or http://www.gnu.org/licenses/lgpl-2.1.html
+ */
+package org.hibernate.testing.orm.junit;
+
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import static org.hibernate.testing.junit5.StandardTags.NOT_IMPLEMENTED_YET;
+
+/**
+ * Marks a test method or class's tested functionality as not being implemented yet.
+ *
+ * @see NotImplementedYetExtension
+ *
+ * @author Jan Schatteman
+ */
+@Inherited
+@Target(value = { ElementType.TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+
+@Tag(NOT_IMPLEMENTED_YET)
+
+@ExtendWith( NotImplementedYetExtension.class )
+public @interface NotImplementedYet {
+
+	/**
+	 * A reason why the failure is expected
+	 */
+	String reason() default "";
+
+	/**
+	 * A version expectation by when this feature is supposed to become implemented
+	 */
+	String expectedVersion() default "";
+}

hibernate-testing/src/main/java/org/hibernate/testing/orm/junit/NotImplementedYetExtension.java

@@ -0,0 +1,97 @@
+/*
+ * Hibernate, Relational Persistence for Idiomatic Java
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or http://www.gnu.org/licenses/lgpl-2.1.html
+ */
+package org.hibernate.testing.orm.junit;
+
+import java.util.Locale;
+
+import org.hibernate.NotImplementedYetException;
+
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
+
+import org.jboss.logging.Logger;
+
+/**
+ * JUnit 5 extension used to support {@link NotImplementedYet} handling
+ *
+ * @author Jan Schatteman
+ */
+public class NotImplementedYetExtension
+		implements ExecutionCondition, AfterEachCallback, TestExecutionExceptionHandler {
+
+	private static final Logger log = Logger.getLogger( NotImplementedYetExtension.class );
+
+	private static final String NOTIMPLEMENTED_STORE_KEY = "NOT_IMPLEMENTED";
+
+	@Override
+	public void afterEach(ExtensionContext context) throws Exception {
+		log.debugf( "#afterEach(%s)", context.getDisplayName() );
+
+		class NotImplementedYetExceptionExpected extends RuntimeException {
+			private NotImplementedYetExceptionExpected() {
+				super(
+						String.format(
+								Locale.ROOT,
+								"`%s#%s` is marked as '@NotImplementedYet' but did not throw a NotImplementedYetException.\n" +
+										" Either it should or, the tested functionality has been implemented, the Test passes," +
+										" and @NotImplementedYet should be removed",
+								context.getRequiredTestClass().getName(),
+								context.getRequiredTestMethod().getName()
+						)
+				);
+			}
+		}
+
+		Throwable throwable = context.getStore( getNamespace( context ) ).remove(
+				NOTIMPLEMENTED_STORE_KEY,
+				Throwable.class
+		);
+		if ( throwable == null ) {
+			throw new NotImplementedYetExceptionExpected();
+		}
+	}
+
+	@Override
+	public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
+		log.debugf( "#handleTestExecutionException(%s)", context.getDisplayName() );
+
+		// If an exception is thrown, then it needs to be of type NotImplementedYetException
+		context.getStore( getNamespace( context ) ).put( NOTIMPLEMENTED_STORE_KEY, throwable );
+		if ( throwable instanceof NotImplementedYetException ) {
+			log.debugf( "#Captured exception %s - ignoring it", throwable );
+			return;
+		}
+		// If not, rethrow
+		throw throwable;
+	}
+
+	@Override
+	public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+		log.debugf( "#evaluateExecutionCondition(%s)", context.getDisplayName() );
+
+		if ( !context.getElement().isPresent() ) {
+			throw new RuntimeException( "Unable to determine how to handle given ExtensionContext : " + context.getDisplayName() );
+		}
+
+		// Test this in case some other annotation were extended with NotImplementedYetExtension
+		if ( !TestingUtil.hasEffectiveAnnotation( context, NotImplementedYet.class ) ) {
+			return ConditionEvaluationResult.disabled( context.getDisplayName() + " is not marked as `@NotImplementedYet`" );
+		}
+		return ConditionEvaluationResult.enabled( "Always enabled" );
+	}
+
+	private ExtensionContext.Namespace getNamespace(ExtensionContext context) {
+		return ExtensionContext.Namespace.create(
+				getClass().getName(),
+				context.getRequiredTestMethod().getClass(),
+				context.getRequiredTestMethod().getName()
+		);
+	}
+}

test-case-guide.adoc

@@ -26,7 +26,8 @@ NOTE: the test templates are generally not a good starting point for problems bu
 
 When using "test templates" you can annotate a  single test or a whole test class with one of the following  annotations:
 
-* FailureExpected - allows to skip a single test or all test of a class, because test failures are expected. The test will acutally run, but not lead to an error report. In fact if a test is marked with `@FailureExpected` and the test actually succeed an error occurs. As a parameters to this annotation a jira key is required.
-* RequiresDialect - tests methods/classes annotated with `@RequiresDialect` will only run if the current Dialect is matching the one specified on as annotation parameter. You can also specify a comment and/or jira key explaining why this test requires a certain dialect
-* RequiresDialectFeature - tests methods/classes annotated with `@RequiresDialectFeature` will only run if the current Dialect offers the specified feature. Examples for this features are `SupportsSequences`, `SupportsExpectedLobUsagePattern` or `SupportsIdentityColumns`. You can add more feature if you need to. Have a look at `DialectChecks`.
-* SkipForDialect - tests methods/classes annotated with `@SkipForDialect` will not run if  the current Dialect is matching the one specified on as annotation  parameter. You can also specify a comment and/or jira key explaining why  this test has to be skipped for the Dialect.
+* FailureExpected - allows to skip a single test or all tests of a class, because test failures are expected. The test will actually run, but not lead to an error report. In fact if a test is marked with `@FailureExpected` and the test actually succeeds, an error occurs. As a parameter to this annotation a jira key is required.
+* NotImplementedYet - test classes or methods annotated with @NotImplementedYet will run but not fail if the feature(s) that are being tested are not implemented yet for the current version. Optionally, a message and a version that is expected to have the feature already implemented can be provided as parameters.
+* RequiresDialect - tests methods/classes annotated with `@RequiresDialect` will only run if the current Dialect matches the one specified as annotation parameter. You can also specify a comment and/or jira key explaining why this test requires a certain dialect
+* RequiresDialectFeature - tests methods/classes annotated with `@RequiresDialectFeature` will only run if the current Dialect offers the specified feature. Examples for this features are `SupportsSequences`, `SupportsExpectedLobUsagePattern` or `SupportsIdentityColumns`. You can add more features if you need to. Have a look at `DialectChecks`.
+* SkipForDialect - tests methods/classes annotated with `@SkipForDialect` will not run if  the current Dialect matches the one specified as annotation  parameter. You can also specify a comment and/or jira key explaining why  this test has to be skipped for the Dialect.