HHH-9800 Numerous hibernate-infinispan tests continue to fail transiently

The original cause was JGRP-1931, but this replaces fixed delays with eventual asserts (with 10 seconds timeouts).

hibernate-infinispan/src/test/java/org/hibernate/test/cache/infinispan/AbstractGeneralDataRegionTestCase.java

@@ -8,6 +8,8 @@ package org.hibernate.test.cache.infinispan;
 
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 
 import org.hibernate.boot.registry.StandardServiceRegistry;
 import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
@@ -19,9 +21,9 @@ import org.hibernate.test.cache.infinispan.util.CacheTestUtil;
 import org.infinispan.AdvancedCache;
 import org.infinispan.transaction.tm.BatchModeTransactionManager;
 import org.jboss.logging.Logger;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import static org.hibernate.test.cache.infinispan.util.CacheTestUtil.assertEqualsEventually;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
@@ -59,7 +61,6 @@ public abstract class AbstractGeneralDataRegionTestCase extends AbstractRegionIm
 	}
 
 	@Test
-	@Ignore // currently ignored because of HHH-9800
 	public void testEvict() throws Exception {
 		evictOrRemoveTest();
 	}
@@ -81,7 +82,7 @@ public abstract class AbstractGeneralDataRegionTestCase extends AbstractRegionIm
 			// Sleep a bit to avoid concurrent FLUSH problem
 			avoidConcurrentFlush();
 
-			GeneralDataRegion localRegion = (GeneralDataRegion) createRegion(
+			final GeneralDataRegion localRegion = (GeneralDataRegion) createRegion(
 					regionFactory,
 					getStandardRegionName( REGION_PREFIX ),
 					properties,
@@ -93,7 +94,7 @@ public abstract class AbstractGeneralDataRegionTestCase extends AbstractRegionIm
 					getCacheTestSupport()
 			);
 
-			GeneralDataRegion remoteRegion = (GeneralDataRegion) createRegion(
+			final GeneralDataRegion remoteRegion = (GeneralDataRegion) createRegion(
 					regionFactory,
 					getStandardRegionName( REGION_PREFIX ),
 					properties,
@@ -103,22 +104,29 @@ public abstract class AbstractGeneralDataRegionTestCase extends AbstractRegionIm
 			assertNull( "remote is clean", remoteRegion.get( KEY ) );
 
 			regionPut( localRegion );
-			sleep( 250 );
-			assertEquals( VALUE1, localRegion.get( KEY ) );
 
-			// allow async propagation
+			Callable<Object> getFromLocalRegion = new Callable<Object>() {
-			sleep( 250 );
+				@Override
+				public Object call() throws Exception {
+					return localRegion.get(KEY);
+				}
+			};
+			Callable<Object> getFromRemoteRegion = new Callable<Object>() {
+				@Override
+				public Object call() throws Exception {
+					return remoteRegion.get(KEY);
+				}
+			};
+
+			assertEqualsEventually(VALUE1, getFromLocalRegion, 10, TimeUnit.SECONDS);
 			Object expected = invalidation ? null : VALUE1;
-			assertEquals( expected, remoteRegion.get( KEY ) );
+			assertEqualsEventually(expected, getFromRemoteRegion, 10, TimeUnit.SECONDS);
 
-			regionEvict( localRegion );
+			regionEvict(localRegion);
 
-			// allow async propagation
+			assertEqualsEventually(null, getFromLocalRegion, 10, TimeUnit.SECONDS);
-			sleep( 250 );
+			assertEqualsEventually(null, getFromRemoteRegion, 10, TimeUnit.SECONDS);
-			assertEquals( null, localRegion.get( KEY ) );
+		} finally {
-			assertEquals( null, remoteRegion.get( KEY ) );
-		}
-		finally {
 			StandardServiceRegistryBuilder.destroy( registry1 );
 			StandardServiceRegistryBuilder.destroy( registry2 );
 		}

hibernate-infinispan/src/test/java/org/hibernate/test/cache/infinispan/util/CacheTestUtil.java

@@ -9,6 +9,9 @@ package org.hibernate.test.cache.infinispan.util;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.hibernate.boot.internal.SessionFactoryBuilderImpl;
 import org.hibernate.boot.internal.SessionFactoryOptionsImpl;
@@ -18,8 +21,8 @@ import org.hibernate.cache.infinispan.InfinispanRegionFactory;
 import org.hibernate.cfg.AvailableSettings;
 import org.hibernate.engine.config.spi.ConfigurationService;
 import org.hibernate.engine.config.spi.StandardConverters;
+import org.hibernate.internal.util.compare.EqualsHelper;
 import org.hibernate.service.ServiceRegistry;
-
 import org.hibernate.test.cache.infinispan.functional.SingleNodeTestCase;
 
 /**
@@ -134,10 +137,42 @@ public class CacheTestUtil {
       return properties;
    }
 
+   /**
+    * Executes {@link #assertEqualsEventually(Object, Callable, long, TimeUnit)} without time limit.
+    * @param expected
+    * @param callable
+    * @param <T>
+    */
+   public static <T> void assertEqualsEventually(T expected, Callable<T> callable) throws Exception {
+      assertEqualsEventually(expected, callable, -1, TimeUnit.SECONDS);
+   }
+
+   /**
+    * Periodically calls callable and compares returned value with expected value. If the value matches to expected,
+    * the method returns. If callable throws an exception, this is propagated. If the returned value does not match to
+    * expected before timeout, {@link TimeoutException} is thrown.
+    * @param expected
+    * @param callable
+    * @param timeout If non-positive, there is no limit.
+    * @param timeUnit
+    * @param <T>
+    */
+   public static <T> void assertEqualsEventually(T expected, Callable<T> callable, long timeout, TimeUnit timeUnit) throws Exception {
+      long now, deadline = timeout <= 0 ? Long.MAX_VALUE : System.currentTimeMillis() + timeUnit.toMillis(timeout);
+      for (;;) {
+         T value = callable.call();
+         if (EqualsHelper.equals(value, expected)) return;
+         now = System.currentTimeMillis();
+         if (now < deadline) {
+            Thread.sleep(Math.min(100, deadline - now));
+         } else break;
+      }
+      throw new TimeoutException();
+   }
+
    /**
     * Prevent instantiation.
     */
    private CacheTestUtil() {
    }
-
 }