HHH-16125 fix the encoding of enum arrays to varbinary

hibernate-core/src/main/java/org/hibernate/type/descriptor/java/ArrayJavaType.java

@@ -303,13 +303,11 @@ public class ArrayJavaType<T> extends AbstractArrayJavaType<T[], T> {
 			return (X) value;
 		}
 		else if ( type == byte[].class ) {
-			// byte[] can only be requested if the value should be serialized
-			return (X) SerializationHelper.serialize( value );
+			return (X) toBytes( value );
 		}
 		else if ( type == BinaryStream.class ) {
-			// BinaryStream can only be requested if the value should be serialized
 			//noinspection unchecked
-			return (X) new BinaryStreamImpl( SerializationHelper.serialize( value ) );
+			return (X) new BinaryStreamImpl( toBytes( value ) );
 		}
 		else if ( type.isArray() ) {
 			final Class<?> preferredJavaTypeClass = type.getComponentType();
@@ -360,19 +358,52 @@ public class ArrayJavaType<T> extends AbstractArrayJavaType<T[], T> {
 			return wrapped;
 		}
 		else if ( value instanceof byte[] ) {
-			// When the value is a byte[], this is a deserialization request
-			//noinspection unchecked
-			return (T[]) SerializationHelper.deserialize( (byte[]) value );
+			return fromBytes( (byte[]) value );
 		}
 		else if ( value instanceof BinaryStream ) {
 			// When the value is a BinaryStream, this is a deserialization request
-			//noinspection unchecked
-			return (T[]) SerializationHelper.deserialize( ( (BinaryStream) value ).getBytes() );
+			return fromBytes( ( (BinaryStream) value ).getBytes() );
 		}
 
 		throw unknownWrap( value.getClass() );
 	}
 
+	private static <T> byte[] toBytes(T[] value) {
+		if ( value.getClass().getComponentType().isEnum() ) {
+			final byte[] array = new byte[value.length];
+			for (int i = 0; i < value.length; i++ ) {
+				// encode null enum value as -1
+				array[i] = value[i] == null ? -1 : (byte) ((Enum<?>) value[i]).ordinal();
+			}
+			return array;
+
+		}
+		else {
+			// byte[] can only be requested if the value should be serialized
+			return SerializationHelper.serialize( value );
+		}
+	}
+
+	private T[] fromBytes(byte[] value) {
+		Class<T> elementClass = getElementJavaType().getJavaTypeClass();
+		byte[] bytes = value;
+		if ( elementClass.isEnum() ) {
+			final Object[] array = (Object[]) Array.newInstance( elementClass, bytes.length );
+			for (int i = 0; i < bytes.length; i++ ) {
+				// null enum value was encoded as -1
+				array[i] = bytes[i] == -1 ? null : elementClass.getEnumConstants()[bytes[i]];
+			}
+			//noinspection unchecked
+			return (T[]) array;
+
+		}
+		else {
+			// When the value is a byte[], this is a deserialization request
+			//noinspection unchecked
+			return (T[]) SerializationHelper.deserialize(value);
+		}
+	}
+
 	private static class ArrayMutabilityPlan<T> implements MutabilityPlan<T[]> {
 
 		private final Class<T> componentClass;

hibernate-core/src/test/java/org/hibernate/orm/test/type/EnumArrayTest.java

@@ -13,7 +13,6 @@ import org.hibernate.dialect.HSQLDialect;
 import org.hibernate.dialect.MySQLDialect;
 import org.hibernate.dialect.OracleDialect;
 import org.hibernate.dialect.SybaseASEDialect;
-import org.hibernate.internal.util.SerializationHelper;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
@@ -146,12 +145,11 @@ public class EnumArrayTest extends BaseNonConfigCoreFunctionalTestCase {
 			return array;
 		}
 		else {
-			// We have to always serialize the Byte[] since it can contain nulls, but VARBINARY can't
-			final Byte[] array = new Byte[enums.length];
+			final byte[] array = new byte[enums.length];
 			for ( int i = 0; i < enums.length; i++ ) {
-				array[i] = enums[i] == null ? null : (byte) enums[i].ordinal();
+				array[i] = enums[i] == null ? -1 : (byte) enums[i].ordinal();
 			}
-			return SerializationHelper.serialize( array );
+			return array;
 		}
 	}