Determine ansinull setting for Sybase ASE and implement proper comparison handling when it is off

hibernate-core/src/main/java/org/hibernate/dialect/Dialect.java

@@ -3103,6 +3103,10 @@ public abstract class Dialect implements ConversionContext {
 		return true;
 	}
 
+	public boolean isAnsiNullOn() {
+		return true;
+	}
+
 	/**
 	 * Renders an ordering fragment
 	 *

hibernate-core/src/main/java/org/hibernate/dialect/Sybase11Dialect.java

@@ -16,6 +16,6 @@ package org.hibernate.dialect;
 @Deprecated
 public class Sybase11Dialect extends SybaseASEDialect {
 	public Sybase11Dialect() {
-		super( 1100, false );
+		super( 1100, false, false );
 	}
 }

hibernate-core/src/main/java/org/hibernate/dialect/SybaseASE157Dialect.java

@@ -17,7 +17,7 @@ package org.hibernate.dialect;
 public class SybaseASE157Dialect extends SybaseASEDialect {
 
 	public SybaseASE157Dialect() {
-		super( 1570, false );
+		super( 1570, false, false );
 	}
 
 }

hibernate-core/src/main/java/org/hibernate/dialect/SybaseASE15Dialect.java

@@ -17,7 +17,7 @@ package org.hibernate.dialect;
 public class SybaseASE15Dialect extends SybaseASEDialect {
 
 	public SybaseASE15Dialect() {
-		super( 1500, false );
+		super( 1500, false, false );
 	}
 
 }

hibernate-core/src/main/java/org/hibernate/dialect/SybaseASEDialect.java

@@ -6,6 +6,9 @@
  */
 package org.hibernate.dialect;
 
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Types;
 import java.util.Map;
 
@@ -48,21 +51,23 @@ import static org.hibernate.exception.spi.TemplatedViolatedConstraintNameExtract
 public class SybaseASEDialect extends SybaseDialect {
 
 	private final SizeStrategy sizeStrategy;
+	private final boolean ansiNull;
 
 	public SybaseASEDialect() {
-		this( 1100, false );
+		this( 1100, false, false );
 	}
 
 	public SybaseASEDialect(DialectResolutionInfo info) {
 		this(
 				info.getDatabaseMajorVersion() * 100 + info.getDatabaseMinorVersion() * 10,
-				info.getDriverName() != null && info.getDriverName().contains( "jTDS" )
+				info.getDriverName() != null && info.getDriverName().contains( "jTDS" ),
+				isAnsiNull( info.unwrap( DatabaseMetaData.class ) )
 		);
 	}
 
-	public SybaseASEDialect(int version, boolean jtdsDriver) {
+	public SybaseASEDialect(int version, boolean jtdsDriver, boolean ansiNull) {
 		super( version, jtdsDriver );
-
+		this.ansiNull = ansiNull;
 		//On Sybase ASE, the 'bit' type cannot be null,
 		//and cannot have indexes (while we don't use
 		//tinyint to store signed bytes, we can use it
@@ -121,6 +126,28 @@ public class SybaseASEDialect extends SybaseDialect {
 		};
 	}
 
+	private static boolean isAnsiNull(DatabaseMetaData databaseMetaData) {
+		if ( databaseMetaData != null ) {
+			try (java.sql.Statement s = databaseMetaData.getConnection().createStatement() ) {
+				final ResultSet rs = s.executeQuery( "SELECT @@options" );
+				if ( rs.next() ) {
+					final byte[] optionBytes = rs.getBytes( 1 );
+					// By trial and error, enabling and disabling ansinull revealed that this bit is the indicator
+					return ( optionBytes[4] & 2 ) == 2;
+				}
+			}
+			catch (SQLException ex) {
+				// Ignore
+			}
+		}
+		return false;
+	}
+
+	@Override
+	public boolean isAnsiNullOn() {
+		return ansiNull;
+	}
+
 	@Override
 	public int getDoublePrecision() {
 		return 48;

hibernate-core/src/main/java/org/hibernate/dialect/SybaseASESqlAstTranslator.java

@@ -19,7 +19,9 @@ import org.hibernate.sql.ast.tree.Statement;
 import org.hibernate.sql.ast.tree.cte.CteStatement;
 import org.hibernate.sql.ast.tree.expression.ColumnReference;
 import org.hibernate.sql.ast.tree.expression.Expression;
+import org.hibernate.sql.ast.tree.expression.JdbcParameter;
 import org.hibernate.sql.ast.tree.expression.Literal;
+import org.hibernate.sql.ast.tree.expression.NullnessLiteral;
 import org.hibernate.sql.ast.tree.expression.SqlTuple;
 import org.hibernate.sql.ast.tree.expression.Summarization;
 import org.hibernate.sql.ast.tree.from.TableGroup;
@@ -152,11 +154,28 @@ public class SybaseASESqlAstTranslator<T extends JdbcOperation> extends Abstract
 	@Override
 	protected void renderComparison(Expression lhs, ComparisonOperator operator, Expression rhs) {
 		// I think intersect is only supported in 16.0 SP3
-//		renderComparisonEmulateIntersect( lhs, operator, rhs );
+		if ( getDialect().isAnsiNullOn() ) {
-
+			if ( getDialect().getVersion() >= 1630 ) {
+				renderComparisonEmulateIntersect( lhs, operator, rhs );
+			}
+			else {
+				renderComparisonEmulateCase( lhs, operator, rhs );
+			}
+		}
+		else {
+			// The ansinull setting only matters if using a parameter or literal and the eq operator according to the docs
+			// http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc32300.1570/html/sqlug/sqlug89.htm
+			boolean rhsNotNullPredicate =
+					lhs instanceof NullnessLiteral
+					|| lhs instanceof Literal
+					|| lhs instanceof JdbcParameter;
+			boolean lhsNotNullPredicate =
+					rhs instanceof NullnessLiteral
+					|| rhs instanceof Literal
+					|| rhs instanceof JdbcParameter;
+			if ( rhsNotNullPredicate || lhsNotNullPredicate ) {
 				lhs.accept( this );
 				appendSql( " " );
-		// This relies on the fact that Sybase usually is configured with ANSINULLS OFF
 				switch ( operator ) {
 					case DISTINCT_FROM:
 						appendSql( "<>" );
@@ -164,12 +183,39 @@ public class SybaseASESqlAstTranslator<T extends JdbcOperation> extends Abstract
 					case NOT_DISTINCT_FROM:
 						appendSql( '=' );
 						break;
+					case LESS_THAN:
+					case GREATER_THAN:
+					case LESS_THAN_OR_EQUAL:
+					case GREATER_THAN_OR_EQUAL:
+						// These operators are not affected by ansinull=off
+						lhsNotNullPredicate = false;
+						rhsNotNullPredicate = false;
 					default:
 						appendSql( operator.sqlText() );
 						break;
 				}
 				appendSql( " " );
 				rhs.accept( this );
+				if ( lhsNotNullPredicate ) {
+					appendSql( " and " );
+					lhs.accept( this );
+					appendSql( " is not null" );
+				}
+				if ( rhsNotNullPredicate ) {
+					appendSql( " and " );
+					rhs.accept( this );
+					appendSql( " is not null" );
+				}
+			}
+			else {
+				if ( getDialect().getVersion() >= 1630 ) {
+					renderComparisonEmulateIntersect( lhs, operator, rhs );
+				}
+				else {
+					renderComparisonEmulateCase( lhs, operator, rhs );
+				}
+			}
+		}
 	}
 
 	@Override

hibernate-core/src/main/java/org/hibernate/sql/ast/spi/AbstractSqlAstTranslator.java

@@ -1994,6 +1994,40 @@ public abstract class AbstractSqlAstTranslator<T extends JdbcOperation> implemen
 		}
 	}
 
+	protected void renderComparisonEmulateCase(Expression lhs, ComparisonOperator operator, Expression rhs) {
+		switch ( operator ) {
+			case DISTINCT_FROM:
+				appendSql( "case when " );
+				lhs.accept( this );
+				appendSql( " = " );
+				rhs.accept( this );
+				appendSql( " or " );
+				lhs.accept( this );
+				appendSql( " is null and " );
+				rhs.accept( this );
+				appendSql( " is null then 0 else 1 end = 1" );
+				break;
+			case NOT_DISTINCT_FROM:
+				appendSql( "case when " );
+				lhs.accept( this );
+				appendSql( " = " );
+				rhs.accept( this );
+				appendSql( " or " );
+				lhs.accept( this );
+				appendSql( " is null and " );
+				rhs.accept( this );
+				appendSql( " is null then 0 else 1 end = 0" );
+				break;
+			default:
+				lhs.accept( this );
+				appendSql( ' ' );
+				appendSql( operator.sqlText() );
+				appendSql( ' ' );
+				rhs.accept( this );
+				break;
+		}
+	}
+
 	protected void renderComparisonEmulateIntersect(Expression lhs, ComparisonOperator operator, Expression rhs) {
 		switch ( operator ) {
 			case DISTINCT_FROM: