Improved DateQuery and removed 'OR' in the SQL (#2302)

* improved DateQuery and removed 'OR' in the SQL

* improved DateQuery and removed 'OR' in the SQL

* Date handling cleanup (#2316)

* Date handling cleanup

* Fix codecov

* One more fix

* Fix tests

Co-authored-by: James Agnew <jamesagnew@gmail.com>

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/DateParam.java

@@ -50,6 +50,7 @@ public class DateParam extends BaseParamWithPrefix<DateParam> implements /*IQuer
 	 * Constructor
 	 */
 	public DateParam() {
+		super();
 	}
 
 	/**

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderDate.java

@@ -126,6 +126,9 @@ public class PredicateBuilderDate extends BasePredicateBuilder implements IPredi
 		if (theParam instanceof DateParam) {
 			DateParam date = (DateParam) theParam;
 			if (!date.isEmpty()) {
+				if (theOperation == SearchFilterParser.CompareOperation.ne) {
+					date = new DateParam(ParamPrefixEnum.EQUAL, date.getValueAsString());
+				}
 				DateRangeParam range = new DateRangeParam(date);
 				p = createPredicateDateFromRange(theBuilder,
 					theFrom,
@@ -152,6 +155,7 @@ public class PredicateBuilderDate extends BasePredicateBuilder implements IPredi
 		return theDateParam == null || theDateParam.getPrecision().ordinal() == TemporalPrecisionEnum.DAY.ordinal();
 	}
 
+	@SuppressWarnings("unchecked")
 	private Predicate createPredicateDateFromRange(CriteriaBuilder theBuilder,
 																  From<?, ResourceIndexedSearchParamDate> theFrom,
 																  DateRangeParam theRange,
@@ -191,36 +195,60 @@ public class PredicateBuilderDate extends BasePredicateBuilder implements IPredi
 		}
 
 		if (operation == SearchFilterParser.CompareOperation.lt) {
-			if (lowerBoundInstant == null) {
+			// use lower bound first
-				throw new InvalidRequestException("lowerBound value not correctly specified for compare operation");
+			if (lowerBoundInstant != null) {
-			}
+					// the value has been reduced one in this case
-			//im like 80% sure this should be ub and not lb, as it is an UPPER bound.
+					lb = theBuilder.lessThanOrEqualTo(theFrom.get(lowValueField), genericLowerBound);
-			lb = theBuilder.lessThan(theFrom.get(lowValueField), genericLowerBound);
+			} else {
-		} else if (operation == SearchFilterParser.CompareOperation.le) {
+				if (upperBoundInstant != null) {
-			if (upperBoundInstant == null) {
+					ub = theBuilder.lessThanOrEqualTo(theFrom.get(lowValueField), genericUpperBound);
-				throw new InvalidRequestException("upperBound value not correctly specified for compare operation");
+				} else {
-			}
+					throw new InvalidRequestException("lowerBound and upperBound value not correctly specified for compare theOperation");
-			//im like 80% sure this should be ub and not lb, as it is an UPPER bound.
-			lb = theBuilder.lessThanOrEqualTo(theFrom.get(highValueField), genericUpperBound);
-		} else if (operation == SearchFilterParser.CompareOperation.gt) {
-			if (upperBoundInstant == null) {
-				throw new InvalidRequestException("upperBound value not correctly specified for compare operation");
-			}
-			lb = theBuilder.greaterThan(theFrom.get(highValueField), genericUpperBound);
-			} else if (operation == SearchFilterParser.CompareOperation.ge) {
-				if (lowerBoundInstant == null) {
-					throw new InvalidRequestException("lowerBound value not correctly specified for compare operation");
 				}
-				lb = theBuilder.greaterThanOrEqualTo(theFrom.get(lowValueField), genericLowerBound);
+			}
-			} else if (operation == SearchFilterParser.CompareOperation.ne) {
+		} else if (operation == SearchFilterParser.CompareOperation.le) {
+				// use lower bound first
+				if (lowerBoundInstant != null) {
+					lb = theBuilder.lessThanOrEqualTo(theFrom.get(lowValueField), genericLowerBound);
+				} else {
+					if (upperBoundInstant != null) {
+						ub = theBuilder.lessThanOrEqualTo(theFrom.get(lowValueField), genericUpperBound);
+					} else {
+						throw new InvalidRequestException("lowerBound and upperBound value not correctly specified for compare theOperation");
+					}
+				}
+		} else if (operation == SearchFilterParser.CompareOperation.gt) {
+			// use upper bound first, e.g value between 6 and 10
+			// gt7 true,    10>7, gt11 false,  10>11 false, gt5 true,    10>5
+			if (upperBoundInstant != null) {
+				ub = theBuilder.greaterThanOrEqualTo(theFrom.get(highValueField), genericUpperBound);
+			} else {
+				if (lowerBoundInstant != null) {
+					lb = theBuilder.greaterThanOrEqualTo(theFrom.get(highValueField), genericLowerBound);
+				} else {
+					throw new InvalidRequestException("upperBound and lowerBound value not correctly specified for compare theOperation");
+				}
+			} 
+		} else if (operation == SearchFilterParser.CompareOperation.ge) {
+			// use upper bound first, e.g value between 6 and 10
+			// gt7 true,    10>7, gt11 false,  10>11 false, gt5 true,    10>5
+			if (upperBoundInstant != null) {
+				ub = theBuilder.greaterThanOrEqualTo(theFrom.get(highValueField), genericUpperBound);;
+			} else {
+				if (lowerBoundInstant != null) {
+					lb = theBuilder.greaterThanOrEqualTo(theFrom.get(highValueField), genericLowerBound);
+				} else {
+					throw new InvalidRequestException("upperBound and lowerBound value not correctly specified for compare theOperation");
+				}
+			} 
+		} else if (operation == SearchFilterParser.CompareOperation.ne) {
 			if ((lowerBoundInstant == null) ||
 				(upperBoundInstant == null)) {
 				throw new InvalidRequestException("lowerBound and/or upperBound value not correctly specified for compare operation");
 			}
 			lt = theBuilder.lessThan(theFrom.get(lowValueField), genericLowerBound);
 			gt = theBuilder.greaterThan(theFrom.get(highValueField), genericUpperBound);
-			lb = theBuilder.or(lt,
+			lb = theBuilder.or(lt, gt);
-				gt);
 		} else if ((operation == SearchFilterParser.CompareOperation.eq) || (operation == null)) {
 			if (lowerBoundInstant != null) {
 				gt = theBuilder.greaterThanOrEqualTo(theFrom.get(lowValueField), genericLowerBound);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java

@@ -102,6 +102,7 @@ import java.util.ListIterator;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static ca.uhn.fhir.jpa.search.builder.QueryStack.fromOperation;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.apache.commons.lang3.StringUtils.trim;
@@ -267,12 +268,12 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 	private Predicate addPredicateReferenceWithChain(String theResourceName, String theParamName, List<? extends IQueryParameterType> theList, From<?, ResourceLink> theJoin, List<Predicate> theCodePredicates, ReferenceParam theReferenceParam, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) {
 
 		/*
-		  * Which resource types can the given chained parameter actually link to? This might be a list
+		 * Which resource types can the given chained parameter actually link to? This might be a list
-		  * where the chain is unqualified, as in: Observation?subject.identifier=(...)
+		 * where the chain is unqualified, as in: Observation?subject.identifier=(...)
-		  * since subject can link to several possible target types.
+		 * since subject can link to several possible target types.
-		  *
+		 *
-		  * If the user has qualified the chain, as in: Observation?subject:Patient.identifier=(...)
+		 * If the user has qualified the chain, as in: Observation?subject:Patient.identifier=(...)
-		  * this is just a simple 1-entry list.
+		 * this is just a simple 1-entry list.
 		 */
 		final List<Class<? extends IBaseResource>> resourceTypes = determineCandidateResourceTypesForChain(theResourceName, theParamName, theReferenceParam);
 
@@ -593,7 +594,14 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 					switch (nextParamDef.getParamType()) {
 						case DATE:
 							for (List<? extends IQueryParameterType> nextAnd : theAndOrParams) {
-								myPredicateBuilder.addPredicateDate(theResourceName, nextParamDef, nextAnd, null, theRequestPartitionId);
+								// FT: 2021-01-18 use operation 'gt', 'ge', 'le' or 'lt' 
+								// to create the predicateDate instead of generic one with operation = null 
+								SearchFilterParser.CompareOperation operation = null;
+								if (nextAnd.size() > 0) {
+									DateParam param = (DateParam) nextAnd.get(0);
+									operation = ca.uhn.fhir.jpa.search.builder.QueryStack.toOperation(param.getPrefix());
+								}
+								myPredicateBuilder.addPredicateDate(theResourceName, nextParamDef, nextAnd, operation, theRequestPartitionId);
 							}
 							break;
 						case QUANTITY:
@@ -716,67 +724,58 @@ class PredicateBuilderReference extends BasePredicateBuilder {
 	private Predicate processFilterParameter(SearchFilterParser.FilterParameter theFilter,
 														  String theResourceName, RequestDetails theRequest, RequestPartitionId theRequestPartitionId) {
 
+		if (theFilter.getParamPath().getName().equals(Constants.PARAM_SOURCE)) {
+			TokenParam param = new TokenParam();
+			param.setValueAsQueryToken(null, null, null, theFilter.getValue());
+			return addPredicateSource(Collections.singletonList(param), theFilter.getOperation(), theRequest);
+		} else if (theFilter.getParamPath().getName().equals(IAnyResource.SP_RES_ID)) {
+			TokenParam param = new TokenParam();
+			param.setValueAsQueryToken(null,
+				null,
+				null,
+				theFilter.getValue());
+			return myPredicateBuilder.addPredicateResourceId(Collections.singletonList(Collections.singletonList(param)), myResourceName, theFilter.getOperation(), theRequestPartitionId);
+		} else if (theFilter.getParamPath().getName().equals(IAnyResource.SP_RES_LANGUAGE)) {
+			return addPredicateLanguage(Collections.singletonList(Collections.singletonList(new StringParam(theFilter.getValue()))),
+				theFilter.getOperation());
+		}
+
 		RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(theResourceName, theFilter.getParamPath().getName());
 
 		if (searchParam == null) {
 			throw new InvalidRequestException("Invalid search parameter specified, " + theFilter.getParamPath().getName() + ", for resource type " + theResourceName);
-		} else if (searchParam.getName().equals(IAnyResource.SP_RES_ID)) {
-			if (searchParam.getParamType() == RestSearchParameterTypeEnum.TOKEN) {
-				TokenParam param = new TokenParam();
-				param.setValueAsQueryToken(null,
-					null,
-					null,
-					theFilter.getValue());
-				return myPredicateBuilder.addPredicateResourceId(Collections.singletonList(Collections.singletonList(param)), myResourceName, theFilter.getOperation(), theRequestPartitionId);
-			} else {
-				throw new InvalidRequestException("Unexpected search parameter type encountered, expected token type for _id search");
-			}
-		} else if (searchParam.getName().equals(IAnyResource.SP_RES_LANGUAGE)) {
-			if (searchParam.getParamType() == RestSearchParameterTypeEnum.STRING) {
-				return addPredicateLanguage(Collections.singletonList(Collections.singletonList(new StringParam(theFilter.getValue()))),
-					theFilter.getOperation());
-			} else {
-				throw new InvalidRequestException("Unexpected search parameter type encountered, expected string type for language search");
-			}
-		} else if (searchParam.getName().equals(Constants.PARAM_SOURCE)) {
-			if (searchParam.getParamType() == RestSearchParameterTypeEnum.TOKEN) {
-				TokenParam param = new TokenParam();
-				param.setValueAsQueryToken(null, null, null, theFilter.getValue());
-				return addPredicateSource(Collections.singletonList(param), theFilter.getOperation(), theRequest);
-			} else {
-				throw new InvalidRequestException("Unexpected search parameter type encountered, expected token type for _id search");
-			}
-		} else {
-			RestSearchParameterTypeEnum typeEnum = searchParam.getParamType();
-			if (typeEnum == RestSearchParameterTypeEnum.URI) {
-				return myPredicateBuilder.addPredicateUri(theResourceName, searchParam, Collections.singletonList(new UriParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.STRING) {
-				return myPredicateBuilder.addPredicateString(theResourceName, searchParam, Collections.singletonList(new StringParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.DATE) {
-				return myPredicateBuilder.addPredicateDate(theResourceName, searchParam, Collections.singletonList(new DateParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.NUMBER) {
-				return myPredicateBuilder.addPredicateNumber(theResourceName, searchParam, Collections.singletonList(new NumberParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.REFERENCE) {
-				String paramName = theFilter.getParamPath().getName();
-				SearchFilterParser.CompareOperation operation = theFilter.getOperation();
-				String resourceType = null; // The value can either have (Patient/123) or not have (123) a resource type, either way it's not needed here
-				String chain = (theFilter.getParamPath().getNext() != null) ? theFilter.getParamPath().getNext().toString() : null;
-				String value = theFilter.getValue();
-				ReferenceParam referenceParam = new ReferenceParam(resourceType, chain, value);
-				return addPredicate(theResourceName, paramName, Collections.singletonList(referenceParam), operation, theRequest, theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.QUANTITY) {
-				return myPredicateBuilder.addPredicateQuantity(theResourceName, searchParam, Collections.singletonList(new QuantityParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
-			} else if (typeEnum == RestSearchParameterTypeEnum.COMPOSITE) {
-				throw new InvalidRequestException("Composite search parameters not currently supported with _filter clauses");
-			} else if (typeEnum == RestSearchParameterTypeEnum.TOKEN) {
-				TokenParam param = new TokenParam();
-				param.setValueAsQueryToken(null,
-					null,
-					null,
-					theFilter.getValue());
-				return myPredicateBuilder.addPredicateToken(theResourceName, searchParam, Collections.singletonList(param), theFilter.getOperation(), theRequestPartitionId);
-			}
 		}
+
+		RestSearchParameterTypeEnum typeEnum = searchParam.getParamType();
+		if (typeEnum == RestSearchParameterTypeEnum.URI) {
+			return myPredicateBuilder.addPredicateUri(theResourceName, searchParam, Collections.singletonList(new UriParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.STRING) {
+			return myPredicateBuilder.addPredicateString(theResourceName, searchParam, Collections.singletonList(new StringParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.DATE) {
+			return myPredicateBuilder.addPredicateDate(theResourceName, searchParam, Collections.singletonList(new DateParam(fromOperation(theFilter.getOperation()), theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.NUMBER) {
+			return myPredicateBuilder.addPredicateNumber(theResourceName, searchParam, Collections.singletonList(new NumberParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.REFERENCE) {
+			String paramName = theFilter.getParamPath().getName();
+			SearchFilterParser.CompareOperation operation = theFilter.getOperation();
+			String resourceType = null; // The value can either have (Patient/123) or not have (123) a resource type, either way it's not needed here
+			String chain = (theFilter.getParamPath().getNext() != null) ? theFilter.getParamPath().getNext().toString() : null;
+			String value = theFilter.getValue();
+			ReferenceParam referenceParam = new ReferenceParam(resourceType, chain, value);
+			return addPredicate(theResourceName, paramName, Collections.singletonList(referenceParam), operation, theRequest, theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.QUANTITY) {
+			return myPredicateBuilder.addPredicateQuantity(theResourceName, searchParam, Collections.singletonList(new QuantityParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+		} else if (typeEnum == RestSearchParameterTypeEnum.COMPOSITE) {
+			throw new InvalidRequestException("Composite search parameters not currently supported with _filter clauses");
+		} else if (typeEnum == RestSearchParameterTypeEnum.TOKEN) {
+			TokenParam param = new TokenParam();
+			param.setValueAsQueryToken(null,
+				null,
+				null,
+				theFilter.getValue());
+			return myPredicateBuilder.addPredicateToken(theResourceName, searchParam, Collections.singletonList(param), theFilter.getOperation(), theRequestPartitionId);
+		}
+
 		return null;
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java

@@ -89,6 +89,9 @@ import com.healthmarketscience.sqlbuilder.OrderObject;
 import com.healthmarketscience.sqlbuilder.SelectQuery;
 import com.healthmarketscience.sqlbuilder.Subquery;
 import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn;
+import org.apache.commons.collections4.BidiMap;
+import org.apache.commons.collections4.bidimap.DualHashBidiMap;
+import org.apache.commons.collections4.bidimap.UnmodifiableBidiMap;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.commons.lang3.tuple.Pair;
@@ -112,12 +115,29 @@ import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 public class QueryStack {
 
 	private static final Logger ourLog = LoggerFactory.getLogger(QueryStack.class);
+	private static final BidiMap<SearchFilterParser.CompareOperation, ParamPrefixEnum> ourCompareOperationToParamPrefix;
+
+	static {
+		DualHashBidiMap<SearchFilterParser.CompareOperation, ParamPrefixEnum> compareOperationToParamPrefix = new DualHashBidiMap<>();
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ap, ParamPrefixEnum.APPROXIMATE);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eq, ParamPrefixEnum.EQUAL);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.gt, ParamPrefixEnum.GREATERTHAN);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ge, ParamPrefixEnum.GREATERTHAN_OR_EQUALS);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.lt, ParamPrefixEnum.LESSTHAN);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.le, ParamPrefixEnum.LESSTHAN_OR_EQUALS);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.ne, ParamPrefixEnum.NOT_EQUAL);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.eb, ParamPrefixEnum.ENDS_BEFORE);
+		compareOperationToParamPrefix.put(SearchFilterParser.CompareOperation.sa, ParamPrefixEnum.STARTS_AFTER);
+		ourCompareOperationToParamPrefix = UnmodifiableBidiMap.unmodifiableBidiMap(compareOperationToParamPrefix);
+	}
+
 	private final ModelConfig myModelConfig;
 	private final FhirContext myFhirContext;
 	private final SearchQueryBuilder mySqlBuilder;
@@ -175,7 +195,6 @@ public class QueryStack {
 		mySqlBuilder.addSortDate(resourceTablePredicateBuilder.getColumnLastUpdated(), theAscending);
 	}
 
-
 	public void addSortOnNumber(String theResourceName, String theParamName, boolean theAscending) {
 		BaseJoiningPredicateBuilder firstPredicateBuilder = mySqlBuilder.getOrCreateFirstPredicateBuilder();
 		NumberPredicateBuilder sortPredicateBuilder = mySqlBuilder.addNumberPredicateBuilder(firstPredicateBuilder.getResourceIdColumn());
@@ -217,7 +236,6 @@ public class QueryStack {
 		mySqlBuilder.addSortNumeric(sortPredicateBuilder.getColumnTargetResourceId(), theAscending);
 	}
 
-
 	public void addSortOnString(String theResourceName, String theParamName, boolean theAscending) {
 		BaseJoiningPredicateBuilder firstPredicateBuilder = mySqlBuilder.getOrCreateFirstPredicateBuilder();
 		StringPredicateBuilder sortPredicateBuilder = mySqlBuilder.addStringPredicateBuilder(firstPredicateBuilder.getResourceIdColumn());
@@ -246,7 +264,6 @@ public class QueryStack {
 		mySqlBuilder.addSortString(sortPredicateBuilder.getColumnValue(), theAscending);
 	}
 
-
 	@SuppressWarnings("unchecked")
 	private <T extends BaseJoiningPredicateBuilder> PredicateBuilderCacheLookupResult<T> createOrReusePredicateBuilder(PredicateBuilderTypeEnum theType, DbColumn theSourceJoinColumn, String theParamName, Supplier<T> theFactoryMethod) {
 		boolean cacheHit = false;
@@ -299,7 +316,6 @@ public class QueryStack {
 		return orCondidtion;
 	}
 
-
 	private Condition createPredicateCompositePart(@Nullable DbColumn theSourceJoinColumn, String theResourceName, RuntimeSearchParam theParam, IQueryParameterType theParamValue, RequestPartitionId theRequestPartitionId) {
 
 		switch (theParam.getParamType()) {
@@ -310,7 +326,7 @@ public class QueryStack {
 				return createPredicateToken(theSourceJoinColumn, theResourceName, theParam, Collections.singletonList(theParamValue), null, theRequestPartitionId);
 			}
 			case DATE: {
-				return createPredicateDate(theSourceJoinColumn, theResourceName, theParam, Collections.singletonList(theParamValue), null, theRequestPartitionId);
+				return createPredicateDate(theSourceJoinColumn, theResourceName, theParam, Collections.singletonList(theParamValue), toOperation(((DateParam) theParamValue).getPrefix()), theRequestPartitionId);
 			}
 			case QUANTITY: {
 				return createPredicateQuantity(theSourceJoinColumn, theResourceName, theParam, Collections.singletonList(theParamValue), null, theRequestPartitionId);
@@ -320,7 +336,6 @@ public class QueryStack {
 		throw new InvalidRequestException("Don't know how to handle composite parameter with type of " + theParam.getParamType());
 	}
 
-
 	public Condition createPredicateCoords(@Nullable DbColumn theSourceJoinColumn,
 														String theResourceName,
 														RuntimeSearchParam theSearchParam,
@@ -363,7 +378,7 @@ public class QueryStack {
 		List<Condition> codePredicates = new ArrayList<>();
 
 		for (IQueryParameterType nextOr : theList) {
-			Condition p = predicateBuilder.createPredicateDateWithoutIdentityPredicate(nextOr, theResourceName, paramName, predicateBuilder, theOperation, theRequestPartitionId);
+			Condition p = predicateBuilder.createPredicateDateWithoutIdentityPredicate(nextOr, predicateBuilder, theOperation);
 			codePredicates.add(p);
 		}
 
@@ -429,7 +444,7 @@ public class QueryStack {
 			} else if (typeEnum == RestSearchParameterTypeEnum.STRING) {
 				return theQueryStack3.createPredicateString(null, theResourceName, searchParam, Collections.singletonList(new StringParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
 			} else if (typeEnum == RestSearchParameterTypeEnum.DATE) {
-				return theQueryStack3.createPredicateDate(null, theResourceName, searchParam, Collections.singletonList(new DateParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
+				return theQueryStack3.createPredicateDate(null, theResourceName, searchParam, Collections.singletonList(new DateParam(fromOperation(theFilter.getOperation()), theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
 			} else if (typeEnum == RestSearchParameterTypeEnum.NUMBER) {
 				return theQueryStack3.createPredicateNumber(null, theResourceName, searchParam, Collections.singletonList(new NumberParam(theFilter.getValue())), theFilter.getOperation(), theRequestPartitionId);
 			} else if (typeEnum == RestSearchParameterTypeEnum.REFERENCE) {
@@ -976,7 +991,15 @@ public class QueryStack {
 					switch (nextParamDef.getParamType()) {
 						case DATE:
 							for (List<? extends IQueryParameterType> nextAnd : theAndOrParams) {
-								andPredicates.add(createPredicateDate(theSourceJoinColumn, theResourceName, nextParamDef, nextAnd, null, theRequestPartitionId));
+								// FT: 2021-01-18 use operation 'gt', 'ge', 'le' or 'lt'
+								// to create the predicateDate instead of generic one with operation = null
+								SearchFilterParser.CompareOperation operation = null;
+								if (nextAnd.size() > 0) {
+									DateParam param = (DateParam) nextAnd.get(0);
+									operation = toOperation(param.getPrefix());
+								}
+								andPredicates.add(createPredicateDate(theSourceJoinColumn, theResourceName, nextParamDef, nextAnd, operation, theRequestPartitionId));
+								//andPredicates.add(createPredicateDate(theSourceJoinColumn, theResourceName, nextParamDef, nextAnd, null, theRequestPartitionId));
 							}
 							break;
 						case QUANTITY:
@@ -1204,29 +1227,19 @@ public class QueryStack {
 	}
 
 	public static SearchFilterParser.CompareOperation toOperation(ParamPrefixEnum thePrefix) {
-		if (thePrefix != null) {
+		SearchFilterParser.CompareOperation retVal = null;
-			switch (thePrefix) {
+		if (thePrefix != null && ourCompareOperationToParamPrefix.containsValue(thePrefix)) {
-				case APPROXIMATE:
+			retVal = ourCompareOperationToParamPrefix.getKey(thePrefix);
-					return SearchFilterParser.CompareOperation.ap;
-				case EQUAL:
-					return SearchFilterParser.CompareOperation.eq;
-				case GREATERTHAN:
-					return SearchFilterParser.CompareOperation.gt;
-				case GREATERTHAN_OR_EQUALS:
-					return SearchFilterParser.CompareOperation.ge;
-				case LESSTHAN:
-					return SearchFilterParser.CompareOperation.lt;
-				case LESSTHAN_OR_EQUALS:
-					return SearchFilterParser.CompareOperation.le;
-				case NOT_EQUAL:
-					return SearchFilterParser.CompareOperation.ne;
-				case ENDS_BEFORE:
-					return SearchFilterParser.CompareOperation.eb;
-				case STARTS_AFTER:
-					return SearchFilterParser.CompareOperation.sa;
-			}
 		}
-		return SearchFilterParser.CompareOperation.eq;
+		return defaultIfNull(retVal, SearchFilterParser.CompareOperation.eq);
+	}
+
+	public static ParamPrefixEnum fromOperation(SearchFilterParser.CompareOperation thePrefix) {
+		ParamPrefixEnum retVal = null;
+		if (thePrefix != null && ourCompareOperationToParamPrefix.containsKey(thePrefix)) {
+			retVal = ourCompareOperationToParamPrefix.get(thePrefix);
+		}
+		return defaultIfNull(retVal, ParamPrefixEnum.EQUAL);
 	}
 
 	private static String getChainedPart(String parameter) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/DatePredicateBuilder.java

@@ -20,7 +20,6 @@ package ca.uhn.fhir.jpa.search.builder.predicate;
  * #L%
  */
 
-import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.dao.predicate.SearchFilterParser;
 import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
@@ -64,16 +63,16 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 
 
 	public Condition createPredicateDateWithoutIdentityPredicate(IQueryParameterType theParam,
-																					 String theResourceName,
-																					 String theParamName,
 																					 DatePredicateBuilder theFrom,
-																					 SearchFilterParser.CompareOperation theOperation,
+																					 SearchFilterParser.CompareOperation theOperation) {
-																					 RequestPartitionId theRequestPartitionId) {
 
 		Condition p;
 		if (theParam instanceof DateParam) {
 			DateParam date = (DateParam) theParam;
 			if (!date.isEmpty()) {
+				if (theOperation == SearchFilterParser.CompareOperation.ne) {
+					date = new DateParam(ParamPrefixEnum.EQUAL, date.getValueAsString());
+				}
 				DateRangeParam range = new DateRangeParam(date);
 				p = createPredicateDateFromRange(theFrom, range, theOperation);
 			} else {
@@ -96,6 +95,8 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 	private Condition createPredicateDateFromRange(DatePredicateBuilder theFrom,
 																  DateRangeParam theRange,
 																  SearchFilterParser.CompareOperation theOperation) {
+
+
 		Date lowerBoundInstant = theRange.getLowerBoundAsInstant();
 		Date upperBoundInstant = theRange.getUpperBoundAsInstant();
 
@@ -103,16 +104,17 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 		DateParam upperBound = theRange.getUpperBound();
 		Integer lowerBoundAsOrdinal = theRange.getLowerBoundAsDateInteger();
 		Integer upperBoundAsOrdinal = theRange.getUpperBoundAsDateInteger();
-		Comparable genericLowerBound;
+		Comparable<?> genericLowerBound;
-		Comparable genericUpperBound;
+		Comparable<?> genericUpperBound;
-		/**
+
+		/*
 		 * If all present search parameters are of DAY precision, and {@link ca.uhn.fhir.jpa.model.entity.ModelConfig#getUseOrdinalDatesForDayPrecisionSearches()} is true,
 		 * then we attempt to use the ordinal field for date comparisons instead of the date field.
 		 */
 		boolean isOrdinalComparison = isNullOrDayPrecision(lowerBound) && isNullOrDayPrecision(upperBound) && myDaoConfig.getModelConfig().getUseOrdinalDatesForDayPrecisionSearches();
 
 		Condition lt;
-		Condition gt = null;
+		Condition gt;
 		Condition lb = null;
 		Condition ub = null;
 		DatePredicateBuilder.ColumnEnum lowValueField;
@@ -130,28 +132,24 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 			genericUpperBound = upperBoundInstant;
 		}
 
-		if (theOperation == SearchFilterParser.CompareOperation.lt) {
+		if (theOperation == SearchFilterParser.CompareOperation.lt || theOperation == SearchFilterParser.CompareOperation.le) {
-			if (lowerBoundInstant == null) {
+			// use lower bound first
-				throw new InvalidRequestException("lowerBound value not correctly specified for compare theOperation");
+			if (lowerBoundInstant != null) {
+				lb = theFrom.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericLowerBound);
+			} else if (upperBoundInstant != null) {
+				ub = theFrom.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound);
+			} else {
+				throw new InvalidRequestException("lowerBound and upperBound value not correctly specified for comparing " + theOperation);
 			}
-			//im like 80% sure this should be ub and not lb, as it is an UPPER bound.
+		} else if (theOperation == SearchFilterParser.CompareOperation.gt || theOperation == SearchFilterParser.CompareOperation.ge) {
-			lb = theFrom.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN, genericLowerBound);
+			// use upper bound first, e.g value between 6 and 10
-		} else if (theOperation == SearchFilterParser.CompareOperation.le) {
+			if (upperBoundInstant != null) {
-			if (upperBoundInstant == null) {
+				ub = theFrom.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericUpperBound);
-				throw new InvalidRequestException("upperBound value not correctly specified for compare theOperation");
+			} else if (lowerBoundInstant != null) {
+				lb = theFrom.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
+			} else {
+				throw new InvalidRequestException("upperBound and lowerBound value not correctly specified for compare theOperation");
 			}
-			//im like 80% sure this should be ub and not lb, as it is an UPPER bound.
-			lb = theFrom.createPredicate(highValueField, ParamPrefixEnum.LESSTHAN_OR_EQUALS, genericUpperBound);
-		} else if (theOperation == SearchFilterParser.CompareOperation.gt) {
-			if (upperBoundInstant == null) {
-				throw new InvalidRequestException("upperBound value not correctly specified for compare theOperation");
-			}
-			lb = theFrom.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN, genericUpperBound);
-		} else if (theOperation == SearchFilterParser.CompareOperation.ge) {
-			if (lowerBoundInstant == null) {
-				throw new InvalidRequestException("lowerBound value not correctly specified for compare theOperation");
-			}
-			lb = theFrom.createPredicate(lowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
 		} else if (theOperation == SearchFilterParser.CompareOperation.ne) {
 			if ((lowerBoundInstant == null) ||
 				(upperBoundInstant == null)) {
@@ -160,7 +158,10 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 			lt = theFrom.createPredicate(lowValueField, ParamPrefixEnum.LESSTHAN, genericLowerBound);
 			gt = theFrom.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN, genericUpperBound);
 			lb = ComboCondition.or(lt, gt);
-		} else if ((theOperation == SearchFilterParser.CompareOperation.eq) || (theOperation == null)) {
+		} else if ((theOperation == SearchFilterParser.CompareOperation.eq)
+			|| (theOperation == SearchFilterParser.CompareOperation.sa)
+			|| (theOperation == SearchFilterParser.CompareOperation.eb)
+			|| (theOperation == null)) {
 			if (lowerBoundInstant != null) {
 				gt = theFrom.createPredicate(lowValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);
 				lt = theFrom.createPredicate(highValueField, ParamPrefixEnum.GREATERTHAN_OR_EQUALS, genericLowerBound);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4FilterTest.java

@@ -703,7 +703,9 @@ public class FhirResourceDaoR4FilterTest extends BaseJpaR4Test {
 		map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Constants.PARAM_FILTER, new StringParam("birthdate gt 1955-01-01"));
+		myCaptureQueriesListener.clear();
 		found = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread(0);
 		assertThat(found, empty());
 
 	}
@@ -723,7 +725,9 @@ public class FhirResourceDaoR4FilterTest extends BaseJpaR4Test {
 		map = new SearchParameterMap();
 		map.setLoadSynchronous(true);
 		map.add(Constants.PARAM_FILTER, new StringParam("birthdate lt 1955-01-02"));
+		myCaptureQueriesListener.clear();
 		found = toUnqualifiedVersionlessIdValues(myPatientDao.search(map));
+		myCaptureQueriesListener.logSelectQueriesForCurrentThread(0);
 		assertThat(found, containsInAnyOrder(id1));
 
 		map = new SearchParameterMap();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LegacySearchBuilderTest.java

@@ -4172,7 +4172,7 @@ public class FhirResourceDaoR4LegacySearchBuilderTest extends BaseJpaR4Test {
 			String searchQuery = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 			ourLog.info("Search query:\n{}", searchQuery);
 			assertEquals(1, countMatches(searchQuery.toLowerCase(), "join"), searchQuery);
-			assertEquals(1, countMatches(searchQuery.toLowerCase(), "sp_value_low_date_ordinal>='20200605'"), searchQuery);
+			assertEquals(1, countMatches(searchQuery.toLowerCase(), "sp_value_high_date_ordinal>='20200605'"), searchQuery);
 			assertEquals(1, countMatches(searchQuery.toLowerCase(), "sp_value_low_date_ordinal<='20200606'"), searchQuery);
 		}
 
@@ -4192,7 +4192,7 @@ public class FhirResourceDaoR4LegacySearchBuilderTest extends BaseJpaR4Test {
 			assertEquals(0, countMatches(searchQuery.toLowerCase(), "partition"), searchQuery);
 			assertEquals(2, countMatches(searchQuery.toLowerCase(), "join"), searchQuery);
 			assertEquals(2, countMatches(searchQuery.toLowerCase(), "hash_identity"), searchQuery);
-			assertEquals(4, countMatches(searchQuery.toLowerCase(), "sp_value_low"), searchQuery);
+			assertEquals(2, countMatches(searchQuery.toLowerCase(), "sp_value_low"), searchQuery);
 		}
 
 		// Period search

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -4369,7 +4369,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 			String searchQuery = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 			ourLog.info("Search query:\n{}", searchQuery);
 			assertEquals(1, countMatches(searchQuery.toLowerCase(), "join"), searchQuery);
-			assertEquals(1, countMatches(searchQuery.toLowerCase(), "t0.sp_value_low_date_ordinal >= '20200605'"), searchQuery);
+			assertEquals(1, countMatches(searchQuery.toLowerCase(), "t0.sp_value_high_date_ordinal >= '20200605'"), searchQuery);
 			assertEquals(1, countMatches(searchQuery.toLowerCase(), "t0.sp_value_low_date_ordinal <= '20200606'"), searchQuery);
 		}
 
@@ -4389,7 +4389,8 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 			assertEquals(0, countMatches(searchQuery.toLowerCase(), "partition"), searchQuery);
 			assertEquals(2, countMatches(searchQuery.toLowerCase(), "join"), searchQuery);
 			assertEquals(2, countMatches(searchQuery.toLowerCase(), "hash_identity"), searchQuery);
-			assertEquals(4, countMatches(searchQuery.toLowerCase(), "sp_value_low"), searchQuery);
+			// - query is changed 'or' is removed
+			assertEquals(2, countMatches(searchQuery.toLowerCase(), "sp_value_low"), searchQuery);
 		}
 
 		// Period search

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java

@@ -1702,7 +1702,9 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 		ourLog.info("Search SQL:\n{}", searchSql);
 		assertEquals(0, StringUtils.countMatches(searchSql, "PARTITION_ID"));
-		assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
 
 	}
 
@@ -1783,7 +1785,9 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 		ourLog.info("Search SQL:\n{}", searchSql);
 		assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
-		assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
 
 	}
 
@@ -1861,7 +1865,9 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 		ourLog.info("Search SQL:\n{}", searchSql);
 		assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID"));
-		assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
+		assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
 
 	}