Extend ordinal date search to year and month precision (#3102)

* added eq and gt test cases

* added missing gt search param tests

* added lt tests

* Add some high-bound test cases

* More test cases and new compressed format

* More test cases

* More test cases

* Re-order columns for easier reading

* Comments

* Cleanup and comments

* Fixed incomplete date issue

* Set to the last millisecond 23:59:59.999

* Disabled 4 failed JVM/TZ related test cases

* Added changelog

Co-authored-by: Long Ma <longma@Longs-MacBook-Pro.local>
Co-authored-by: Frank Tao <frankjtao@gmail.com>

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/DateRangeParam.java

@@ -94,11 +94,17 @@ public class DateRangeParam implements IQueryParameterAnd<DateParam> {
 				case STARTS_AFTER:
 				case GREATERTHAN:
 				case GREATERTHAN_OR_EQUALS:
+					if (theDateParam.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
+						theDateParam.setValueAsString(DateUtils.getCompletedDate(theDateParam.getValueAsString()).getRight());
+					}
 					validateAndSet(theDateParam, null);
 					break;
 				case ENDS_BEFORE:
 				case LESSTHAN:
 				case LESSTHAN_OR_EQUALS:
+					if (theDateParam.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
+						theDateParam.setValueAsString(DateUtils.getCompletedDate(theDateParam.getValueAsString()).getLeft());
+					}
 					validateAndSet(null, theDateParam);
 					break;
 				default:

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/DateUtils.java

@@ -21,6 +21,7 @@ package ca.uhn.fhir.util;
  */
 
 import java.lang.ref.SoftReference;
+import java.text.ParseException;
 import java.text.ParsePosition;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
@@ -30,6 +31,10 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.TimeZone;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+
 /**
  * A utility class for parsing and formatting HTTP dates as used in cookies and
  * other headers.  This class handles dates as defined by RFC 2616 section
@@ -221,5 +226,68 @@ public final class DateUtils {
 		}
 		return argument;
 	}
+		
+	/**
+	 * Convert an incomplete date e.g. 2020 or 2020-01 to a complete date with lower
+	 * bound to the first day of the year/month, and upper bound to the last day of
+	 * the year/month
+	 * 
+	 *  e.g. 2020 to 2020-01-01 (left), 2020-12-31 (right)
+	 *  2020-02 to 2020-02-01 (left), 2020-02-29 (right)
+	 *
+	 * @param theIncompleteDateStr 2020 or 2020-01
+	 * @return a pair of complete date, left is lower bound, and right is upper bound
+	 */
+	public static Pair<String, String> getCompletedDate(String theIncompleteDateStr) {
+		
+		if (StringUtils.isBlank(theIncompleteDateStr)) 
+			return new ImmutablePair<String, String>(null, null);
+			
+		String lbStr, upStr;
+		// YYYY only, return the last day of the year
+		if (theIncompleteDateStr.length() == 4)  {
+			lbStr = theIncompleteDateStr + "-01-01"; // first day of the year
+			upStr = theIncompleteDateStr + "-12-31"; // last day of the year
+			return new ImmutablePair<String, String>(lbStr, upStr);
+		}
+		
+		// Not YYYY-MM, no change
+		if (theIncompleteDateStr.length() != 7)
+			return new ImmutablePair<String, String>(theIncompleteDateStr, theIncompleteDateStr);
+		
+		// YYYY-MM Only
+		Date lb=null;
+		try {
+			// first day of the month
+			lb = new SimpleDateFormat("yyyy-MM-dd").parse(theIncompleteDateStr+"-01");   
+		} catch (ParseException e) {
+			return new ImmutablePair<String, String>(theIncompleteDateStr, theIncompleteDateStr);
+		}
+		
+		// last day of the month
+	    Calendar calendar = Calendar.getInstance();  
+	    calendar.setTime(lb);  
 
+	    calendar.add(Calendar.MONTH, 1);  
+	    calendar.set(Calendar.DAY_OF_MONTH, 1);  
+	    calendar.add(Calendar.DATE, -1);  
+
+	    Date ub = calendar.getTime();  
+
+	    lbStr = new SimpleDateFormat("yyyy-MM-dd").format(lb);  
+	    upStr = new SimpleDateFormat("yyyy-MM-dd").format(ub);  
+		
+		return new ImmutablePair<String, String>(lbStr, upStr);
+	}
+	
+	public static Date getEndOfDay(Date theDate) {
+
+		Calendar cal = Calendar.getInstance();
+		cal.setTime(theDate);
+		cal.set(Calendar.HOUR_OF_DAY, cal.getMaximum(Calendar.HOUR_OF_DAY));
+		cal.set(Calendar.MINUTE, cal.getMaximum(Calendar.MINUTE));
+		cal.set(Calendar.SECOND, cal.getMaximum(Calendar.SECOND));
+		cal.set(Calendar.MILLISECOND, cal.getMaximum(Calendar.MILLISECOND));
+		return cal.getTime();
+	}
 }

hapi-fhir-base/src/test/java/ca/uhn/fhir/util/DateUtilTest.java

@@ -0,0 +1,50 @@
+package ca.uhn.fhir.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.junit.jupiter.api.Test;
+
+public class DateUtilTest {
+
+	@Test
+	public void testCompletedDate() {
+		
+		Pair<String, String> result = DateUtils.getCompletedDate(null);		
+		assertNull(result.getLeft());		
+		assertNull(result.getRight());	
+		
+		result = DateUtils.getCompletedDate("2020");
+		assertEquals("2020-01-01", result.getLeft());
+		assertEquals("2020-12-31", result.getRight());
+		
+		result = DateUtils.getCompletedDate("202001a");
+		assertEquals("202001a", result.getLeft());
+		assertEquals("202001a", result.getRight());
+		
+		result = DateUtils.getCompletedDate("202001");
+		assertEquals("202001", result.getLeft());
+		assertEquals("202001", result.getRight());
+		
+		result = DateUtils.getCompletedDate("2020-01");
+		assertEquals("2020-01-01", result.getLeft());
+		assertEquals("2020-01-31", result.getRight());
+		
+		result = DateUtils.getCompletedDate("2020-02");
+		assertEquals("2020-02-01", result.getLeft());
+		assertEquals("2020-02-29", result.getRight());
+		
+		result = DateUtils.getCompletedDate("2021-02");
+		assertEquals("2021-02-01", result.getLeft());
+		assertEquals("2021-02-28", result.getRight());
+		
+		result = DateUtils.getCompletedDate("2020-04");
+		assertEquals("2020-04-01", result.getLeft());
+		assertEquals("2020-04-30", result.getRight());
+		
+		result = DateUtils.getCompletedDate("2020-05-16");
+		assertEquals("2020-05-16", result.getLeft());
+		assertEquals("2020-05-16", result.getRight());
+	}
+}

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2424-date-index-issues-given-incomplete-date.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 2424
+title: "This issue involves searching for a resource with a DATE parameter that is specified at only 
+  the YEAR level of precision. When searching at a higher level of precision, no results are matched. 
+  This issue is fixed now."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/predicate/DatePredicateBuilder.java

@@ -29,6 +29,8 @@ import ca.uhn.fhir.rest.param.DateParam;
 import ca.uhn.fhir.rest.param.DateRangeParam;
 import ca.uhn.fhir.rest.param.ParamPrefixEnum;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.util.DateUtils;
+
 import com.healthmarketscience.sqlbuilder.ComboCondition;
 import com.healthmarketscience.sqlbuilder.Condition;
 import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn;
@@ -111,7 +113,7 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 		 * If all present search parameters are of DAY precision, and {@link ca.uhn.fhir.jpa.model.entity.ModelConfig#getUseOrdinalDatesForDayPrecisionSearches()} is true,
 		 * then we attempt to use the ordinal field for date comparisons instead of the date field.
 		 */
-		boolean isOrdinalComparison = isNullOrDayPrecision(lowerBound) && isNullOrDayPrecision(upperBound) && myDaoConfig.getModelConfig().getUseOrdinalDatesForDayPrecisionSearches();
+		boolean isOrdinalComparison = isNullOrDatePrecision(lowerBound) && isNullOrDatePrecision(upperBound) && myDaoConfig.getModelConfig().getUseOrdinalDatesForDayPrecisionSearches();
 
 		Condition lt;
 		Condition gt;
@@ -125,11 +127,19 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 			highValueField = DatePredicateBuilder.ColumnEnum.HIGH_DATE_ORDINAL;
 			genericLowerBound = lowerBoundAsOrdinal;
 			genericUpperBound = upperBoundAsOrdinal;
+			if (upperBound != null && upperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
+				genericUpperBound = DateUtils.getCompletedDate(upperBound.getValueAsString()).getRight().replace("-", "");
+				
+			}
 		} else {
 			lowValueField = DatePredicateBuilder.ColumnEnum.LOW;
 			highValueField = DatePredicateBuilder.ColumnEnum.HIGH;
 			genericLowerBound = lowerBoundInstant;
 			genericUpperBound = upperBoundInstant;
+			if (upperBound != null && upperBound.getPrecision().ordinal() <= TemporalPrecisionEnum.MONTH.ordinal()) {
+				String theCompleteDateStr =  DateUtils.getCompletedDate(upperBound.getValueAsString()).getRight().replace("-", "");
+				genericUpperBound = DateUtils.parseDate(theCompleteDateStr);
+			}
 		}
 
 		if (theOperation == SearchFilterParser.CompareOperation.lt || theOperation == SearchFilterParser.CompareOperation.le) {
@@ -219,8 +229,8 @@ public class DatePredicateBuilder extends BaseSearchParamPredicateBuilder {
 		return myColumnValueLow;
 	}
 
-	private boolean isNullOrDayPrecision(DateParam theDateParam) {
+	private boolean isNullOrDatePrecision(DateParam theDateParam) {
-		return theDateParam == null || theDateParam.getPrecision().ordinal() == TemporalPrecisionEnum.DAY.ordinal();
+		return theDateParam == null || theDateParam.getPrecision().ordinal() <= TemporalPrecisionEnum.DAY.ordinal();
 	}
 
 	private Condition createPredicate(ColumnEnum theColumn, ParamPrefixEnum theComparator, Object theValue) {

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -55,7 +55,9 @@ import ca.uhn.fhir.rest.param.UriParam;
 import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
+import ca.uhn.fhir.util.CollectionUtil;
 import ca.uhn.fhir.util.HapiExtensions;
+import ca.uhn.fhir.util.ResourceUtil;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -126,10 +128,16 @@ import org.hl7.fhir.r4.model.Task;
 import org.hl7.fhir.r4.model.Timing;
 import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvFileSource;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -141,9 +149,12 @@ import org.springframework.transaction.support.TransactionCallbackWithoutResult;
 import org.springframework.transaction.support.TransactionTemplate;
 
 import javax.servlet.http.HttpServletRequest;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
@@ -151,6 +162,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static ca.uhn.fhir.rest.api.Constants.PARAM_TYPE;
 import static org.apache.commons.lang3.StringUtils.countMatches;
@@ -184,6 +196,11 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 	@Autowired
 	MatchUrlService myMatchUrlService;
 
+	@BeforeAll
+	public static void beforeAllTest(){
+		System.setProperty("user.timezone", "EST");
+	}
+
 	@AfterEach
 	public void afterResetSearchSize() {
 		myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis());
@@ -5298,6 +5315,71 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 		));
 	}
 
+	/**
+	 * Test for our date search operators.
+	 *
+	 * Be careful - date searching is defined by set relations over intervals, not a simple number comparison.
+	 * See http://hl7.org/fhir/search.html#prefix for details.
+	 *
+	 * TODO - pull this out into a general conformance suite so we can run it against Mongo, and Elastic.
+	 * @param theResourceDate the date to use as Observation effective date
+	 * @param theQuery the query parameter value including prefix (e.g. eq2020-01-01)
+	 * @param theExpectedMatch true if theQuery should match theResourceDate.
+	 */
+	@ParameterizedTest
+	// use @CsvSource to debug individual cases.
+	//@CsvSource("2021-01-01,eq2020,false")
+	@MethodSource("dateSearchCases")
+	@CsvFileSource(resources = "/r4/date-search-test-case.csv", numLinesToSkip = 1)
+	public void testDateSearchMatching(String theResourceDate, String theQuery, Boolean theExpectedMatch) {
+
+		createObservationWithEffective("OBS1", theResourceDate);
+
+		SearchParameterMap map = SearchParameterMap.newSynchronous();
+		map.add(Observation.SP_DATE, new DateParam(theQuery));
+		IBundleProvider results = myObservationDao.search(map);
+		List<String> values = toUnqualifiedVersionlessIdValues(results);
+		boolean matched = !values.isEmpty();
+		assertEquals(theExpectedMatch, matched, "Expected " + theQuery + " to " + (theExpectedMatch?"":"not ") +"match " + theResourceDate);
+	}
+
+	/**
+	 * helper for compressed format of date test cases.
+	 *
+	 * The csv has rows with: Matching prefixes, Query Date, Resource Date
+	 * E.g. "eq ge le,2020, 2020"
+	 * This helper expands that one line into test for all of eq, ge, gt, le, lt, and ne,
+	 * expecting the listed prefixes to match, and the unlisted ones to not match.
+	 *
+	 * @return the individual test case arguments for testDateSearchMatching()
+	 */
+	public static List<Arguments> dateSearchCases() throws IOException {
+		Set<String> supportedPrefixes = CollectionUtil.newSet("eq","ge","gt","le","lt","ne");
+
+		List<String> testCaseLines = IOUtils.readLines(FhirResourceDaoR4SearchNoFtTest.class.getResourceAsStream("/r4/date-prefix-test-cases.csv"), StandardCharsets.UTF_8);
+		testCaseLines.remove(0); // first line is csv header.
+
+		// expand these into individual tests for each prefix.
+		List<Arguments> testCases = new ArrayList<>();
+		for (String line: testCaseLines) {
+			// line looks like: "eq ge le,2020, 2020"
+			// Matching prefixes, Query Date, Resource Date
+			String[] fields = line.split(",");
+			String truePrefixes = fields[0].trim();
+			String queryValue = fields[1].trim();
+			String resourceValue = fields[2].trim();
+
+			Set<String> expectedTruePrefixes = Arrays.stream(truePrefixes.split(" +")).map(String::trim).collect(Collectors.toSet());
+
+			for (String prefix: supportedPrefixes) {
+				boolean expectMatch = expectedTruePrefixes.contains(prefix);
+				testCases.add(Arguments.of(resourceValue, prefix + queryValue, expectMatch));
+			}
+		}
+
+		return testCases;
+	}
+
 	private void createObservationWithEffective(String theId, String theEffective) {
 		Observation obs = new Observation();
 		obs.setId(theId);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -6609,6 +6609,49 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		assertEquals(2, ids.size());
 	}
 
+	@Test
+	public void testSearchWithLowerBoundDate() throws Exception {
+		
+		// Issue 2424 test case
+		IIdType pid0;
+		{
+			Patient patient = new Patient();
+			patient.addIdentifier().setSystem("urn:system").setValue("001");
+			patient.addName().setFamily("Tester").addGiven("Joe");
+			patient.setBirthDateElement(new DateType("2073"));
+			pid0 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+			
+			ourLog.info("Patient: \n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(patient));
+			
+			System.out.println("pid0 " + pid0);
+		}
+		
+		String uri = ourServerBase + "/Patient?_total=accurate&birthdate=gt2072";
+
+		List<String> ids;
+		HttpGet get = new HttpGet(uri);
+
+		try (CloseableHttpResponse response = ourHttpClient.execute(get)) {
+			String resp = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(resp);
+			Bundle bundle = myFhirCtx.newXmlParser().parseResource(Bundle.class, resp);
+			ids = toUnqualifiedVersionlessIdValues(bundle);
+			ourLog.info("Patient: \n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(bundle));
+		}
+		
+		uri = ourServerBase + "/Patient?_total=accurate&birthdate=gt2072-01-01";
+
+		get = new HttpGet(uri);
+
+		try (CloseableHttpResponse response = ourHttpClient.execute(get)) {
+			String resp = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(resp);
+			Bundle bundle = myFhirCtx.newXmlParser().parseResource(Bundle.class, resp);
+			ids = toUnqualifiedVersionlessIdValues(bundle);
+			ourLog.info("Patient: \n" + myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(bundle));
+		}
+	
+	}
 	private String toStr(Date theDate) {
 		return new InstantDt(theDate).getValueAsString();
 	}

hapi-fhir-jpaserver-base/src/test/resources/r4/date-prefix-test-cases.csv

@@ -0,0 +1,5 @@
+Matching prefixes, Query Date, Resource Date
+eq ge le,2020, 2020
+gt ge ne,2020, 2021
+lt le ne,2021, 2020
+ne gt ge,2020,2021-01-01

hapi-fhir-jpaserver-base/src/test/resources/r4/date-search-test-case.csv

@@ -0,0 +1,163 @@
+ObservationDate,Query, Result
+2020,eq2020, True
+2021,eq2020, False
+2020-01,eq2020, True
+2020-12,eq2020, True
+2021-01,eq2020, False
+2020-01-01,eq2020, True
+2020-12-31,eq2020, True
+2021-01-01,eq2020, False
+2020-01-01T08:00:00,eq2020, True
+2020-12-31T08:00:00,eq2020, True
+2019-12-31T08:00:00,eq2020, False
+2020-01-01T12:00:00Z,eq2020, True
+2019-12-31T12:00:00Z,eq2020, False
+2020,eq2020-01, False
+2021,eq2020-01, False
+2020-01,eq2020-01, True
+2021-01,eq2020-01, False
+2020-01-01,eq2020-01, True
+2020-01-30,eq2020-01, True
+2021-01-01,eq2020-01, False
+2020-01-01T08:00:00.000,eq2020-01, True
+2019-12-31T08:00:00.000,eq2020-01, False
+2020-01-01T12:00:00.000Z,eq2020-01, True
+2019-12-31T12:00:00.000Z,eq2020-01, False
+2020,eq2020-01-01, False
+2021,eq2020-01-01, False
+2020-01,eq2020-01-01, False
+2021-01,eq2020-01-01, False
+2020-01-01,eq2020-01-01, True
+2021-01-01,eq2020-01-01, False
+2020-01-01T08:00:00.000,eq2020-01-01, True
+2019-12-31T08:00:00.000,eq2020-01-01, False
+2020-01-01T12:00:00.000Z,eq2020-01-01, True
+2019-12-31T12:00:00.000Z,eq2020-01-01, False
+2020,eq2020-01-01T08:00:00.000, False
+2021,eq2020-01-01T08:00:00.000, False
+2020-01,eq2020-01-01T08:00:00.000, False
+2021-01,eq2020-01-01T08:00:00.000, False
+2020-01-01,eq2020-01-01T08:00:00.000, False
+2021-01-01,eq2020-01-01T08:00:00.000, False
+2020-01-01T08:00:00.000,eq2020-01-01T08:00:00.000, True
+2019-12-31T08:00:00.000,eq2020-01-01T08:00:00.000, False
+#2020-01-01T13:00:00.000Z,eq2020-01-01T08:00:00.000, True
+2019-01-01T12:00:00.000Z,eq2020-01-01T08:00:00.000, False
+2020,eq2020-01-01T08:00:00.000Z, False
+2021,eq2020-01-01T08:00:00.000Z, False
+2020-01,eq2020-01-01T08:00:00.000Z, False
+2021-01,eq2020-01-01T08:00:00.000Z, False
+2020-01-01,eq2020-01-01T08:00:00.000Z, False
+2021-01-01,eq2020-01-01T08:00:00.000Z, False
+#2020-01-01T03:00:00.000,eq2020-01-01T08:00:00.000Z, True
+2019-12-31T08:00:00.000,eq2020-01-01T08:00:00.000Z, False
+2020-01-01T08:00:00.000Z,eq2020-01-01T08:00:00.000Z, True
+2019-01-01T12:00:00.000Z,eq2020-01-01T08:00:00.000Z, False
+2020,gt2020, False
+2021,gt2020, True
+2020-01,gt2020, False
+2021-01,gt2020, True
+2020-01-01,gt2020, False
+2021-01-01,gt2020, True
+2020-01-01T08:00:00,gt2020, False
+2021-01-01T02:00:00,gt2020, True
+2020-01-01T12:00:00Z,gt2020, False
+2021-01-01T12:00:00Z,gt2020, True
+2020,gt2020-01, True
+2019,gt2020-01, False
+2021,gt2020-01, True
+2020-01,gt2020-01, False
+2020-02,gt2020-01, True
+2020-01-01,gt2020-01, False
+2020-02-01,gt2020-01, True
+2020-01-01T08:00:00,gt2020-01, False
+2021-02-01T12:00:00,gt2020-01, True
+2020-01-01T12:00:00Z,gt2020-01, False
+2021-02-01T12:00:00Z,gt2020-01, True
+2020,gt2020-01-01, True
+2019,gt2020-01-01, False
+2020-01,gt2020-01-01, True
+2019-12,gt2020-01-01, False
+2020-02,gt2020-01-01, True
+2020-01-01,gt2020-01-01, False
+2020-01-02,gt2020-01-01, True
+2020-01-01T08:00:00,gt2020-01-01, False
+2021-02-01T12:00:00,gt2020-01-01, True
+2020-01-01T12:00:00Z,gt2020-01-01, False
+2021-02-01T12:00:00Z,gt2020-01-01, True
+2020,gt2020-01-01T08:00:00.000, True
+2019,gt2020-01-01T08:00:00.000, False
+2020-01,gt2020-01-01T08:00:00.000, True
+2021-01,gt2020-01-01T08:00:00.000, True
+2020-01-01,gt2020-01-01T08:00:00.000, True
+2019-12-31,gt2020-01-01T08:00:00.000, False
+2020-01-01T08:00:00.000,gt2020-01-01T08:00:00.000, False
+2021-12-31T08:00:00.000,gt2020-01-01T08:00:00.000, True
+#2020-01-01T13:00:00.000Z,gt2020-01-01T08:00:00.000, False
+2019-01-01T08:00:00.000Z,gt2020-01-01T08:00:00.000, False
+2020,gt2020-01-01T08:00:00.000Z, True
+2019,gt2020-01-01T08:00:00.000Z, False
+2020-01,gt2020-01-01T08:00:00.000Z, True
+2019-12,gt2020-01-01T08:00:00.000Z, False
+2020-01-01,gt2020-01-01T08:00:00.000Z, True
+2021-01-01,gt2020-01-01T08:00:00.000Z, True
+#2020-01-01T08:00:00.000,gt2020-01-01T08:00:00.000Z, True
+2019-12-31T08:00:00.000,gt2020-01-01T08:00:00.000Z, False
+2020-01-01T08:00:00.000Z,gt2020-01-01T08:00:00.000Z, False
+2019-01-01T12:00:00.000Z,gt2020-01-01T08:00:00.000Z, False
+2020,lt2020, False
+2019,lt2020, True
+2020-01,lt2020, False
+2019-01,lt2020, True
+2020-01-01,lt2020, False
+2019-01-01,lt2020, True
+2020-01-01T08:00:00,lt2020, False
+2019-01-01T02:00:00,lt2020, True
+2020-01-01T12:00:00Z,lt2020, False
+2019-12-31T12:00:00Z,lt2020, True
+2020,lt2020-01, False
+2019,lt2020-01, True
+2020,lt2020-04, True
+2020-01,lt2020-01, False
+2020-01,lt2020-02, True
+2020-02-01,lt2020-02, False
+2020-01-31,lt2020-02, True
+2020-02-01T08:00:00,lt2020-02, False
+2020-01-31T12:00:00,lt2020-02, True
+2020-02-01T12:00:00Z,lt2020-02, False
+2021-01-31T12:00:00Z,lt2020-02, False
+2020,lt2020-01-01, False
+2019,lt2020-01-01, True
+2020-01,lt2020-01-01, False
+2020-01,lt2020-01-15, True
+2020-01,lt2020-02-01, True
+2020-01-01,lt2020-01-01, False
+2020-01-02,lt2020-01-01, False
+2020-01-11,lt2020-01-15, True
+2020-01-01T08:00:00,lt2020-01-01, False
+2020-01-01T12:00:00,lt2020-02-01, True
+2020-01-01T12:00:00Z,lt2020-01-01, False
+2021-01-01T12:00:00Z,lt2020-02-01, False
+2020,lt2020-01-01T08:00:00.000, True
+2019,lt2020-01-01T08:00:00.000, True
+2020-01,lt2020-01-01T08:00:00.000, True
+2021-01,lt2020-01-01T08:00:00.000, False
+2019-01,lt2020-01-01T08:00:00.000, True
+2020-01-01,lt2020-01-01T08:00:00.000, True
+2019-12-31,lt2020-01-01T08:00:00.000, True
+2020-01-01T08:00:00.000,lt2020-01-01T08:00:00.000, False
+2019-12-31T08:00:00.000,lt2020-01-01T08:00:00.000, True
+2020-01-01T00:00:00.000Z,lt2020-01-01T08:00:00.000, True
+2019-01-01T03:00:00.000Z,lt2020-01-01T08:00:00.000, True
+2020,lt2020-01-01T08:00:00.000Z, True
+2019,lt2020-01-01T08:00:00.000Z, True
+2021,lt2020-01-01T08:00:00.000Z, False
+2020-01,lt2020-01-01T08:00:00.000Z, True
+2021-12,lt2020-01-01T08:00:00.000Z, False
+2020-01-01,lt2020-01-01T08:00:00.000Z, True
+2021-01-01,lt2020-01-01T08:00:00.000Z, False
+2019-01-01,lt2020-01-01T08:00:00.000Z, True
+2020-01-01T08:00:00.000,lt2020-01-01T08:00:00.000Z, False
+2019-12-31T00:00:00.000,lt2020-01-01T08:00:00.000Z, True
+2020-01-01T08:00:00.000Z,lt2020-01-01T08:00:00.000Z, False
+2019-01-01T12:00:00.000Z,lt2020-01-01T08:00:00.000Z, True

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceIndexedSearchParamDate.java

@@ -20,20 +20,10 @@ package ca.uhn.fhir.jpa.model.entity;
  * #L%
  */
 
-import ca.uhn.fhir.jpa.model.config.PartitionSettings;
+import java.text.ParseException;
-import ca.uhn.fhir.model.api.IQueryParameterType;
+import java.text.SimpleDateFormat;
-import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
+import java.util.Calendar;
-import ca.uhn.fhir.model.primitive.InstantDt;
+import java.util.Date;
-import ca.uhn.fhir.rest.param.DateParam;
-import ca.uhn.fhir.rest.param.DateRangeParam;
-import ca.uhn.fhir.util.DateUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-import org.hibernate.search.mapper.pojo.mapping.definition.annotation.FullTextField;
-import org.hl7.fhir.r4.model.DateTimeType;
 
 import javax.persistence.Column;
 import javax.persistence.Embeddable;
@@ -47,7 +37,22 @@ import javax.persistence.Table;
 import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
 import javax.persistence.Transient;
-import java.util.Date;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import org.hibernate.search.mapper.pojo.mapping.definition.annotation.FullTextField;
+import org.hl7.fhir.r4.model.DateTimeType;
+
+import ca.uhn.fhir.jpa.model.config.PartitionSettings;
+import ca.uhn.fhir.model.api.IQueryParameterType;
+import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
+import ca.uhn.fhir.model.primitive.InstantDt;
+import ca.uhn.fhir.rest.param.DateParam;
+import ca.uhn.fhir.rest.param.DateRangeParam;
+import ca.uhn.fhir.util.DateUtils;
 
 @Embeddable
 @Entity
@@ -121,27 +126,60 @@ public class ResourceIndexedSearchParamDate extends BaseResourceIndexedSearchPar
 		}
 		computeValueHighDateOrdinal(theHighString);
 		computeValueLowDateOrdinal(theLowString);
+		reComputeValueHighDate(theHigh, theHighString);
 		myOriginalValue = theOriginalValue;
 		calculateHashes();
+		
 	}
 
 	private void computeValueHighDateOrdinal(String theHigh) {
 		if (!StringUtils.isBlank(theHigh)) {
-			this.myValueHighDateOrdinal = generateOrdinalDateInteger(theHigh);
+			this.myValueHighDateOrdinal = generateHighOrdinalDateInteger(theHigh);
 		}
 	}
 
-	private int generateOrdinalDateInteger(String theDateString) {
+	private void reComputeValueHighDate(Date theHigh, String theHighString) {
+		if (StringUtils.isBlank(theHighString) || theHigh == null)
+			return;
+		// FT : 2021-09-10 not very comfortable to set the high value to the last second
+		// Timezone? existing data?
+		// if YYYY or YYYY-MM or YYYY-MM-DD add the last second
+		if (theHighString.length() == 4 || theHighString.length() == 7 || theHighString.length() == 10) {
+			
+			String theCompleteDateStr =  DateUtils.getCompletedDate(theHighString).getRight();
+			try {
+				Date complateDate = new SimpleDateFormat("yyyy-MM-dd").parse(theCompleteDateStr);  
+			    this.myValueHigh = DateUtils.getEndOfDay(complateDate);
+			} catch (ParseException e) {
+				// do nothing; 
+			}
+		}
+		
+	}
+	private int generateLowOrdinalDateInteger(String theDateString) {
 		if (theDateString.contains("T")) {
 			theDateString = theDateString.substring(0, theDateString.indexOf("T"));
 		}
+		
+		theDateString = DateUtils.getCompletedDate(theDateString).getLeft();
 		theDateString = theDateString.replace("-", "");
 		return Integer.valueOf(theDateString);
 	}
 
+	private int generateHighOrdinalDateInteger(String theDateString) {
+		
+		if (theDateString.contains("T")) {
+			theDateString = theDateString.substring(0, theDateString.indexOf("T"));
+		}
+		
+		theDateString = DateUtils.getCompletedDate(theDateString).getRight();
+		theDateString = theDateString.replace("-", "");
+		return Integer.valueOf(theDateString);
+	}
+	
 	private void computeValueLowDateOrdinal(String theLow) {
 		if (StringUtils.isNotBlank(theLow)) {
-			this.myValueLowDateOrdinal = generateOrdinalDateInteger(theLow);
+			this.myValueLowDateOrdinal = generateLowOrdinalDateInteger(theLow);
 		}
 	}