Introduce standard date search test cases (#3096)

* Introduce standard date search test cases

hapi-fhir-jpaserver-base/pom.xml

@@ -150,6 +150,12 @@
 			<artifactId>hapi-fhir-batch</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>ca.uhn.hapi.fhir</groupId>
+			<artifactId>hapi-fhir-test-utilities</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
 
 		<dependency>
 			<groupId>ch.qos.logback</groupId>

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseDAODateSearchTest.java

@@ -0,0 +1,106 @@
+package ca.uhn.fhir.jpa.dao;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
+import ca.uhn.fhir.jpa.conformance.DateSearchTestCase;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import ca.uhn.fhir.rest.param.DateParam;
+import ca.uhn.fhir.util.FhirTerser;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.Observation;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.transaction.support.TransactionCallback;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Run the tests defined in {@link DateSearchTestCase} in a DAO test as a @Nested suite.
+ */
+public abstract class BaseDAODateSearchTest {
+	private static final Logger ourLog = LoggerFactory.getLogger(BaseDAODateSearchTest.class);
+
+	/** Id of test Observation */
+	IIdType myObservationId;
+
+	/**
+	 * Test for our date search operators.
+	 * <p>
+	 * Be careful - date searching is defined by set relations over intervals, not a simple number comparison.
+	 * See http://hl7.org/fhir/search.html#prefix for details.
+	 * <p>
+	 *
+	 * @param theResourceDate  the date to use as Observation effective date
+	 * @param theQuery         the query parameter value including prefix (e.g. eq2020-01-01)
+	 * @param theExpectedMatch true if tdheQuery should match theResourceDate.
+	 */
+	@ParameterizedTest
+	// use @CsvSource to debug individual cases.
+	//@CsvSource("2019-12-31T08:00:00,eq2020,false,inline,1")
+	@MethodSource("dateSearchCases")
+	public void testDateSearchMatching(String theResourceDate, String theQuery, Boolean theExpectedMatch, String theFileName, int theLineNumber) {
+		if (isShouldSkip(theResourceDate, theQuery)) {
+			return;
+		}
+		// setup
+		createObservationWithEffectiveDate(theResourceDate);
+
+		// run the query
+		boolean matched = isSearchMatch(theQuery);
+
+		String message =
+			"Expected " + theQuery + " to " +
+				(theExpectedMatch ? "" : "not ") + "match " + theResourceDate +
+			" (" + theFileName + ":" + theLineNumber + ")"; // wrap this in () so tools recognize the line reference.
+		assertEquals(theExpectedMatch, matched, message);
+	}
+
+	protected boolean isShouldSkip(String theResourceDate, String theQuery) {
+		return false;
+	}
+
+	// we need these from the test container
+	abstract protected FhirContext getMyFhirCtx();
+	abstract protected <T> T doInTransaction(TransactionCallback<T> daoMethodOutcomeTransactionCallback);
+	abstract protected <T extends IBaseResource> IFhirResourceDao<T> getObservationDao();
+
+	protected void createObservationWithEffectiveDate(String theResourceDate) {
+		IBaseResource obs = getMyFhirCtx().getResourceDefinition("Observation").newInstance();
+		FhirTerser fhirTerser = getMyFhirCtx().newTerser();
+		fhirTerser.addElement(obs, "effectiveDateTime", theResourceDate);
+		ourLog.info("obs {}", getMyFhirCtx().newJsonParser().encodeResourceToString(obs));
+
+		DaoMethodOutcome createOutcome = doInTransaction(s -> getObservationDao().create(obs));
+		myObservationId = createOutcome.getId();
+	}
+
+	/**
+	 * Does the query string match the observation created during setup?
+	 */
+	protected boolean isSearchMatch(String theQuery) {
+		SearchParameterMap map = SearchParameterMap.newSynchronous();
+		map.add(Observation.SP_DATE, new DateParam(theQuery));
+		ourLog.info("Searching for observation {}", map);
+
+		IBundleProvider results = getObservationDao().search(map);
+
+		boolean matched = results.getAllResourceIds().contains(myObservationId.getIdPart());
+		return matched;
+	}
+
+	static List<Arguments> dateSearchCases() {
+		return DateSearchTestCase.ourCases.stream()
+			.map(DateSearchTestCase::toArguments)
+			.collect(Collectors.toList());
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LegacySearchBuilderTest.java

@@ -1,10 +1,13 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
+import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.interceptor.api.HookParams;
 import ca.uhn.fhir.interceptor.api.IAnonymousInterceptor;
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.dao.BaseDAODateSearchTest;
 import ca.uhn.fhir.jpa.entity.Search;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.entity.ModelConfig;
@@ -123,6 +126,7 @@ import org.hl7.fhir.r4.model.ValueSet;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
@@ -175,13 +179,15 @@ public class FhirResourceDaoR4LegacySearchBuilderTest extends BaseJpaR4Test {
 
 	@AfterEach
 	public void afterResetSearchSize() {
-		myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis());
-		myDaoConfig.setFetchSizeDefaultMaximum(new DaoConfig().getFetchSizeDefaultMaximum());
-		myDaoConfig.setAllowContainsSearches(new DaoConfig().isAllowContainsSearches());
-		myDaoConfig.setSearchPreFetchThresholds(new DaoConfig().getSearchPreFetchThresholds());
-		myDaoConfig.setIndexMissingFields(new DaoConfig().getIndexMissingFields());
-		myDaoConfig.setUseLegacySearchBuilder(false);
-		myDaoConfig.setAccountForDateIndexNulls(false);
+		DaoConfig defaultConfig = new DaoConfig();
+		myDaoConfig.setReuseCachedSearchResultsForMillis(defaultConfig.getReuseCachedSearchResultsForMillis());
+		myDaoConfig.setFetchSizeDefaultMaximum(defaultConfig.getFetchSizeDefaultMaximum());
+		myDaoConfig.setAllowContainsSearches(defaultConfig.isAllowContainsSearches());
+		myDaoConfig.setSearchPreFetchThresholds(defaultConfig.getSearchPreFetchThresholds());
+		myDaoConfig.setIndexMissingFields(defaultConfig.getIndexMissingFields());
+		myDaoConfig.setUseLegacySearchBuilder(defaultConfig.isUseLegacySearchBuilder());
+		myDaoConfig.setAccountForDateIndexNulls(defaultConfig.isAccountForDateIndexNulls());
+		myModelConfig.setUseOrdinalDatesForDayPrecisionSearches(new ModelConfig().getUseOrdinalDatesForDayPrecisionSearches());
 	}
 
 	@BeforeEach
@@ -189,6 +195,7 @@ public class FhirResourceDaoR4LegacySearchBuilderTest extends BaseJpaR4Test {
 		myModelConfig.setSuppressStringIndexingInTokens(new ModelConfig().isSuppressStringIndexingInTokens());
 		myDaoConfig.setReuseCachedSearchResultsForMillis(null);
 		myDaoConfig.setUseLegacySearchBuilder(true);
+		myDaoConfig.getModelConfig().setUseOrdinalDatesForDayPrecisionSearches(true);
 	}
 
 	@Test
@@ -5308,6 +5315,31 @@ public class FhirResourceDaoR4LegacySearchBuilderTest extends BaseJpaR4Test {
 
 	}
 
+	@Nested
+	public class DateSearchTests extends BaseDAODateSearchTest {
+
+		/**
+		 * legacy builder didn't get the year/month date search fixes, so skip anything wider than a day.
+		 */
+		@Override
+		protected boolean isShouldSkip(String theResourceDate, String theQuery) {
+			// skip anything with just year or month resolution.
+			return (theResourceDate.length()<10 || theQuery.length()<10);
+		}
+
+		@Override
+		protected FhirContext getMyFhirCtx() {
+			return myFhirCtx;
+		}
+		@Override
+		protected <T> T doInTransaction(TransactionCallback<T> theCallback) {
+			return new TransactionTemplate(myTxManager).execute(theCallback);
+		}
+		@Override
+		protected IFhirResourceDao<Observation> getObservationDao() {
+			return myObservationDao;
+		}
+	}
 
 	private String toStringMultiline(List<?> theResults) {
 		StringBuilder b = new StringBuilder();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4LuceneDisabledStandardQueries.java

@@ -0,0 +1,67 @@
+package ca.uhn.fhir.jpa.dao.r4;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.config.TestR4WithLuceneDisabledConfig;
+import ca.uhn.fhir.jpa.dao.BaseDAODateSearchTest;
+import ca.uhn.fhir.jpa.dao.BaseJpaTest;
+import org.hl7.fhir.r4.model.Observation;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ExtendWith(SpringExtension.class)
+@ContextConfiguration(classes = {TestR4WithLuceneDisabledConfig.class})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
+public class FhirResourceDaoR4LuceneDisabledStandardQueries extends BaseJpaTest {
+	private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoR4LuceneDisabledStandardQueries.class);
+	@Autowired
+	PlatformTransactionManager myTxManager;
+	@Autowired
+	FhirContext myFhirCtx;
+	@Autowired
+	@Qualifier("myObservationDaoR4")
+	IFhirResourceDao<Observation> myObservationDao;
+
+	@Override
+	protected PlatformTransactionManager getTxManager() {
+		return myTxManager;
+	}
+
+	@Override
+	protected FhirContext getContext() {
+		return myFhirCtx;
+	}
+
+	@Nested
+	public class DateSearchTests extends BaseDAODateSearchTest {
+		@Override
+		protected FhirContext getMyFhirCtx() {
+			return myFhirCtx;
+		}
+
+		@Override
+		protected <T> T doInTransaction(TransactionCallback<T> theCallback) {
+			return new TransactionTemplate(myTxManager).execute(
+				theCallback
+			);
+		}
+
+		@Override
+		protected IFhirResourceDao<Observation> getObservationDao() {
+			return myObservationDao;
+		}
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java

@@ -5315,70 +5315,6 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
 		));
 	}
 
-	/**
-	 * Test for our date search operators.
-	 *
-	 * Be careful - date searching is defined by set relations over intervals, not a simple number comparison.
-	 * See http://hl7.org/fhir/search.html#prefix for details.
-	 *
-	 * TODO - pull this out into a general conformance suite so we can run it against Mongo, and Elastic.
-	 * @param theResourceDate the date to use as Observation effective date
-	 * @param theQuery the query parameter value including prefix (e.g. eq2020-01-01)
-	 * @param theExpectedMatch true if theQuery should match theResourceDate.
-	 */
-	@ParameterizedTest
-	// use @CsvSource to debug individual cases.
-	//@CsvSource("2021-01-01,eq2020,false")
-	@MethodSource("dateSearchCases")
-	@CsvFileSource(resources = "/r4/date-search-test-case.csv", numLinesToSkip = 1)
-	public void testDateSearchMatching(String theResourceDate, String theQuery, Boolean theExpectedMatch) {
-
-		createObservationWithEffective("OBS1", theResourceDate);
-
-		SearchParameterMap map = SearchParameterMap.newSynchronous();
-		map.add(Observation.SP_DATE, new DateParam(theQuery));
-		IBundleProvider results = myObservationDao.search(map);
-		List<String> values = toUnqualifiedVersionlessIdValues(results);
-		boolean matched = !values.isEmpty();
-		assertEquals(theExpectedMatch, matched, "Expected " + theQuery + " to " + (theExpectedMatch?"":"not ") +"match " + theResourceDate);
-	}
-
-	/**
-	 * helper for compressed format of date test cases.
-	 *
-	 * The csv has rows with: Matching prefixes, Query Date, Resource Date
-	 * E.g. "eq ge le,2020, 2020"
-	 * This helper expands that one line into test for all of eq, ge, gt, le, lt, and ne,
-	 * expecting the listed prefixes to match, and the unlisted ones to not match.
-	 *
-	 * @return the individual test case arguments for testDateSearchMatching()
-	 */
-	public static List<Arguments> dateSearchCases() throws IOException {
-		Set<String> supportedPrefixes = CollectionUtil.newSet("eq","ge","gt","le","lt","ne");
-
-		List<String> testCaseLines = IOUtils.readLines(FhirResourceDaoR4SearchNoFtTest.class.getResourceAsStream("/r4/date-prefix-test-cases.csv"), StandardCharsets.UTF_8);
-		testCaseLines.remove(0); // first line is csv header.
-
-		// expand these into individual tests for each prefix.
-		List<Arguments> testCases = new ArrayList<>();
-		for (String line: testCaseLines) {
-			// line looks like: "eq ge le,2020, 2020"
-			// Matching prefixes, Query Date, Resource Date
-			String[] fields = line.split(",");
-			String truePrefixes = fields[0].trim();
-			String queryValue = fields[1].trim();
-			String resourceValue = fields[2].trim();
-
-			Set<String> expectedTruePrefixes = Arrays.stream(truePrefixes.split(" +")).map(String::trim).collect(Collectors.toSet());
-
-			for (String prefix: supportedPrefixes) {
-				boolean expectMatch = expectedTruePrefixes.contains(prefix);
-				testCases.add(Arguments.of(resourceValue, prefix + queryValue, expectMatch));
-			}
-		}
-
-		return testCases;
-	}
 
 	private void createObservationWithEffective(String theId, String theEffective) {
 		Observation obs = new Observation();

hapi-fhir-jpaserver-base/src/test/resources/r4/date-prefix-test-cases.csv

@@ -1,5 +0,0 @@
-Matching prefixes, Query Date, Resource Date
-eq ge le,2020, 2020
-gt ge ne,2020, 2021
-lt le ne,2021, 2020
-ne gt ge,2020,2021-01-01

hapi-fhir-test-utilities/pom.xml

@@ -131,6 +131,11 @@
 			<artifactId>junit-jupiter-engine</artifactId>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-params</artifactId>
+			<scope>compile</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.testcontainers</groupId>
 			<artifactId>testcontainers</artifactId>
@@ -150,8 +155,7 @@
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
         </dependency>
-
-    </dependencies>
+	</dependencies>
 
 	<build>
 		<plugins>

hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/jpa/conformance/DateSearchTestCase.java

@@ -0,0 +1,116 @@
+package ca.uhn.fhir.jpa.conformance;
+
+import ca.uhn.fhir.util.CollectionUtil;
+import org.junit.jupiter.params.provider.Arguments;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Collection of test cases for date type search.
+ *
+ * Each test case includes a resource value, a query value, the operator to test, and the expected result.
+ *
+ * @see <a href="https://www.hl7.org/fhir/search.html#date">the spec</a>
+ */
+public class DateSearchTestCase {
+	final String myResourceValue;
+	final String myQueryValue;
+	final boolean expectedResult;
+	final String myFileName;
+	final int myLineNumber;
+
+	public DateSearchTestCase(String myResourceValue, String myQueryValue, boolean expectedResult, String theFileName, int theLineNumber) {
+		this.myResourceValue = myResourceValue;
+		this.myQueryValue = myQueryValue;
+		this.expectedResult = expectedResult;
+		this.myFileName = theFileName;
+		this.myLineNumber = theLineNumber;
+	}
+
+	public Arguments toArguments() {
+		return Arguments.of(myResourceValue, myQueryValue, expectedResult, myFileName, myLineNumber);
+	}
+
+	/**
+	 * We have two sources of test cases:
+	 * - DateSearchTestCase.csv which holds one test case per line
+	 * - DateSearchTestCase-compact.csv which specifies all operators for each value pair
+ 	 */
+	public final static List<DateSearchTestCase> ourCases;
+	static {
+		String csv = "DateSearchTestCase.csv";
+		InputStream resource = DateSearchTestCase.class.getResourceAsStream(csv);
+		assert resource != null;
+		InputStreamReader inputStreamReader = new InputStreamReader(resource, StandardCharsets.UTF_8);
+		ourCases = parseCsvCases(inputStreamReader, csv);
+		try {
+			resource.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		ourCases.addAll(compactCases());
+	}
+
+	static List<DateSearchTestCase> parseCsvCases(Reader theSource, String theFileName) {
+		LineNumberReader lineNumberReader = new LineNumberReader(theSource);
+		return lineNumberReader.lines()
+			.filter(l->!l.startsWith("#")) // strip comments
+			.map(l -> l.split(","))
+			.map(fields -> new DateSearchTestCase(fields[0].trim(), fields[1].trim(), Boolean.parseBoolean(fields[2].trim()), theFileName, lineNumberReader.getLineNumber()))
+			.collect(Collectors.toList());
+	}
+
+	public static List<DateSearchTestCase> compactCases() {
+		String compactCsv = "DateSearchTestCase-compact.csv";
+		InputStream compactStream = DateSearchTestCase.class.getResourceAsStream(compactCsv);
+		assert compactStream != null;
+		return expandPrefixCases(new InputStreamReader(compactStream, StandardCharsets.UTF_8), compactCsv);
+	}
+
+	/**
+	 * helper for compressed format of date test cases.
+	 * <p>
+	 * The csv has rows with: Matching prefixes, Query Date, Resource Date
+	 * E.g. "eq ge le,2020, 2020"
+	 * This helper expands that one line into test for all of: eq, ge, gt, le, lt, and ne,
+	 * expecting the listed prefixes to match, and the unlisted ones to not match.
+	 *
+	 * @return List of test cases
+	 */
+	@Nonnull
+	static List<DateSearchTestCase> expandPrefixCases(Reader theSource, String theFileName) {
+		Set<String> supportedPrefixes = CollectionUtil.newSet("eq", "ge", "gt", "le", "lt", "ne");
+
+		// expand these into individual tests for each prefix.
+		LineNumberReader lineNumberReader = new LineNumberReader(theSource);
+		return lineNumberReader.lines()
+			.filter(l->!l.startsWith("#")) // strip comments
+			.map(l -> l.split(","))
+			.flatMap(fields -> {
+				// line looks like: "eq ge le,2020, 2020"
+				// Matching prefixes, Query Date, Resource Date
+				String resourceValue = fields[0].trim();
+				String truePrefixes = fields[1].trim();
+				String queryValue = fields[2].trim();
+				Set<String> expectedTruePrefixes = Arrays.stream(truePrefixes.split(" +")).map(String::trim).collect(Collectors.toSet());
+
+				// expand to one test case per supportedPrefixes
+				return supportedPrefixes.stream()
+					.map(prefix -> {
+						boolean expectMatch = expectedTruePrefixes.contains(prefix);
+						return new DateSearchTestCase(resourceValue, prefix + queryValue, expectMatch, theFileName, lineNumberReader.getLineNumber());
+					});
+			})
+			.collect(Collectors.toList());
+	}
+}

hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/jpa/conformance/package-info.java

@@ -0,0 +1,6 @@
+/**
+ * Collection of fhir standard behaviour tests.
+ *
+ * These require binding into specific contexts (JPA Spring test, full server IT, etc.)
+ */
+package ca.uhn.fhir.jpa.conformance;

hapi-fhir-test-utilities/src/main/resources/ca/uhn/fhir/jpa/conformance/DateSearchTestCase-compact.csv

@@ -0,0 +1,5 @@
+#Resource Date, Matching prefixes, Query Date,
+2020, eq ge le,2020,
+2021, gt ge ne,2020,
+2020, lt le ne,2021,
+2021-01-01, ne gt ge,2020

hapi-fhir-test-utilities/src/main/resources/ca/uhn/fhir/jpa/conformance/DateSearchTestCase.csv

@@ -1,5 +1,5 @@
-ObservationDate,Query, Result
-2020,eq2020, True
+#ObservationDate, Query, Result, Comment (ignored)
+2020,eq2020, True, A harmless comment
 2021,eq2020, False
 2020-01,eq2020, True
 2020-12,eq2020, True
@@ -41,15 +41,15 @@ ObservationDate,Query, Result
 2021-01-01,eq2020-01-01T08:00:00.000, False
 2020-01-01T08:00:00.000,eq2020-01-01T08:00:00.000, True
 2019-12-31T08:00:00.000,eq2020-01-01T08:00:00.000, False
-#2020-01-01T13:00:00.000Z,eq2020-01-01T08:00:00.000, True
-2019-01-01T12:00:00.000Z,eq2020-01-01T08:00:00.000, False
+#2020-01-01T13:00:00.000Z,eq2020-01-01T08:00:00.000, True, assumes -04 tz - not safe
+2019-01-01T12:00:00.000Z,eq2020-01-01T08:00:00.000, False, assumes < +16 tz.  OK.
 2020,eq2020-01-01T08:00:00.000Z, False
 2021,eq2020-01-01T08:00:00.000Z, False
 2020-01,eq2020-01-01T08:00:00.000Z, False
 2021-01,eq2020-01-01T08:00:00.000Z, False
 2020-01-01,eq2020-01-01T08:00:00.000Z, False
 2021-01-01,eq2020-01-01T08:00:00.000Z, False
-#2020-01-01T03:00:00.000,eq2020-01-01T08:00:00.000Z, True
+#2020-01-01T03:00:00.000,eq2020-01-01T08:00:00.000Z, True, assumes -04tz - not safe
 2019-12-31T08:00:00.000,eq2020-01-01T08:00:00.000Z, False
 2020-01-01T08:00:00.000Z,eq2020-01-01T08:00:00.000Z, True
 2019-01-01T12:00:00.000Z,eq2020-01-01T08:00:00.000Z, False
@@ -89,7 +89,7 @@ ObservationDate,Query, Result
 2019,gt2020-01-01T08:00:00.000, False
 2020-01,gt2020-01-01T08:00:00.000, True
 2021-01,gt2020-01-01T08:00:00.000, True
-2020-01-01,gt2020-01-01T08:00:00.000, True
+#2020-01-01,gt2020-01-01T08:00:00.000, False, Dodgy case - depends on our local time which we randomize
 2019-12-31,gt2020-01-01T08:00:00.000, False
 2020-01-01T08:00:00.000,gt2020-01-01T08:00:00.000, False
 2021-12-31T08:00:00.000,gt2020-01-01T08:00:00.000, True
@@ -99,9 +99,7 @@ ObservationDate,Query, Result
 2019,gt2020-01-01T08:00:00.000Z, False
 2020-01,gt2020-01-01T08:00:00.000Z, True
 2019-12,gt2020-01-01T08:00:00.000Z, False
-2020-01-01,gt2020-01-01T08:00:00.000Z, True
-2021-01-01,gt2020-01-01T08:00:00.000Z, True
-#2020-01-01T08:00:00.000,gt2020-01-01T08:00:00.000Z, True
+#2020-01-01,gt2020-01-01T08:00:00.000Z, True
 2019-12-31T08:00:00.000,gt2020-01-01T08:00:00.000Z, False
 2020-01-01T08:00:00.000Z,gt2020-01-01T08:00:00.000Z, False
 2019-01-01T12:00:00.000Z,gt2020-01-01T08:00:00.000Z, False
@@ -160,4 +158,4 @@ ObservationDate,Query, Result
 2020-01-01T08:00:00.000,lt2020-01-01T08:00:00.000Z, False
 2019-12-31T00:00:00.000,lt2020-01-01T08:00:00.000Z, True
 2020-01-01T08:00:00.000Z,lt2020-01-01T08:00:00.000Z, False
-2019-01-01T12:00:00.000Z,lt2020-01-01T08:00:00.000Z, True
+2019-01-01T12:00:00.000Z,lt2020-01-01T08:00:00.000Z, True