Merge 3c776ab363 into fb7571185a

* Composite Unique Search Parameter with dateTime component does not reject resource duplication - failing tests

* Composite Unique Search Parameter with dateTime component does not reject resource duplication - implementation

* Composite Unique Search Parameter with dateTime component does not reject resource duplication - changelog and test fixes

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6261-upgrade-to-jakarta-mail.yaml

@@ -0,0 +1,5 @@
+---
+type: change
+issue: 6261
+title: "Upgrading to Jakarta had caused a problem with the version of java-simple-mail that was in use. This has been updated to be conformant
+with the Jakarta Mail APIs. Thanks to Thomas Papke(@thopap) for the contribution!"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6317-fix-resource-duplication-for-composite-unique-sp-with-date-time-component.yaml

@@ -0,0 +1,7 @@
+---
+type: fix
+issue: 6317
+title: "Previously, defining a unique combo Search Parameter with the DateTime component and submitting multiple 
+resources with the same dateTime element (e.g. Observation.effectiveDateTime) resulted in duplicate resource creation. 
+This has been fixed."
+

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/QueryStack.java

@@ -2675,6 +2675,7 @@ public class QueryStack {
 			}
 		}
 
+		// LUKETODO:  we already have the HASH_SYS WHERE SQL here and nothing else
 		return toAndPredicate(andPredicates);
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -297,11 +297,14 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 				continue;
 			}
 			List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName);
-			Condition predicate = theQueryStack.searchForIdsWithAndOr(with().setResourceName(myResourceName)
+			final QueryStack.SearchForIdsParams searchForIdsParams = with().setResourceName(myResourceName)
 					.setParamName(nextParamName)
 					.setAndOrParams(andOrParams)
 					.setRequest(theRequest)
-					.setRequestPartitionId(myRequestPartitionId));
+					//				.setSourceJoinColumn(theQueryStack.) ????
+					.setRequestPartitionId(myRequestPartitionId);
+			// LUKETODO:  setResourceName appears NOT to work here?!?!?!??!??!?!
+			Condition predicate = theQueryStack.searchForIdsWithAndOr(searchForIdsParams);
 			if (predicate != null) {
 				theSearchSqlBuilder.addPredicate(predicate);
 			}
@@ -677,14 +680,26 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 
 		// If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
 		// partition ID predicate in that case.
-		if (!sqlBuilder.haveAtLeastOnePredicate()) {
-			Condition partitionIdPredicate = sqlBuilder
-					.getOrCreateResourceTablePredicateBuilder()
-					.createPartitionIdPredicate(myRequestPartitionId);
-			if (partitionIdPredicate != null) {
-				sqlBuilder.addPredicate(partitionIdPredicate);
-			}
+		// LUKETODO:  commenting this out seems to fix the problem
+		//		if (!sqlBuilder.haveAtLeastOnePredicate()) {
+		// LUKETODO: this is where we add the RES_TYPE clause
+		// LUKETODO: we skip this in the failing case because there's no predicate??????
+		Condition partitionIdPredicate =
+				sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
+		if (partitionIdPredicate != null) {
+			sqlBuilder.addPredicate(partitionIdPredicate);
 		}
+		//		}
+		//		// If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
+
+		//		// partition ID predicate in that case.
+		//		if (!sqlBuilder.haveAtLeastOnePredicate()) {
+		//			Condition partitionIdPredicate =
+		// sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
+		//			if (partitionIdPredicate != null) {
+		//				sqlBuilder.addPredicate(partitionIdPredicate);
+		//			}
+		//		}
 
 		// Add PID list predicate for full text search and/or lastn operation
 		addPidListPredicate(thePidList, sqlBuilder);
@@ -743,6 +758,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	private void executeSearch(
 			Integer theOffset, List<ISearchQueryExecutor> theSearchQueryExecutors, SearchQueryBuilder sqlBuilder) {
 		GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch);
+		ourLog.info("6086: generatedSql:\n{}\nparams:\n{}", generatedSql.getSql(), generatedSql.getBindVariables());
 		if (!generatedSql.isMatchNothing()) {
 			SearchQueryExecutor executor =
 					mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch);
@@ -1972,7 +1988,7 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 			 * The loop allows us to create multiple combo index joins if there
 			 * are multiple AND expressions for the related parameters.
 			 */
-			while (validateParamValuesAreValidForComboParam(theRequest, theParams, comboParamNames)) {
+			while (validateParamValuesAreValidForComboParam(theRequest, theParams, comboParamNames, comboParam)) {
 				applyComboSearchParam(theQueryStack, theParams, theRequest, comboParamNames, comboParam);
 			}
 		}
@@ -2068,7 +2084,10 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 	 * (e.g. <code>?date=gt2024-02-01</code>), etc.
 	 */
 	private boolean validateParamValuesAreValidForComboParam(
-			RequestDetails theRequest, @Nonnull SearchParameterMap theParams, List<String> theComboParamNames) {
+			RequestDetails theRequest,
+			@Nonnull SearchParameterMap theParams,
+			List<String> theComboParamNames,
+			RuntimeSearchParam theComboParam) {
 		boolean paramValuesAreValidForCombo = true;
 		List<List<IQueryParameterType>> paramOrValues = new ArrayList<>(theComboParamNames.size());
 
@@ -2129,6 +2148,19 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
 					break;
 				}
 			}
+
+			// Date params are not eligible for using composite unique index
+			// as index could contain date with different precision (e.g. DAY, SECOND)
+			if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.DATE
+					&& theComboParam.getComboSearchParamType() == ComboSearchParamType.UNIQUE) {
+				ourLog.debug(
+						"Search with params {} is not a candidate for combo searching - "
+								+ "Unique combo search parameter '{}' has DATE type",
+						theComboParamNames,
+						nextParamName);
+				paramValuesAreValidForCombo = false;
+				break;
+			}
 		}
 
 		if (CartesianProductUtil.calculateCartesianProductSize(paramOrValues) > 500) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java

@@ -646,6 +646,7 @@ public class SearchQueryBuilder {
 			ResourceTablePredicateBuilder resourceTable = mySqlBuilderFactory.resourceTable(this);
 			addTable(resourceTable, null);
 			if (theIncludeResourceTypeAndNonDeletedFlag) {
+				// LUKETODO:  this is where we add the RES_TYPE clause
 				Condition typeAndDeletionPredicate = resourceTable.createResourceTypeAndNonDeletedPredicates();
 				addPredicate(typeAndDeletionPredicate);
 			}

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/extractor/BaseSearchParamExtractor.java

@@ -566,7 +566,8 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
 				for (BaseResourceIndexedSearchParam nextParam : paramsListForCompositePart) {
 					IQueryParameterType nextParamAsClientParam = nextParam.toQueryParameterType();
 
-					if (nextParamAsClientParam instanceof DateParam) {
+					if (theParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE
+							&& nextParamAsClientParam instanceof DateParam) {
 						DateParam date = (DateParam) nextParamAsClientParam;
 						if (date.getPrecision() != TemporalPrecisionEnum.DAY) {
 							continue;

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/BasePartitioningR4Test.java

@@ -135,10 +135,10 @@ public abstract class BasePartitioningR4Test extends BaseJpaR4SystemTest {
 		addCreateDefaultPartition();
 		addReadDefaultPartition(); // one for search param validation
 		SearchParameter sp = new SearchParameter();
-		sp.setId("SearchParameter/patient-birthdate");
-		sp.setType(Enumerations.SearchParamType.DATE);
-		sp.setCode("birthdate");
-		sp.setExpression("Patient.birthDate");
+		sp.setId("SearchParameter/patient-gender");
+		sp.setType(Enumerations.SearchParamType.TOKEN);
+		sp.setCode("gender");
+		sp.setExpression("Patient.gender");
 		sp.setStatus(Enumerations.PublicationStatus.ACTIVE);
 		sp.addBase("Patient");
 		mySearchParameterDao.update(sp, mySrd);
@@ -156,13 +156,13 @@ public abstract class BasePartitioningR4Test extends BaseJpaR4SystemTest {
 
 		addCreateDefaultPartition();
 		sp = new SearchParameter();
-		sp.setId("SearchParameter/patient-birthdate-unique");
+		sp.setId("SearchParameter/patient-gender-family-unique");
 		sp.setType(Enumerations.SearchParamType.COMPOSITE);
 		sp.setStatus(Enumerations.PublicationStatus.ACTIVE);
 		sp.addBase("Patient");
 		sp.addComponent()
 			.setExpression("Patient")
-			.setDefinition("SearchParameter/patient-birthdate");
+			.setDefinition("SearchParameter/patient-gender");
 		sp.addComponent()
 			.setExpression("Patient")
 			.setDefinition("SearchParameter/patient-family");

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboUniqueParamTest.java

@@ -13,10 +13,10 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
 import ca.uhn.fhir.rest.api.server.IBundleProvider;
-import ca.uhn.fhir.rest.param.DateAndListParam;
-import ca.uhn.fhir.rest.param.DateOrListParam;
 import ca.uhn.fhir.rest.param.DateParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
+import ca.uhn.fhir.rest.param.StringOrListParam;
+import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.param.TokenAndListParam;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
@@ -33,6 +33,7 @@ import org.hl7.fhir.r4.model.DateType;
 import org.hl7.fhir.r4.model.Encounter;
 import org.hl7.fhir.r4.model.Enumerations;
 import org.hl7.fhir.r4.model.Enumerations.PublicationStatus;
+import org.hl7.fhir.r4.model.HumanName;
 import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Organization;
@@ -115,6 +116,46 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		myMessages.clear();
 	}
 
+	private void createUniqueGenderFamilyComboSp() {
+		SearchParameter sp = new SearchParameter();
+		sp.setId("SearchParameter/patient-gender");
+		sp.setType(Enumerations.SearchParamType.TOKEN);
+		sp.setCode("gender");
+		sp.setExpression("Patient.gender");
+		sp.setStatus(PublicationStatus.ACTIVE);
+		sp.addBase("Patient");
+		mySearchParameterDao.update(sp, mySrd);
+
+		sp = new SearchParameter();
+		sp.setId("SearchParameter/patient-family");
+		sp.setType(Enumerations.SearchParamType.STRING);
+		sp.setCode("family");
+		sp.setExpression("Patient.name.family");
+		sp.setStatus(Enumerations.PublicationStatus.ACTIVE);
+		sp.addBase("Patient");
+		mySearchParameterDao.update(sp, mySrd);
+
+		sp = new SearchParameter();
+		sp.setId("SearchParameter/patient-gender-family");
+		sp.setType(Enumerations.SearchParamType.COMPOSITE);
+		sp.setStatus(PublicationStatus.ACTIVE);
+		sp.addBase("Patient");
+		sp.addComponent()
+			.setExpression("Patient")
+			.setDefinition("SearchParameter/patient-gender");
+		sp.addComponent()
+			.setExpression("Patient")
+			.setDefinition("SearchParameter/patient-family");
+		sp.addExtension()
+			.setUrl(HapiExtensions.EXT_SP_UNIQUE)
+			.setValue(new BooleanType(true));
+		mySearchParameterDao.update(sp, mySrd);
+
+		mySearchParamRegistry.forceRefresh();
+
+		myMessages.clear();
+	}
+
 	private void createUniqueIndexCoverageBeneficiary() {
 		SearchParameter sp = new SearchParameter();
 		sp.setId("SearchParameter/coverage-beneficiary");
@@ -276,6 +317,45 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		mySearchParamRegistry.forceRefresh();
 	}
 
+	private void createUniqueObservationDateCode() {
+		SearchParameter sp = new SearchParameter();
+		sp.setId("SearchParameter/obs-effective");
+		sp.setType(Enumerations.SearchParamType.DATE);
+		sp.setCode("date");
+		sp.setExpression("Observation.effective");
+		sp.setStatus(PublicationStatus.ACTIVE);
+		sp.addBase("Observation");
+		mySearchParameterDao.update(sp, mySrd);
+
+		sp = new SearchParameter();
+		sp.setId("SearchParameter/obs-code");
+		sp.setType(Enumerations.SearchParamType.TOKEN);
+		sp.setCode("code");
+		sp.setExpression("Observation.code");
+		sp.setStatus(PublicationStatus.ACTIVE);
+		sp.addBase("Observation");
+		mySearchParameterDao.update(sp, mySrd);
+
+		sp = new SearchParameter();
+		sp.setId("SearchParameter/observation-date-code");
+		sp.setType(Enumerations.SearchParamType.COMPOSITE);
+		sp.setStatus(PublicationStatus.ACTIVE);
+		sp.addBase("Observation");
+		sp.setExpression("Observation.code");
+		sp.addComponent()
+			.setExpression("Observation")
+			.setDefinition("SearchParameter/obs-effective");
+		sp.addComponent()
+			.setExpression("Observation")
+			.setDefinition("SearchParameter/obs-code");
+		sp.addExtension()
+			.setUrl(HapiExtensions.EXT_SP_UNIQUE)
+			.setValue(new BooleanType(true));
+		mySearchParameterDao.update(sp, mySrd);
+
+		mySearchParamRegistry.forceRefresh();
+	}
+
 	private void createUniqueObservationSubjectDateCode() {
 		SearchParameter sp = new SearchParameter();
 		sp.setId("SearchParameter/obs-subject");
@@ -471,11 +551,11 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 	public void testDoubleMatchingOnAnd_Search_TwoAndOrValues() {
 		myStorageSettings.setUniqueIndexesCheckedBeforeSave(false);
 
-		createUniqueBirthdateAndGenderSps();
+		createUniqueGenderFamilyComboSp();
 
 		Patient pt1 = new Patient();
 		pt1.setGender(Enumerations.AdministrativeGender.MALE);
-		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		String id1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless().getValue();
 
 		// Two OR values on the same resource - Currently composite SPs don't work for this
@@ -484,17 +564,22 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		sp.setLoadSynchronous(true);
 		sp.add(Patient.SP_GENDER,
 			new TokenAndListParam()
-				.addAnd(new TokenParam("http://hl7.org/fhir/administrative-gender","male"), new TokenParam( "http://hl7.org/fhir/administrative-gender","female"))
+				.addAnd(new TokenParam("http://hl7.org/fhir/administrative-gender","male"),
+					new TokenParam( "http://hl7.org/fhir/administrative-gender","female"))
 		);
-		sp.add(Patient.SP_BIRTHDATE,
-			new DateAndListParam()
-				.addAnd(new DateParam("2011-01-01"), new DateParam( "2011-02-02"))
+		sp.add(Patient.SP_FAMILY,
+			new StringOrListParam()
+				.addOr(new StringParam("Family1")).addOr(new StringParam("Family2"))
 		);
 		IBundleProvider outcome = myPatientDao.search(sp, mySrd);
 		myCaptureQueriesListener.logFirstSelectQueryForCurrentThread();
 		assertThat(toUnqualifiedVersionlessIdValues(outcome)).containsExactlyInAnyOrder(id1);
 		String unformattedSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
-		assertEquals("SELECT t0.RES_ID FROM HFJ_IDX_CMP_STRING_UNIQ t0 WHERE (t0.IDX_STRING IN ('Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cfemale','Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale','Patient?birthdate=2011-02-02&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cfemale','Patient?birthdate=2011-02-02&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale') )", unformattedSql);
+		assertEquals("SELECT t0.RES_ID FROM HFJ_IDX_CMP_STRING_UNIQ t0 WHERE (t0.IDX_STRING IN (" +
+			"'Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cfemale'," +
+			"'Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale'," +
+			"'Patient?family=Family2&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cfemale'," +
+			"'Patient?family=Family2&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale') )", unformattedSql);
 	}
 
 
@@ -1167,16 +1252,16 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 	@Test
 	public void testOrQuery() {
 		myStorageSettings.setAdvancedHSearchIndexing(false);
-		createUniqueBirthdateAndGenderSps();
+		createUniqueGenderFamilyComboSp();
 
 		Patient pt1 = new Patient();
 		pt1.setGender(Enumerations.AdministrativeGender.MALE);
-		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		IIdType id1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless();
 
 		Patient pt2 = new Patient();
 		pt2.setGender(Enumerations.AdministrativeGender.MALE);
-		pt2.setBirthDateElement(new DateType("2011-01-02"));
+		pt2.getName().add(new HumanName().setFamily("Family2"));
 		IIdType id2 = myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless();
 
 		myCaptureQueriesListener.clear();
@@ -1184,16 +1269,21 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		SearchParameterMap params = new SearchParameterMap();
 		params.setLoadSynchronousUpTo(100);
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
-		params.add("birthdate", new DateOrListParam().addOr(new DateParam("2011-01-01")).addOr(new DateParam("2011-01-02")));
+		params.add("family", new StringOrListParam()
+			.addOr(new StringParam("Family1")).addOr(new StringParam("Family2")));
 		myCaptureQueriesListener.clear();
 		IBundleProvider results = myPatientDao.search(params, mySrd);
 		myCaptureQueriesListener.logFirstSelectQueryForCurrentThread();
 		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(id1.getValue(), id2.getValue());
 
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false))
-			.contains("SELECT t0.RES_ID FROM HFJ_IDX_CMP_STRING_UNIQ t0 WHERE (t0.IDX_STRING IN ('Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale','Patient?birthdate=2011-01-02&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale') )");
+			.contains("SELECT t0.RES_ID FROM HFJ_IDX_CMP_STRING_UNIQ t0 WHERE (t0.IDX_STRING IN " +
+				"('Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale'," +
+				"'Patient?family=Family2&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale') )");
 		logCapturedMessages();
-		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: [Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale, Patient?birthdate=2011-01-02&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale]");
+		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: " +
+			"[Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale, " +
+			"Patient?family=Family2&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale]");
 		myMessages.clear();
 
 	}
@@ -1201,16 +1291,16 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 	@Test
 	public void testSearchSynchronousUsingUniqueComposite() {
 		myStorageSettings.setAdvancedHSearchIndexing(false);
-		createUniqueBirthdateAndGenderSps();
+		createUniqueGenderFamilyComboSp();
 
 		Patient pt1 = new Patient();
 		pt1.setGender(Enumerations.AdministrativeGender.MALE);
-		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		IIdType id1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless();
 
 		Patient pt2 = new Patient();
 		pt2.setGender(Enumerations.AdministrativeGender.MALE);
-		pt2.setBirthDateElement(new DateType("2011-01-02"));
+		pt2.getName().add(new HumanName().setFamily("Family2"));
 		myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless();
 
 		myCaptureQueriesListener.clear();
@@ -1218,13 +1308,14 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		SearchParameterMap params = new SearchParameterMap();
 		params.setLoadSynchronousUpTo(100);
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
-		params.add("birthdate", new DateParam("2011-01-01"));
+		params.add("family", new StringParam("Family1"));
 		IBundleProvider results = myPatientDao.search(params, mySrd);
 		myCaptureQueriesListener.logFirstSelectQueryForCurrentThread();
 		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(id1.getValue());
 
 		logCapturedMessages();
-		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
+		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: " +
+			"Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
 		myMessages.clear();
 
 	}
@@ -1232,33 +1323,34 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 
 	@Test
 	public void testSearchUsingUniqueComposite() {
-		createUniqueBirthdateAndGenderSps();
+		createUniqueGenderFamilyComboSp();
 
 		Patient pt1 = new Patient();
 		pt1.setGender(Enumerations.AdministrativeGender.MALE);
-		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		String id1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless().getValue();
 
 		Patient pt2 = new Patient();
 		pt2.setGender(Enumerations.AdministrativeGender.MALE);
-		pt2.setBirthDateElement(new DateType("2011-01-02"));
+		pt2.getName().add(new HumanName().setFamily("Family2"));
 		myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless();
 
 		myMessages.clear();
 		SearchParameterMap params = new SearchParameterMap();
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
-		params.add("birthdate", new DateParam("2011-01-01"));
+		params.add("family", new StringParam("Family1"));
 		IBundleProvider results = myPatientDao.search(params, mySrd);
 		String searchId = results.getUuid();
 		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(id1);
 		logCapturedMessages();
-		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
+		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: " +
+			"Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
 		myMessages.clear();
 
 		// Other order
 		myMessages.clear();
 		params = new SearchParameterMap();
-		params.add("birthdate", new DateParam("2011-01-01"));
+		params.add("family", new StringParam("Family1"));
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
 		results = myPatientDao.search(params, mySrd);
 		assertEquals(searchId, results.getUuid());
@@ -1272,16 +1364,17 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		myMessages.clear();
 		params = new SearchParameterMap();
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
-		params.add("birthdate", new DateParam("2011-01-03"));
+		params.add("family", new StringParam("Family3"));
 		results = myPatientDao.search(params, mySrd);
 		assertThat(toUnqualifiedVersionlessIdValues(results)).isEmpty();
 		logCapturedMessages();
-		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: Patient?birthdate=2011-01-03&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
+		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: " +
+			"Patient?family=Family3&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
 		myMessages.clear();
 
 		myMessages.clear();
 		params = new SearchParameterMap();
-		params.add("birthdate", new DateParam("2011-01-03"));
+		params.add("family", new StringParam("Family3"));
 		results = myPatientDao.search(params, mySrd);
 		assertThat(toUnqualifiedVersionlessIdValues(results)).isEmpty();
 		// STANDARD QUERY
@@ -1666,7 +1759,7 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 	}
 
 	@Test
-	public void testDuplicateUniqueValuesAreRejected() {
+	public void testDuplicateUniqueValuesWithDateAreRejected() {
 		createUniqueBirthdateAndGenderSps();
 
 		Patient pt1 = new Patient();
@@ -1699,13 +1792,75 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 	}
 
 	@Test
-	public void testReplaceOneWithAnother() {
-		myStorageSettings.setAdvancedHSearchIndexing(false);
+	public void testDuplicateUniqueValuesWithDateTimeAreRejected() {
+		createUniqueObservationDateCode();
+
+		Observation obs = new Observation();
+		obs.getCode().addCoding().setSystem("foo").setCode("bar");
+		obs.setEffective(new DateTimeType("2017-10-10T00:00:00"));
+		myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless();
+
+		try {
+			myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless();
+			fail();
+		} catch (ResourceVersionConflictException e) {
+			assertThat(e.getMessage())
+				.contains("new unique index created by SearchParameter/observation-date-code");
+		}
+	}
+
+	@Test
+	public void testUniqueComboSearchWithDateNotUsingUniqueIndex() {
 		createUniqueBirthdateAndGenderSps();
 
 		Patient pt1 = new Patient();
 		pt1.setGender(Enumerations.AdministrativeGender.MALE);
 		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		String pId = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless().getValue();
+
+		myCaptureQueriesListener.clear();
+		SearchParameterMap params = new SearchParameterMap();
+		params.setLoadSynchronousUpTo(100);
+		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
+		params.add("birthdate", new DateParam("2011-01-01"));
+
+		IBundleProvider results = myPatientDao.search(params, mySrd);
+		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(pId);
+		myCaptureQueriesListener.logFirstSelectQueryForCurrentThread();
+		String unformattedSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
+		assertThat(unformattedSql).doesNotContain("HFJ_IDX_CMP_STRING_UNIQ");
+	}
+
+	@Test
+	public void testUniqueComboSearchWithDateTimeNotUsingUniqueIndex() {
+		createUniqueObservationDateCode();
+
+		Observation obs = new Observation();
+		obs.getCode().addCoding().setSystem("foo").setCode("bar");
+		obs.setEffective(new DateTimeType("2017-10-10T00:00:00"));
+		String obsId = myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless().getValue();
+
+		myCaptureQueriesListener.clear();
+		SearchParameterMap params = new SearchParameterMap();
+		params.setLoadSynchronousUpTo(100);
+		params.add("code", new TokenParam("foo", "bar"));
+		params.add("date", new DateParam("2017-10-10T00:00:00"));
+
+		IBundleProvider results = myObservationDao.search(params, mySrd);
+		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(obsId);
+		myCaptureQueriesListener.logFirstSelectQueryForCurrentThread();
+		String unformattedSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
+		assertThat(unformattedSql).doesNotContain("HFJ_IDX_CMP_STRING_UNIQ");
+	}
+
+	@Test
+	public void testReplaceOneWithAnother() {
+		myStorageSettings.setAdvancedHSearchIndexing(false);
+		createUniqueGenderFamilyComboSp();
+
+		Patient pt1 = new Patient();
+		pt1.setGender(Enumerations.AdministrativeGender.MALE);
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		IIdType id1 = myPatientDao.create(pt1, mySrd).getId().toUnqualified();
 		assertNotNull(id1);
 
@@ -1714,27 +1869,28 @@ public class FhirResourceDaoR4ComboUniqueParamTest extends BaseComboParamsR4Test
 		pt1 = new Patient();
 		pt1.setId(id1);
 		pt1.setGender(Enumerations.AdministrativeGender.FEMALE);
-		pt1.setBirthDateElement(new DateType("2011-01-01"));
+		pt1.getName().add(new HumanName().setFamily("Family1"));
 		id1 = myPatientDao.update(pt1, mySrd).getId().toUnqualified();
 		assertNotNull(id1);
 		assertEquals("2", id1.getVersionIdPart());
 
 		Patient pt2 = new Patient();
 		pt2.setGender(Enumerations.AdministrativeGender.MALE);
-		pt2.setBirthDateElement(new DateType("2011-01-01"));
+		pt2.getName().add(new HumanName().setFamily("Family1"));
 		IIdType id2 = myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless();
 
 		myMessages.clear();
 		SearchParameterMap params = new SearchParameterMap();
 		params.add("gender", new TokenParam("http://hl7.org/fhir/administrative-gender", "male"));
-		params.add("birthdate", new DateParam("2011-01-01"));
+		params.add("family", new StringParam("Family1"));
 		IBundleProvider results = myPatientDao.search(params, mySrd);
 		String searchId = results.getUuid();
 		assertThat(searchId).isNotBlank();
 		assertThat(toUnqualifiedVersionlessIdValues(results)).containsExactlyInAnyOrder(id2.getValue());
 
 		logCapturedMessages();
-		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: Patient?birthdate=2011-01-01&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
+		assertThat(myMessages.toString()).contains("Using UNIQUE index(es) for query for search: " +
+			"Patient?family=Family1&gender=http%3A%2F%2Fhl7.org%2Ffhir%2Fadministrative-gender%7Cmale");
 		myMessages.clear();
 
 	}

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QuerySandbox.java

@@ -1,8 +1,11 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
+import ca.uhn.fhir.batch2.jobs.step.ResourceIdListStep;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.interceptor.api.Hook;
 import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.interceptor.model.RequestPartitionId;
+import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc;
 import ca.uhn.fhir.jpa.dao.TestDaoSearch;
 import ca.uhn.fhir.jpa.test.BaseJpaTest;
 import ca.uhn.fhir.jpa.test.config.TestHSearchAddInConfig;
@@ -30,7 +33,10 @@ import org.springframework.test.context.support.DependencyInjectionTestExecution
 import org.springframework.test.context.support.DirtiesContextTestExecutionListener;
 import org.springframework.transaction.PlatformTransactionManager;
 
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
@@ -66,6 +72,8 @@ public class FhirResourceDaoR4QuerySandbox extends BaseJpaTest {
 	DaoTestDataBuilder myDataBuilder;
 	@Autowired
 	TestDaoSearch myTestDaoSearch;
+	@Autowired
+	IBatch2DaoSvc myBatch2DaoSvc;
 
 	@Override
 	protected PlatformTransactionManager getTxManager() {
@@ -162,6 +170,42 @@ public class FhirResourceDaoR4QuerySandbox extends BaseJpaTest {
 		assertTrue(actualIds.containsAll(List.of(id1, id2, id3)));
 	}
 
+	// LUKETODO:   consider @Nested class
+	@Test
+	void testDeleteExpunge1() {
+//		SearchParameterMap[params={event=[[TokenParam[system=http://terminology.hl7.org/CodeSystem/v2-0003,value=], TokenParam[system=https://harris-chr.com/fhir/CodeSystem/arc-interface-trigger,value=]]]}]
+//		MessageHeader?event=http://terminology.hl7.org/CodeSystem/v2-0003|,https://harris-chr.com/fhir/CodeSystem/arc-interface-trigger|&_lastUpdated=ge2023-09-17T00:00:00.000000Z&_lastUpdated=le2025-09-23T23:59:59.999999Z&_expunge=true
+//		myTestDaoSearch.searchForResources("MessageHeader?event=http://terminology.hl7.org/CodeSystem/v2-0003|,https://harris-chr.com/fhir/CodeSystem/arc-interface-trigger|&_lastUpdated=ge2023-09-17T00:00:00.000000Z&_lastUpdated=le2025-09-23T23:59:59.999999Z&_expunge=true");
+		final LocalDateTime startDateTime = LocalDateTime.now().minusDays(2);
+		final Date startDate = Date.from(startDateTime.atZone(ZoneId.systemDefault()).toInstant());
+		final LocalDateTime endDateTime = LocalDateTime.now().minusDays(1);
+		final Date endDate = Date.from(endDateTime.atZone(ZoneId.systemDefault()).toInstant());
+
+		myBatch2DaoSvc.fetchResourceIdStream(startDate, endDate, RequestPartitionId.allPartitions(), "MessageHeader?event=http://terminology.hl7.org/CodeSystem/v2-0003|,https://harris-chr.com/fhir/CodeSystem/arc-interface-trigger|&_lastUpdated=ge2023-09-17T00:00:00.000000Z&_lastUpdated=le2025-09-23T23:59:59.999999Z&_expunge=true")
+			.visitStreamNoResult(x -> {
+				var z = x.toList();
+			});
+		final List<String> queries = myCapturedQueries.stream().toList();
+
+		ourLog.info("queries: {}", queries);
+	}
+
+	@Test
+	void testDeleteExpunge2() {
+		final LocalDateTime startDateTime = LocalDateTime.now().minusDays(2);
+		final Date startDate = Date.from(startDateTime.atZone(ZoneId.systemDefault()).toInstant());
+		final LocalDateTime endDateTime = LocalDateTime.now().minusDays(1);
+		final Date endDate = Date.from(endDateTime.atZone(ZoneId.systemDefault()).toInstant());
+
+		myBatch2DaoSvc.fetchResourceIdStream(startDate, endDate, RequestPartitionId.allPartitions(), "MessageHeader?_lastUpdated=ge2023-09-17T00:00:00.000000Z&_lastUpdated=le2025-09-23T23:59:59.999999Z&_expunge=true")
+			.visitStreamNoResult(x -> {
+				var z = x.toList();
+			});
+		final List<String> queries = myCapturedQueries.stream().toList();
+
+		ourLog.info("queries: {}", queries);
+	}
+
 	public static final class TestDirtiesContextTestExecutionListener extends DirtiesContextTestExecutionListener {
 
 		@Override

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java

@@ -413,7 +413,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		p.getMeta().addTag("http://system", "code", "diisplay");
 		p.addName().setFamily("FAM");
 		p.addIdentifier().setSystem("system").setValue("value");
-		p.setBirthDateElement(new DateType("2020-01-01"));
+		p.setGender(Enumerations.AdministrativeGender.MALE);
 		p.getManagingOrganization().setReferenceElement(orgId);
 		Long patientId = myPatientDao.create(p, mySrd).getId().getIdPartAsLong();
 
@@ -502,7 +502,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		p.getMeta().addTag("http://system", "code", "diisplay");
 		p.addName().setFamily("FAM");
 		p.addIdentifier().setSystem("system").setValue("value");
-		p.setBirthDate(new Date());
+		p.setGender(Enumerations.AdministrativeGender.MALE);
 		p.getManagingOrganization().setReferenceElement(orgId);
 		Long patientId = myPatientDao.create(p, mySrd).getId().getIdPartAsLong();
 
@@ -679,7 +679,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		p.getMeta().addTag("http://system", "code", "display");
 		p.addName().setFamily("FAM");
 		p.addIdentifier().setSystem("system").setValue("value");
-		p.setBirthDate(new Date());
+		p.setGender(Enumerations.AdministrativeGender.MALE);
 		p.getManagingOrganization().setReference(org.getId());
 		input.addEntry()
 			.setFullUrl(p.getId())
@@ -2541,14 +2541,14 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_UniqueParam_SearchAllPartitions() {
 		createUniqueComboSp();
 
-		IIdType id = createPatient(withPartition(1), withBirthdate("2020-01-01"), withFamily("FAM"));
+		IIdType id = createPatient(withPartition(1), withGender("male"), withFamily("FAM"));
 
 		addReadAllPartitions();
 
 		myCaptureQueriesListener.clear();
 		SearchParameterMap map = new SearchParameterMap();
 		map.add(Patient.SP_FAMILY, new StringParam("FAM"));
-		map.add(Patient.SP_BIRTHDATE, new DateParam("2020-01-01"));
+		map.add(Patient.SP_GENDER, new TokenParam(null, "male"));
 		map.setLoadSynchronous(true);
 		IBundleProvider results = myPatientDao.search(map, mySrd);
 		List<IIdType> ids = toUnqualifiedVersionlessIds(results);
@@ -2558,7 +2558,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 		ourLog.info("Search SQL:\n{}", searchSql);
 		assertThat(searchSql).doesNotContain("PARTITION_ID");
-		assertThat(searchSql).containsOnlyOnce("IDX_STRING = 'Patient?birthdate=2020-01-01&family=FAM'");
+		assertThat(searchSql).containsOnlyOnce("IDX_STRING = 'Patient?family=FAM&gender=male'");
 	}
 
 
@@ -2566,13 +2566,13 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_UniqueParam_SearchOnePartition() {
 		createUniqueComboSp();
 
-		IIdType id = createPatient(withPartition(1), withBirthdate("2020-01-01"), withFamily("FAM"));
+		IIdType id = createPatient(withPartition(1), withGender("male"), withFamily("FAM"));
 
 		addReadPartition(1);
 		myCaptureQueriesListener.clear();
 		SearchParameterMap map = new SearchParameterMap();
 		map.add(Patient.SP_FAMILY, new StringParam("FAM"));
-		map.add(Patient.SP_BIRTHDATE, new DateParam("2020-01-01"));
+		map.add(Patient.SP_GENDER, new TokenParam(null, "male"));
 		map.setLoadSynchronous(true);
 		IBundleProvider results = myPatientDao.search(map, mySrd);
 		List<IIdType> ids = toUnqualifiedVersionlessIds(results);
@@ -2582,13 +2582,13 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 		String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
 		ourLog.info("Search SQL:\n{}", searchSql);
 		assertThat(searchSql).containsOnlyOnce( "PARTITION_ID = '1'");
-		assertThat(searchSql).containsOnlyOnce("IDX_STRING = 'Patient?birthdate=2020-01-01&family=FAM'");
+		assertThat(searchSql).containsOnlyOnce("IDX_STRING = 'Patient?family=FAM&gender=male'");
 
 		// Same query, different partition
 		addReadPartition(2);
 		myCaptureQueriesListener.clear();
 		map = new SearchParameterMap();
-		map.add(Patient.SP_BIRTHDATE, new DateParam("2020-01-01"));
+		map.add(Patient.SP_GENDER, new TokenParam(null, "male"));
 		map.setLoadSynchronous(true);
 		results = myPatientDao.search(map, mySrd);
 		ids = toUnqualifiedVersionlessIds(results);
@@ -2661,7 +2661,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_RefParam_TargetPid_SearchOnePartition() {
 		createUniqueComboSp();
 
-		IIdType patientId = createPatient(withPartition(myPartitionId), withBirthdate("2020-01-01"));
+		IIdType patientId = createPatient(withPartition(myPartitionId), withGender("male"));
 		IIdType observationId = createObservation(withPartition(myPartitionId), withSubject(patientId));
 
 		addReadPartition(myPartitionId);
@@ -2698,7 +2698,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_RefParam_TargetPid_SearchDefaultPartition() {
 		createUniqueComboSp();
 
-		IIdType patientId = createPatient(withPartition(null), withBirthdate("2020-01-01"));
+		IIdType patientId = createPatient(withPartition(null), withGender("male"));
 		IIdType observationId = createObservation(withPartition(null), withSubject(patientId));
 
 		addReadDefaultPartition();
@@ -2735,7 +2735,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_RefParam_TargetForcedId_SearchOnePartition() {
 		createUniqueComboSp();
 
-		IIdType patientId = createPatient(withPartition(myPartitionId), withId("ONE"), withBirthdate("2020-01-01"));
+		IIdType patientId = createPatient(withPartition(myPartitionId), withId("ONE"), withGender("male"));
 		IIdType observationId = createObservation(withPartition(myPartitionId), withSubject(patientId));
 
 		addReadPartition(myPartitionId);
@@ -2805,7 +2805,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
 	public void testSearch_RefParam_TargetForcedId_SearchDefaultPartition() {
 		createUniqueComboSp();
 
-		IIdType patientId = createPatient(withPartition(null), withId("ONE"), withBirthdate("2020-01-01"));
+		IIdType patientId = createPatient(withPartition(null), withId("ONE"), withGender("male"));
 		IIdType observationId = createObservation(withPartition(null), withSubject(patientId));
 
 		addReadDefaultPartition();