lucene fulltext search fix

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6404-fulltext-search-with-lastupdate.yaml

@@ -0,0 +1,8 @@
+---
+type: fix
+issue: 6404
+title: "Searches using fulltext search that combined `_lastUpdated` query parameter with
+        any other (supported) fulltext query parameter would find no matches, even
+        if matches existed.
+        This has been corrected.
+"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/upgrade.md

@@ -0,0 +1,4 @@
+# Fulltext Search with _lastUpdated Filter
+
+Fulltext searches have been updated to support `_lastUpdated` search parameter. A reindexing of Search Parameters
+is required to migrate old data to support the `_lastUpdated` search parameter.

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -981,7 +981,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
 					entity.setIndexStatus(INDEX_STATUS_INDEXED);
 				}
 
-				if (myFulltextSearchSvc != null && !myFulltextSearchSvc.isDisabled()) {
+				if (myFulltextSearchSvc != null && !myFulltextSearchSvc.isDisabled() && changed.isChanged()) {
 					populateFullTextFields(myContext, theResource, entity, newParams);
 				}
 
@@ -1688,7 +1688,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
 			theEntity.setContentText(parseContentTextIntoWords(theContext, theResource));
 			if (myStorageSettings.isAdvancedHSearchIndexing()) {
 				ExtendedHSearchIndexData hSearchIndexData =
-						myFulltextSearchSvc.extractLuceneIndexData(theResource, theNewParams);
+						myFulltextSearchSvc.extractLuceneIndexData(theResource, theEntity, theNewParams);
 				theEntity.setLuceneIndexData(hSearchIndexData);
 			}
 		}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java

@@ -135,13 +135,13 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
 
 	@Override
 	public ExtendedHSearchIndexData extractLuceneIndexData(
-			IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
+			IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams) {
 		String resourceType = myFhirContext.getResourceType(theResource);
 		ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(
 				resourceType, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
 		ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
 				myStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
-		return extractor.extract(theResource, theNewParams);
+		return extractor.extract(theResource, theEntity, theNewParams);
 	}
 
 	@Override

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java

@@ -89,7 +89,7 @@ public interface IFulltextSearchSvc {
 	boolean isDisabled();
 
 	ExtendedHSearchIndexData extractLuceneIndexData(
-			IBaseResource theResource, ResourceIndexedSearchParams theNewParams);
+			IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams);
 
 	/**
 	 * Returns true if the parameter map can be handled for hibernate search.

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java

@@ -392,7 +392,7 @@ public class ExtendedHSearchClauseBuilder {
 
 	/**
 	 * Create date clause from date params. The date lower and upper bounds are taken
-	 * into considertion when generating date query ranges
+	 * into consideration when generating date query ranges
 	 *
 	 * <p>Example 1 ('eq' prefix/empty): <code>http://fhirserver/Observation?date=eq2020</code>
 	 * would generate the following search clause

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractor.java

@@ -25,6 +25,8 @@ import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
 import ca.uhn.fhir.jpa.model.entity.ResourceLink;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.model.entity.TagDefinition;
 import ca.uhn.fhir.jpa.model.search.CompositeSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.DateSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@@ -37,6 +39,7 @@ import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
 import ca.uhn.fhir.util.MetaUtil;
 import com.google.common.base.Strings;
 import jakarta.annotation.Nonnull;
+import org.apache.commons.lang3.ObjectUtils;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseCoding;
 import org.hl7.fhir.instance.model.api.IBaseResource;
@@ -74,8 +77,8 @@ public class ExtendedHSearchIndexExtractor {
 	}
 
 	@Nonnull
-	public ExtendedHSearchIndexData extract(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
+	public ExtendedHSearchIndexData extract(IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams) {
-		ExtendedHSearchIndexData retVal = new ExtendedHSearchIndexData(myContext, myJpaStorageSettings, theResource);
+		ExtendedHSearchIndexData retVal = new ExtendedHSearchIndexData(myContext, myJpaStorageSettings, theResource, theEntity);
 
 		if (myJpaStorageSettings.isStoreResourceInHSearchIndex()) {
 			retVal.setRawResourceData(myContext.newJsonParser().encodeResourceToString(theResource));
@@ -113,11 +116,28 @@ public class ExtendedHSearchIndexExtractor {
 				.filter(nextParam -> !nextParam.isMissing())
 				.forEach(nextParam -> retVal.addUriIndexData(nextParam.getParamName(), nextParam.getUri()));
 
-		theResource.getMeta().getTag().forEach(tag -> retVal.addTokenIndexData("_tag", tag));
 
-		theResource.getMeta().getSecurity().forEach(sec -> retVal.addTokenIndexData("_security", sec));
+		theEntity.getTags().forEach(tag -> {
+			TagDefinition td = tag.getTag();
 
-		theResource.getMeta().getProfile().forEach(prof -> retVal.addUriIndexData("_profile", prof.getValue()));
+			IBaseCoding coding = (IBaseCoding) myContext.getVersion().newCodingDt();
+			coding.setVersion(td.getVersion());
+			coding.setDisplay(td.getDisplay());
+			coding.setCode(td.getCode());
+			coding.setSystem(td.getSystem());
+			coding.setUserSelected(ObjectUtils.defaultIfNull(td.getUserSelected(), false));
+			switch (td.getTagType()) {
+				case TAG:
+					retVal.addTokenIndexData("_tag", coding);
+					break;
+				case PROFILE:
+					retVal.addUriIndexData("_profile", coding.getCode());
+					break;
+				case SECURITY_LABEL:
+					retVal.addTokenIndexData("_security", coding);
+					break;
+			}
+		});
 
 		String source = MetaUtil.getSource(myContext, theResource.getMeta());
 		if (isNotBlank(source)) {
@@ -127,20 +147,19 @@ public class ExtendedHSearchIndexExtractor {
 		theNewParams.myCompositeParams.forEach(nextParam ->
 				retVal.addCompositeIndexData(nextParam.getSearchParamName(), buildCompositeIndexData(nextParam)));
 
-		if (theResource.getMeta().getLastUpdated() != null) {
+		if (theEntity.getUpdated() != null && !theEntity.getUpdated().isEmpty()) {
 			int ordinal = ResourceIndexedSearchParamDate.calculateOrdinalValue(
-							theResource.getMeta().getLastUpdated())
+							theEntity.getUpdatedDate())
 					.intValue();
 			retVal.addDateIndexData(
 					"_lastUpdated",
-					theResource.getMeta().getLastUpdated(),
+					theEntity.getUpdatedDate(),
 					ordinal,
-					theResource.getMeta().getLastUpdated(),
+					theEntity.getUpdatedDate(),
 					ordinal);
 		}
 
 		if (!theNewParams.myLinks.isEmpty()) {
-
 			// awkwardly, links are indexed by jsonpath, not by search param.
 			// so we re-build the linkage.
 			Map<String, List<String>> linkPathToParamName = new HashMap<>();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java

@@ -28,6 +28,7 @@ import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
 import ca.uhn.fhir.rest.param.CompositeParam;
 import ca.uhn.fhir.rest.param.DateParam;
+import ca.uhn.fhir.rest.param.DateRangeParam;
 import ca.uhn.fhir.rest.param.NumberParam;
 import ca.uhn.fhir.rest.param.QuantityParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
@@ -89,19 +90,28 @@ public class ExtendedHSearchSearchBuilder {
 	 * be inaccurate and wrong.
 	 */
 	public boolean canUseHibernateSearch(
-			String theResourceType, SearchParameterMap myParams, ISearchParamRegistry theSearchParamRegistry) {
+			String theResourceType, SearchParameterMap theParams, ISearchParamRegistry theSearchParamRegistry) {
 		boolean canUseHibernate = false;
 
 		ResourceSearchParams resourceActiveSearchParams = theSearchParamRegistry.getActiveSearchParams(
 				theResourceType, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
-		for (String paramName : myParams.keySet()) {
+		for (String paramName : theParams.keySet()) {
+			// special SearchParam handling:
+			// _lastUpdated
+			if (theParams.getLastUpdated() != null) {
+				canUseHibernate = !illegalForHibernateSearch(Constants.PARAM_LASTUPDATED, resourceActiveSearchParams);
+				if (!canUseHibernate) {
+					return false;
+				}
+			}
+
 			// is this parameter supported?
 			if (illegalForHibernateSearch(paramName, resourceActiveSearchParams)) {
 				canUseHibernate = false;
 			} else {
 				// are the parameter values supported?
 				canUseHibernate =
-						myParams.get(paramName).stream()
+					theParams.get(paramName).stream()
 								.flatMap(Collection::stream)
 								.collect(Collectors.toList())
 								.stream()
@@ -136,6 +146,7 @@ public class ExtendedHSearchSearchBuilder {
 
 				// not yet supported in HSearch
 				myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE
+				&& supportsLastUpdated(myParams)
 				&& // ???
 				myParams.entrySet().stream()
 						.filter(e -> !ourUnsafeSearchParmeters.contains(e.getKey()))
@@ -145,6 +156,19 @@ public class ExtendedHSearchSearchBuilder {
 						.allMatch(this::isParamTypeSupported);
 	}
 
+	private boolean supportsLastUpdated(SearchParameterMap theMap) {
+		if (theMap.getLastUpdated() == null || theMap.getLastUpdated().isEmpty()) {
+			return true;
+		}
+
+		DateRangeParam lastUpdated = theMap.getLastUpdated();
+
+		return lastUpdated.getLowerBound() != null
+			&& isParamTypeSupported(lastUpdated.getLowerBound())
+			&& lastUpdated.getUpperBound() != null
+			&& isParamTypeSupported(lastUpdated.getUpperBound());
+	}
+
 	/**
 	 * Do we support this query param type+modifier?
 	 * <p>

hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java

@@ -294,6 +294,20 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
 		}
 	}
 
+	@Test
+	public void testNoOpUpdateDoesNotModifyLastUpdated() throws InterruptedException {
+		myStorageSettings.setAdvancedHSearchIndexing(true);
+		Patient patient = new Patient();
+		patient.getNameFirstRep().setFamily("graham").addGiven("gary");
+
+		patient = (Patient) myPatientDao.create(patient).getResource();
+		Date originalLastUpdated = patient.getMeta().getLastUpdated();
+
+		patient = (Patient) myPatientDao.update(patient).getResource();
+		Date newLastUpdated = patient.getMeta().getLastUpdated();
+
+		assertThat(originalLastUpdated).isEqualTo(newLastUpdated);
+	}
 
 	@Test
 	public void testFullTextSearchesArePerformanceLogged() {

hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchIndexData.java

@@ -20,6 +20,7 @@
 package ca.uhn.fhir.jpa.model.search;
 
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.entity.StorageSettings;
 import ca.uhn.fhir.model.dstu2.composite.CodingDt;
 import com.google.common.collect.HashMultimap;
@@ -57,12 +58,17 @@ public class ExtendedHSearchIndexData {
 	private String myForcedId;
 	private String myResourceJSON;
 	private IBaseResource myResource;
+	private ResourceTable myEntity;
 
 	public ExtendedHSearchIndexData(
-			FhirContext theFhirContext, StorageSettings theStorageSettings, IBaseResource theResource) {
+		FhirContext theFhirContext,
+		StorageSettings theStorageSettings,
+		IBaseResource theResource,
+		ResourceTable theEntity) {
 		this.myFhirContext = theFhirContext;
 		this.myStorageSettings = theStorageSettings;
 		myResource = theResource;
+		myEntity = theEntity;
 	}
 
 	private <V> BiConsumer<String, V> ifNotContained(BiConsumer<String, V> theIndexWriter) {

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSearchDaoR4Test.java

@@ -305,168 +305,4 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test implements IR4SearchIndex
 		}
 	}
 
-	@Test
-	public void testSearchNarrativeWithLuceneSearch() {
-		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
-		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
-
-		for (int i = 0; i < numberOfPatientsToCreate; i++) {
-			Patient patient = new Patient();
-			patient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
-			expectedActivePatientIds.add(myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless().getIdPart());
-		}
-
-		{
-			Patient patient = new Patient();
-			patient.getText().setDivAsString("<div>AAAB<p>FOO</p> CCC    </div>");
-			myPatientDao.create(patient, mySrd);
-		}
-		{
-			Patient patient = new Patient();
-			patient.getText().setDivAsString("<div>ZZYZXY</div>");
-			myPatientDao.create(patient, mySrd);
-		}
-
-		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
-		map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
-
-		IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
-		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
-
-		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
-	}
-
-	@Test
-	public void searchLuceneAndJPA_withLuceneMatchingButJpaNot_returnsNothing() {
-		// setup
-		int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
-
-		// create resources
-		for (int i = 0; i < numToCreate; i++) {
-			Patient patient = new Patient();
-			patient.setActive(true);
-			patient.addIdentifier()
-				.setSystem("http://fhir.com")
-				.setValue("ZYX");
-			patient.getText().setDivAsString("<div>ABC</div>");
-			myPatientDao.create(patient, mySrd);
-		}
-
-		// test
-		SearchParameterMap map = new SearchParameterMap();
-		map.setLoadSynchronous(true);
-		map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
-		TokenAndListParam tokenAndListParam = new TokenAndListParam();
-		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
-		map.add("active", tokenAndListParam);
-		map.add(Constants.PARAM_TEXT, new StringParam("ABC"));
-		map.add("identifier", new TokenParam(null, "not found"));
-		IBundleProvider provider = myPatientDao.search(map, mySrd);
-
-		// verify
-		assertEquals(0, provider.getAllResources().size());
-	}
-
-	@Test
-	public void searchLuceneAndJPA_withLuceneBroadAndJPASearchNarrow_returnsFoundResults() {
-		// setup
-		int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
-		String identifierToFind = "bcde";
-
-		// create patients
-		for (int i = 0; i < numToCreate; i++) {
-			Patient patient = new Patient();
-			patient.setActive(true);
-			String identifierVal = i == numToCreate - 10 ? identifierToFind:
-			"abcd";
-			patient.addIdentifier()
-				.setSystem("http://fhir.com")
-				.setValue(identifierVal);
-
-			patient.getText().setDivAsString(
-				"<div>FINDME</div>"
-			);
-			myPatientDao.create(patient, mySrd);
-		}
-
-		// test
-		SearchParameterMap map = new SearchParameterMap();
-		map.setLoadSynchronous(true);
-		map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
-		TokenAndListParam tokenAndListParam = new TokenAndListParam();
-		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
-		map.add("active", tokenAndListParam);
-		map.add(Constants.PARAM_TEXT, new StringParam("FINDME"));
-		map.add("identifier", new TokenParam(null, identifierToFind));
-		IBundleProvider provider = myPatientDao.search(map, mySrd);
-
-		// verify
-		List<String> ids = provider.getAllResourceIds();
-		assertEquals(1, ids.size());
-	}
-
-	@Test
-	public void testLuceneNarrativeSearchQueryIntersectingJpaQuery() {
-		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
-		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
-
-		// create active and non-active patients with the same narrative
-		for (int i = 0; i < numberOfPatientsToCreate; i++) {
-			Patient activePatient = new Patient();
-			activePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
-			activePatient.setActive(true);
-			String patientId = myPatientDao.create(activePatient, mySrd).getId().toUnqualifiedVersionless().getIdPart();
-			expectedActivePatientIds.add(patientId);
-
-			Patient nonActivePatient = new Patient();
-			nonActivePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
-			nonActivePatient.setActive(false);
-			myPatientDao.create(nonActivePatient, mySrd);
-		}
-
-		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
-
-		TokenAndListParam tokenAndListParam = new TokenAndListParam();
-		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
-
-		map.add("active", tokenAndListParam);
-		map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
-
-		IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
-		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
-
-		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
-	}
-
-	@Test
-	public void testLuceneContentSearchQueryIntersectingJpaQuery() {
-		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
-		final String patientFamilyName = "Flanders";
-		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
-
-		// create active and non-active patients with the same narrative
-		for (int i = 0; i < numberOfPatientsToCreate; i++) {
-			Patient activePatient = new Patient();
-			activePatient.addName().setFamily(patientFamilyName);
-			activePatient.setActive(true);
-			String patientId = myPatientDao.create(activePatient, mySrd).getId().toUnqualifiedVersionless().getIdPart();
-			expectedActivePatientIds.add(patientId);
-
-			Patient nonActivePatient = new Patient();
-			nonActivePatient.addName().setFamily(patientFamilyName);
-			nonActivePatient.setActive(false);
-			myPatientDao.create(nonActivePatient, mySrd);
-		}
-
-		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
-		TokenAndListParam tokenAndListParam = new TokenAndListParam();
-		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
-		map.add("active", tokenAndListParam);
-		map.add(Constants.PARAM_CONTENT, new StringParam(patientFamilyName));
-
-		IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
-		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
-
-		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
-	}
 }

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4StandardQueriesLuceneTest.java

@@ -2,19 +2,29 @@ package ca.uhn.fhir.jpa.dao.r4;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
+import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient;
+import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
+import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
+import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper;
 import ca.uhn.fhir.jpa.dao.TestDaoSearch;
 import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases;
 import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases;
+import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
 import ca.uhn.fhir.jpa.test.BaseJpaTest;
 import ca.uhn.fhir.jpa.test.config.TestR4Config;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
+import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import ca.uhn.fhir.storage.test.BaseDateSearchDaoTests;
 import ca.uhn.fhir.storage.test.DaoTestDataBuilder;
+import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.HumanName;
 import org.hl7.fhir.r4.model.IdType;
+import org.hl7.fhir.r4.model.Meta;
 import org.hl7.fhir.r4.model.Observation;
+import org.hl7.fhir.r4.model.Patient;
 import org.hl7.fhir.r4.model.Practitioner;
 import org.hl7.fhir.r4.model.PractitionerRole;
 import org.hl7.fhir.r4.model.Reference;
@@ -32,15 +42,15 @@ import org.springframework.transaction.PlatformTransactionManager;
 
 import java.util.List;
 
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
 @ExtendWith(SpringExtension.class)
 @ContextConfiguration(classes = {
 	TestR4Config.class,
 	DaoTestDataBuilder.Config.class,
 	TestDaoSearch.Config.class
 })
-public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
+public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest
+	implements ILuceneSearchR4Test {
+
 	FhirContext myFhirContext = FhirContext.forR4Cached();
 	@Autowired
 	PlatformTransactionManager myTxManager;
@@ -53,6 +63,19 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
 	@Qualifier("myObservationDaoR4")
 	IFhirResourceDao<Observation> myObservationDao;
 	@Autowired
+	@Qualifier("myPatientDaoR4")
+	protected IFhirResourceDaoPatient<Patient> myPatientDao;
+	@Autowired
+	private IFhirSystemDao<Bundle, Meta> mySystemDao;
+	@Autowired
+	private IResourceReindexingSvc myResourceReindexingSvc;
+	@Autowired
+	protected ISearchCoordinatorSvc mySearchCoordinatorSvc;
+	@Autowired
+	protected ISearchParamRegistry mySearchParamRegistry;
+	@Autowired
+	private IBulkDataExportJobSchedulingHelper myBulkDataScheduleHelper;
+	@Autowired
 	IFhirResourceDao<Practitioner> myPractitionerDao;
 	@Autowired
 	IFhirResourceDao<PractitionerRole> myPractitionerRoleDao;
@@ -60,6 +83,7 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
 	// todo mb create an extension to restore via clone or xstream + BeanUtils.copyProperties().
 	@BeforeEach
 	void setUp() {
+		purgeDatabase(myStorageSettings, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);
 		myStorageSettings.setAdvancedHSearchIndexing(true);
 	}
 
@@ -79,6 +103,11 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
 		return myTxManager;
 	}
 
+	@Override
+	public DaoRegistry getDaoRegistry() {
+		return myDaoRegistry;
+	}
+
 	@Nested
 	public class DateSearchTests extends BaseDateSearchDaoTests {
 		@Override

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/ILuceneSearchR4Test.java

@@ -0,0 +1,315 @@
+package ca.uhn.fhir.jpa.dao.r4;
+import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
+import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
+import ca.uhn.fhir.jpa.model.dao.JpaPid;
+import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
+import ca.uhn.fhir.rest.api.server.IBundleProvider;
+import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
+import ca.uhn.fhir.rest.param.DateRangeParam;
+import ca.uhn.fhir.rest.param.StringParam;
+import ca.uhn.fhir.rest.param.TokenAndListParam;
+import ca.uhn.fhir.rest.param.TokenOrListParam;
+import ca.uhn.fhir.rest.param.TokenParam;
+import ca.uhn.fhir.util.DateRangeUtil;
+import org.hl7.fhir.r4.model.Patient;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public interface ILuceneSearchR4Test {
+	DaoRegistry getDaoRegistry();
+
+	@SuppressWarnings("rawtypes")
+	private IFhirResourceDao getResourceDao(String theResourceType) {
+		return getDaoRegistry()
+			.getResourceDao(theResourceType);
+	}
+
+	void runInTransaction(Runnable theRunnable);
+
+	@Test
+	default void testNoOpUpdateDoesNotModifyLastUpdated() throws InterruptedException {
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+		Patient patient = new Patient();
+		patient.getNameFirstRep().setFamily("graham").addGiven("gary");
+
+		patient = (Patient) patientDao.create(patient).getResource();
+		Date originalLastUpdated = patient.getMeta().getLastUpdated();
+
+		patient = (Patient) patientDao.update(patient).getResource();
+		Date newLastUpdated = patient.getMeta().getLastUpdated();
+
+		assertThat(originalLastUpdated).isEqualTo(newLastUpdated);
+	}
+
+	@Test
+	default void luceneSearch_forTagsAndLastUpdated_shouldReturn() {
+		// setup
+		SystemRequestDetails requestDeatils = new SystemRequestDetails();
+		String system = "http://fhir";
+		String code = "cv";
+		Date start = Date.from(Instant.now().minus(1, ChronoUnit.SECONDS).truncatedTo(ChronoUnit.SECONDS));
+		Date end = Date.from(Instant.now().plus(10, ChronoUnit.SECONDS).truncatedTo(ChronoUnit.SECONDS));
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+		// create a patient with some tag
+		Patient patient = new Patient();
+		patient.getMeta()
+			.addTag(system, code, "");
+		patient.addName().addGiven("homer")
+			.setFamily("simpson");
+		patient.addAddress()
+			.setCity("springfield")
+			.addLine("742 evergreen terrace");
+		Long id = patientDao.create(patient, requestDeatils).getId().toUnqualifiedVersionless().getIdPartAsLong();
+
+		// create base search map
+		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
+		TokenOrListParam goldenRecordStatusToken = new TokenOrListParam(system, code);
+		map.add(Constants.PARAM_TAG, goldenRecordStatusToken);
+		DateRangeParam lastUpdated = DateRangeUtil.narrowDateRange(map.getLastUpdated(), start, end);
+		map.setLastUpdated(lastUpdated);
+
+		runInTransaction(() -> {
+			Stream<JpaPid> stream;
+			List<JpaPid> list;
+			Optional<JpaPid> first;
+
+			// tag search only; should return our resource
+			map.setLastUpdated(null);
+			stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
+			list = stream.toList();
+			assertEquals(1, list.size());
+			first = list.stream().findFirst();
+			assertTrue(first.isPresent());
+			assertEquals(id, first.get().getId());
+
+			// last updated search only; should return our resource
+			map.setLastUpdated(lastUpdated);
+			map.remove(Constants.PARAM_TAG);
+			stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
+			list = stream.toList();
+			assertEquals(1, list.size());
+			first = list.stream().findFirst();
+			assertTrue(first.isPresent());
+			assertEquals(id, first.get().getId());
+
+			// both last updated and tags; should return our resource
+			map.add(Constants.PARAM_TAG, goldenRecordStatusToken);
+			stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
+			list = stream.toList();
+			assertEquals(1, list.size());
+			first = list.stream().findFirst();
+			assertTrue(first.isPresent());
+			assertEquals(id, first.get().getId());
+		});
+	}
+
+	@Test
+	default void searchLuceneAndJPA_withLuceneMatchingButJpaNot_returnsNothing() {
+		// setup
+		int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
+		SystemRequestDetails requestDetails = new SystemRequestDetails();
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+		// create resources
+		for (int i = 0; i < numToCreate; i++) {
+			Patient patient = new Patient();
+			patient.setActive(true);
+			patient.addIdentifier()
+				.setSystem("http://fhir.com")
+				.setValue("ZYX");
+			patient.getText().setDivAsString("<div>ABC</div>");
+			patientDao.create(patient, requestDetails);
+		}
+
+		// test
+		SearchParameterMap map = new SearchParameterMap();
+		map.setLoadSynchronous(true);
+		map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
+		TokenAndListParam tokenAndListParam = new TokenAndListParam();
+		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
+		map.add("active", tokenAndListParam);
+		map.add(Constants.PARAM_TEXT, new StringParam("ABC"));
+		map.add("identifier", new TokenParam(null, "not found"));
+		IBundleProvider provider = patientDao.search(map, requestDetails);
+
+		// verify
+		assertEquals(0, provider.getAllResources().size());
+	}
+
+	@Test
+	default void testLuceneNarrativeSearchQueryIntersectingJpaQuery() {
+		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
+		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
+
+		SystemRequestDetails requestDetails = new SystemRequestDetails();
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+
+		// create active and non-active patients with the same narrative
+		for (int i = 0; i < numberOfPatientsToCreate; i++) {
+			Patient activePatient = new Patient();
+			activePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
+			activePatient.setActive(true);
+			String patientId = patientDao.create(activePatient, requestDetails).getId().toUnqualifiedVersionless().getIdPart();
+			expectedActivePatientIds.add(patientId);
+
+			Patient nonActivePatient = new Patient();
+			nonActivePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
+			nonActivePatient.setActive(false);
+			patientDao.create(nonActivePatient, requestDetails);
+		}
+
+		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
+
+		TokenAndListParam tokenAndListParam = new TokenAndListParam();
+		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
+
+		map.add("active", tokenAndListParam);
+		map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
+
+		IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
+		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
+
+		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
+	}
+
+	@Test
+	default void testLuceneContentSearchQueryIntersectingJpaQuery() {
+		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
+		final String patientFamilyName = "Flanders";
+		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
+
+		SystemRequestDetails requestDetails = new SystemRequestDetails();
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+
+		// create active and non-active patients with the same narrative
+		for (int i = 0; i < numberOfPatientsToCreate; i++) {
+			Patient activePatient = new Patient();
+			activePatient.addName().setFamily(patientFamilyName);
+			activePatient.setActive(true);
+			String patientId = patientDao.create(activePatient, requestDetails).getId().toUnqualifiedVersionless().getIdPart();
+			expectedActivePatientIds.add(patientId);
+
+			Patient nonActivePatient = new Patient();
+			nonActivePatient.addName().setFamily(patientFamilyName);
+			nonActivePatient.setActive(false);
+			patientDao.create(nonActivePatient, requestDetails);
+		}
+
+		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
+		TokenAndListParam tokenAndListParam = new TokenAndListParam();
+		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
+		map.add("active", tokenAndListParam);
+		map.add(Constants.PARAM_CONTENT, new StringParam(patientFamilyName));
+
+		IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
+		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
+
+		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
+	}
+
+	@Test
+	default void searchLuceneAndJPA_withLuceneBroadAndJPASearchNarrow_returnsFoundResults() {
+		// setup
+		int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
+		String identifierToFind = "bcde";
+		SystemRequestDetails requestDetails = new SystemRequestDetails();
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+		// create patients
+		for (int i = 0; i < numToCreate; i++) {
+			Patient patient = new Patient();
+			patient.setActive(true);
+			String identifierVal = i == numToCreate - 10 ? identifierToFind:
+				"abcd";
+			patient.addIdentifier()
+				.setSystem("http://fhir.com")
+				.setValue(identifierVal);
+
+			patient.getText().setDivAsString(
+				"<div>FINDME</div>"
+			);
+			patientDao.create(patient, requestDetails);
+		}
+
+		// test
+		SearchParameterMap map = new SearchParameterMap();
+		map.setLoadSynchronous(true);
+		map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
+		TokenAndListParam tokenAndListParam = new TokenAndListParam();
+		tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
+		map.add("active", tokenAndListParam);
+		map.add(Constants.PARAM_TEXT, new StringParam("FINDME"));
+		map.add("identifier", new TokenParam(null, identifierToFind));
+		IBundleProvider provider = patientDao.search(map, requestDetails);
+
+		// verify
+		List<String> ids = provider.getAllResourceIds();
+		assertEquals(1, ids.size());
+	}
+
+	@Test
+	default void testSearchNarrativeWithLuceneSearch() {
+		final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
+		List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
+
+		SystemRequestDetails requestDetails = new SystemRequestDetails();
+
+		@SuppressWarnings("unchecked")
+		IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
+
+
+		for (int i = 0; i < numberOfPatientsToCreate; i++) {
+			Patient patient = new Patient();
+			patient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC    </div>");
+			expectedActivePatientIds.add(patientDao.create(patient, requestDetails).getId().toUnqualifiedVersionless().getIdPart());
+		}
+
+		{
+			Patient patient = new Patient();
+			patient.getText().setDivAsString("<div>AAAB<p>FOO</p> CCC    </div>");
+			patientDao.create(patient, requestDetails);
+		}
+		{
+			Patient patient = new Patient();
+			patient.getText().setDivAsString("<div>ZZYZXY</div>");
+			patientDao.create(patient, requestDetails);
+		}
+
+		SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
+		map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
+
+		IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
+		List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
+
+		assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
+	}
+
+}

hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractorTest.java

@@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
 import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.search.CompositeSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.DateSearchIndexData;
 import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@@ -55,7 +56,7 @@ class ExtendedHSearchIndexExtractorTest implements ITestDataBuilder.WithSupport
 		ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Observation", ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
 		ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
 			myJpaStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
-		ExtendedHSearchIndexData indexData = extractor.extract(new Observation(), extractedParams);
+		ExtendedHSearchIndexData indexData = extractor.extract(new Observation(), new ResourceTable(), extractedParams);
 
 		// validate
 		Set<CompositeSearchIndexData> spIndexData = indexData.getSearchParamComposites().get("component-code-value-concept");
@@ -78,7 +79,7 @@ class ExtendedHSearchIndexExtractorTest implements ITestDataBuilder.WithSupport
 		ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
 		ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
 			myJpaStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
-		ExtendedHSearchIndexData indexData = extractor.extract(new SearchParameter(), searchParams);
+		ExtendedHSearchIndexData indexData = extractor.extract(new SearchParameter(), new ResourceTable(), searchParams);
 
 		// validate
 		Set<DateSearchIndexData> dIndexData = indexData.getDateIndexData().get("Date");

hapi-fhir-storage-test-utilities/src/main/java/ca/uhn/fhir/storage/test/DaoTestDataBuilder.java

@@ -24,6 +24,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
+import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import ca.uhn.fhir.test.utilities.ITestDataBuilder;
 import ca.uhn.fhir.util.BundleBuilder;
 import com.google.common.collect.HashMultimap;
@@ -66,18 +67,42 @@ public class DaoTestDataBuilder implements ITestDataBuilder.WithSupport, ITestDa
 		}
 		//noinspection rawtypes
 		IFhirResourceDao dao = myDaoRegistry.getResourceDao(theResource.getClass());
+
+		// manipulate the transaction details to provide a fake transaction date
+		TransactionDetails details = null;
+		if (theResource.getMeta() != null && theResource.getMeta().getLastUpdated() != null) {
+			details = new TransactionDetails(theResource.getMeta().getLastUpdated());
+		} else {
+			details = new TransactionDetails();
+		}
+
 		//noinspection unchecked
-		IIdType id = dao.create(theResource, mySrd).getId().toUnqualifiedVersionless();
+		IIdType id = dao.create(theResource, null, true, mySrd, details)
+			.getId().toUnqualifiedVersionless();
 		myIds.put(theResource.fhirType(), id);
 		return id;
 	}
 
 	@Override
 	public IIdType doUpdateResource(IBaseResource theResource) {
+		// manipulate the transaction details to provdie a fake transaction date
+		TransactionDetails details = null;
+		if (theResource.getMeta() != null && theResource.getMeta().getLastUpdated() != null) {
+			details = new TransactionDetails(theResource.getMeta().getLastUpdated());
+		} else {
+			details = new TransactionDetails();
+		}
+
 		//noinspection rawtypes
 		IFhirResourceDao dao = myDaoRegistry.getResourceDao(theResource.getClass());
 		//noinspection unchecked
-		IIdType id = dao.update(theResource, mySrd).getId().toUnqualifiedVersionless();
+		IIdType id = dao.update(theResource,
+				null,
+				true,
+				false,
+				mySrd,
+				details)
+			.getId().toUnqualifiedVersionless();
 		myIds.put(theResource.fhirType(), id);
 		return id;
 	}