honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 0d39d4c4fe into 061390d76b

This commit is contained in:
TipzCM 2024-11-27 08:08:09 -05:00 committed by GitHub
parent 061390d76b 0d39d4c4fe
commit d97beda01f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 472 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6404-fulltext-search-with-lastupdate.yaml Normal file
View File

@ -0,0 +1,8 @@
---
type: fix
issue: 6404
title: "Searches using fulltext search that combined `_lastUpdated` query parameter with
any other (supported) fulltext query parameter would find no matches, even
if matches existed.
This has been corrected.
"

4
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/upgrade.md
View File

@ -0,0 +1,4 @@
# Fulltext Search with _lastUpdated Filter
Fulltext searches have been updated to support `_lastUpdated` search parameter. A reindexing of Search Parameters
is required to migrate old data to support the `_lastUpdated` search parameter.

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -1689,7 +1689,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
theEntity.setContentText(parseContentTextIntoWords(theContext, theResource));
if (myStorageSettings.isAdvancedHSearchIndexing()) {
ExtendedHSearchIndexData hSearchIndexData =
myFulltextSearchSvc.extractLuceneIndexData(theResource, theNewParams);
myFulltextSearchSvc.extractLuceneIndexData(theResource, theEntity, theNewParams);
theEntity.setLuceneIndexData(hSearchIndexData);
}
}

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java
View File

@ -135,13 +135,13 @@ public class FulltextSearchSvcImpl implements IFulltextSearchSvc {
@Override
public ExtendedHSearchIndexData extractLuceneIndexData(
IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams) {
String resourceType = myFhirContext.getResourceType(theResource);
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(
resourceType, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
myStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
return extractor.extract(theResource, theNewParams);
return extractor.extract(theResource, theEntity, theNewParams);
}
@Override

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java
View File

@ -89,7 +89,7 @@ public interface IFulltextSearchSvc {
boolean isDisabled();
ExtendedHSearchIndexData extractLuceneIndexData(
IBaseResource theResource, ResourceIndexedSearchParams theNewParams);
IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams);
/**
* Returns true if the parameter map can be handled for hibernate search.

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchClauseBuilder.java
View File

@ -392,7 +392,7 @@ public class ExtendedHSearchClauseBuilder {
/**
* Create date clause from date params. The date lower and upper bounds are taken
* into considertion when generating date query ranges
* into consideration when generating date query ranges
*
* <p>Example 1 ('eq' prefix/empty): <code>http://fhirserver/Observation?date=eq2020</code>
* would generate the following search clause

45
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractor.java
View File

@ -25,6 +25,8 @@ import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.model.entity.ResourceLink;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.entity.TagDefinition;
import ca.uhn.fhir.jpa.model.search.CompositeSearchIndexData;
import ca.uhn.fhir.jpa.model.search.DateSearchIndexData;
import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@ -37,6 +39,7 @@ import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
import ca.uhn.fhir.util.MetaUtil;
import com.google.common.base.Strings;
import jakarta.annotation.Nonnull;
import org.apache.commons.lang3.ObjectUtils;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -74,8 +77,10 @@ public class ExtendedHSearchIndexExtractor {
}
@Nonnull
public ExtendedHSearchIndexData extract(IBaseResource theResource, ResourceIndexedSearchParams theNewParams) {
ExtendedHSearchIndexData retVal = new ExtendedHSearchIndexData(myContext, myJpaStorageSettings, theResource);
public ExtendedHSearchIndexData extract(
IBaseResource theResource, ResourceTable theEntity, ResourceIndexedSearchParams theNewParams) {
ExtendedHSearchIndexData retVal =
new ExtendedHSearchIndexData(myContext, myJpaStorageSettings, theResource, theEntity);
if (myJpaStorageSettings.isStoreResourceInHSearchIndex()) {
retVal.setRawResourceData(myContext.newJsonParser().encodeResourceToString(theResource));
@ -113,11 +118,27 @@ public class ExtendedHSearchIndexExtractor {
.filter(nextParam -> !nextParam.isMissing())
.forEach(nextParam -> retVal.addUriIndexData(nextParam.getParamName(), nextParam.getUri()));
theResource.getMeta().getTag().forEach(tag -> retVal.addTokenIndexData("_tag", tag));
theEntity.getTags().forEach(tag -> {
TagDefinition td = tag.getTag();
theResource.getMeta().getSecurity().forEach(sec -> retVal.addTokenIndexData("_security", sec));
theResource.getMeta().getProfile().forEach(prof -> retVal.addUriIndexData("_profile", prof.getValue()));
IBaseCoding coding = (IBaseCoding) myContext.getVersion().newCodingDt();
coding.setVersion(td.getVersion());
coding.setDisplay(td.getDisplay());
coding.setCode(td.getCode());
coding.setSystem(td.getSystem());
coding.setUserSelected(ObjectUtils.defaultIfNull(td.getUserSelected(), false));
switch (td.getTagType()) {
case TAG:
retVal.addTokenIndexData("_tag", coding);
break;
case PROFILE:
retVal.addUriIndexData("_profile", coding.getCode());
break;
case SECURITY_LABEL:
retVal.addTokenIndexData("_security", coding);
break;
}
});
String source = MetaUtil.getSource(myContext, theResource.getMeta());
if (isNotBlank(source)) {
@ -127,20 +148,14 @@ public class ExtendedHSearchIndexExtractor {
theNewParams.myCompositeParams.forEach(nextParam ->
retVal.addCompositeIndexData(nextParam.getSearchParamName(), buildCompositeIndexData(nextParam)));
if (theResource.getMeta().getLastUpdated() != null) {
int ordinal = ResourceIndexedSearchParamDate.calculateOrdinalValue(
theResource.getMeta().getLastUpdated())
if (theEntity.getUpdated() != null && !theEntity.getUpdated().isEmpty()) {
int ordinal = ResourceIndexedSearchParamDate.calculateOrdinalValue(theEntity.getUpdatedDate())
.intValue();
retVal.addDateIndexData(
"_lastUpdated",
theResource.getMeta().getLastUpdated(),
ordinal,
theResource.getMeta().getLastUpdated(),
ordinal);
"_lastUpdated", theEntity.getUpdatedDate(), ordinal, theEntity.getUpdatedDate(), ordinal);
}
if (!theNewParams.myLinks.isEmpty()) {
// awkwardly, links are indexed by jsonpath, not by search param.
// so we re-build the linkage.
Map<String, List<String>> linkPathToParamName = new HashMap<>();

30
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java
View File

@ -28,6 +28,7 @@ import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
import ca.uhn.fhir.rest.param.CompositeParam;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
@ -89,19 +90,28 @@ public class ExtendedHSearchSearchBuilder {
* be inaccurate and wrong.
*/
public boolean canUseHibernateSearch(
String theResourceType, SearchParameterMap myParams, ISearchParamRegistry theSearchParamRegistry) {
String theResourceType, SearchParameterMap theParams, ISearchParamRegistry theSearchParamRegistry) {
boolean canUseHibernate = false;
ResourceSearchParams resourceActiveSearchParams = theSearchParamRegistry.getActiveSearchParams(
theResourceType, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
for (String paramName : myParams.keySet()) {
for (String paramName : theParams.keySet()) {
// special SearchParam handling:
// _lastUpdated
if (theParams.getLastUpdated() != null) {
canUseHibernate = !illegalForHibernateSearch(Constants.PARAM_LASTUPDATED, resourceActiveSearchParams);
if (!canUseHibernate) {
return false;
}
}
// is this parameter supported?
if (illegalForHibernateSearch(paramName, resourceActiveSearchParams)) {
canUseHibernate = false;
} else {
// are the parameter values supported?
canUseHibernate =
myParams.get(paramName).stream()
theParams.get(paramName).stream()
.flatMap(Collection::stream)
.collect(Collectors.toList())
.stream()
@ -136,6 +146,7 @@ public class ExtendedHSearchSearchBuilder {
// not yet supported in HSearch
myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE
&& supportsLastUpdated(myParams)
&& // ???
myParams.entrySet().stream()
.filter(e -> !ourUnsafeSearchParmeters.contains(e.getKey()))
@ -145,6 +156,19 @@ public class ExtendedHSearchSearchBuilder {
.allMatch(this::isParamTypeSupported);
}
private boolean supportsLastUpdated(SearchParameterMap theMap) {
if (theMap.getLastUpdated() == null || theMap.getLastUpdated().isEmpty()) {
return true;
}
DateRangeParam lastUpdated = theMap.getLastUpdated();
return lastUpdated.getLowerBound() != null
&& isParamTypeSupported(lastUpdated.getLowerBound())
&& lastUpdated.getUpperBound() != null
&& isParamTypeSupported(lastUpdated.getUpperBound());
}
/**
* Do we support this query param type+modifier?
* <p>

14
hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java
View File

@ -294,6 +294,20 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl
}
}
@Test
public void testNoOpUpdateDoesNotModifyLastUpdated() throws InterruptedException {
myStorageSettings.setAdvancedHSearchIndexing(true);
Patient patient = new Patient();
patient.getNameFirstRep().setFamily("graham").addGiven("gary");
patient = (Patient) myPatientDao.create(patient).getResource();
Date originalLastUpdated = patient.getMeta().getLastUpdated();
patient = (Patient) myPatientDao.update(patient).getResource();
Date newLastUpdated = patient.getMeta().getLastUpdated();
assertThat(originalLastUpdated).isEqualTo(newLastUpdated);
}
@Test
public void testFullTextSearchesArePerformanceLogged() {

8
hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchIndexData.java
View File

@ -20,6 +20,7 @@
package ca.uhn.fhir.jpa.model.search;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.entity.StorageSettings;
import ca.uhn.fhir.model.dstu2.composite.CodingDt;
import com.google.common.collect.HashMultimap;
@ -57,12 +58,17 @@ public class ExtendedHSearchIndexData {
private String myForcedId;
private String myResourceJSON;
private IBaseResource myResource;
private ResourceTable myEntity;
public ExtendedHSearchIndexData(
FhirContext theFhirContext, StorageSettings theStorageSettings, IBaseResource theResource) {
FhirContext theFhirContext,
StorageSettings theStorageSettings,
IBaseResource theResource,
ResourceTable theEntity) {
this.myFhirContext = theFhirContext;
this.myStorageSettings = theStorageSettings;
myResource = theResource;
myEntity = theEntity;
}
private <V> BiConsumer<String, V> ifNotContained(BiConsumer<String, V> theIndexWriter) {

164
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirSearchDaoR4Test.java
View File

@ -305,168 +305,4 @@ public class FhirSearchDaoR4Test extends BaseJpaR4Test implements IR4SearchIndex
}
}
@Test
public void testSearchNarrativeWithLuceneSearch() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient patient = new Patient();
patient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
expectedActivePatientIds.add(myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless().getIdPart());
}
{
Patient patient = new Patient();
patient.getText().setDivAsString("<div>AAAB<p>FOO</p> CCC </div>");
myPatientDao.create(patient, mySrd);
}
{
Patient patient = new Patient();
patient.getText().setDivAsString("<div>ZZYZXY</div>");
myPatientDao.create(patient, mySrd);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
@Test
public void searchLuceneAndJPA_withLuceneMatchingButJpaNot_returnsNothing() {
// setup
int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
// create resources
for (int i = 0; i < numToCreate; i++) {
Patient patient = new Patient();
patient.setActive(true);
patient.addIdentifier()
.setSystem("http://fhir.com")
.setValue("ZYX");
patient.getText().setDivAsString("<div>ABC</div>");
myPatientDao.create(patient, mySrd);
}
// test
SearchParameterMap map = new SearchParameterMap();
map.setLoadSynchronous(true);
map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("ABC"));
map.add("identifier", new TokenParam(null, "not found"));
IBundleProvider provider = myPatientDao.search(map, mySrd);
// verify
assertEquals(0, provider.getAllResources().size());
}
@Test
public void searchLuceneAndJPA_withLuceneBroadAndJPASearchNarrow_returnsFoundResults() {
// setup
int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
String identifierToFind = "bcde";
// create patients
for (int i = 0; i < numToCreate; i++) {
Patient patient = new Patient();
patient.setActive(true);
String identifierVal = i == numToCreate - 10 ? identifierToFind:
"abcd";
patient.addIdentifier()
.setSystem("http://fhir.com")
.setValue(identifierVal);
patient.getText().setDivAsString(
"<div>FINDME</div>"
);
myPatientDao.create(patient, mySrd);
}
// test
SearchParameterMap map = new SearchParameterMap();
map.setLoadSynchronous(true);
map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("FINDME"));
map.add("identifier", new TokenParam(null, identifierToFind));
IBundleProvider provider = myPatientDao.search(map, mySrd);
// verify
List<String> ids = provider.getAllResourceIds();
assertEquals(1, ids.size());
}
@Test
public void testLuceneNarrativeSearchQueryIntersectingJpaQuery() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
activePatient.setActive(true);
String patientId = myPatientDao.create(activePatient, mySrd).getId().toUnqualifiedVersionless().getIdPart();
expectedActivePatientIds.add(patientId);
Patient nonActivePatient = new Patient();
nonActivePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
nonActivePatient.setActive(false);
myPatientDao.create(nonActivePatient, mySrd);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
@Test
public void testLuceneContentSearchQueryIntersectingJpaQuery() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
final String patientFamilyName = "Flanders";
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.addName().setFamily(patientFamilyName);
activePatient.setActive(true);
String patientId = myPatientDao.create(activePatient, mySrd).getId().toUnqualifiedVersionless().getIdPart();
expectedActivePatientIds.add(patientId);
Patient nonActivePatient = new Patient();
nonActivePatient.addName().setFamily(patientFamilyName);
nonActivePatient.setActive(false);
myPatientDao.create(nonActivePatient, mySrd);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_CONTENT, new StringParam(patientFamilyName));
IBundleProvider searchResultBundle = myPatientDao.search(map, mySrd);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
}

35
hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4StandardQueriesLuceneTest.java
View File

@ -2,19 +2,29 @@ package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoPatient;
import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao;
import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper;
import ca.uhn.fhir.jpa.dao.TestDaoSearch;
import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases;
import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases;
import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases;
import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
import ca.uhn.fhir.jpa.test.BaseJpaTest;
import ca.uhn.fhir.jpa.test.config.TestR4Config;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.storage.test.BaseDateSearchDaoTests;
import ca.uhn.fhir.storage.test.DaoTestDataBuilder;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.HumanName;
import org.hl7.fhir.r4.model.IdType;
import org.hl7.fhir.r4.model.Meta;
import org.hl7.fhir.r4.model.Observation;
import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.Practitioner;
import org.hl7.fhir.r4.model.PractitionerRole;
import org.hl7.fhir.r4.model.Reference;
@ -32,15 +42,15 @@ import org.springframework.transaction.PlatformTransactionManager;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertThrows;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = {
TestR4Config.class,
DaoTestDataBuilder.Config.class,
TestDaoSearch.Config.class
})
public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest
implements ILuceneSearchR4Test {
FhirContext myFhirContext = FhirContext.forR4Cached();
@Autowired
PlatformTransactionManager myTxManager;
@ -53,6 +63,19 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
@Qualifier("myObservationDaoR4")
IFhirResourceDao<Observation> myObservationDao;
@Autowired
@Qualifier("myPatientDaoR4")
protected IFhirResourceDaoPatient<Patient> myPatientDao;
@Autowired
private IFhirSystemDao<Bundle, Meta> mySystemDao;
@Autowired
private IResourceReindexingSvc myResourceReindexingSvc;
@Autowired
protected ISearchCoordinatorSvc mySearchCoordinatorSvc;
@Autowired
protected ISearchParamRegistry mySearchParamRegistry;
@Autowired
private IBulkDataExportJobSchedulingHelper myBulkDataScheduleHelper;
@Autowired
IFhirResourceDao<Practitioner> myPractitionerDao;
@Autowired
IFhirResourceDao<PractitionerRole> myPractitionerRoleDao;
@ -60,6 +83,7 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
// todo mb create an extension to restore via clone or xstream + BeanUtils.copyProperties().
@BeforeEach
void setUp() {
purgeDatabase(myStorageSettings, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);
myStorageSettings.setAdvancedHSearchIndexing(true);
}
@ -79,6 +103,11 @@ public class FhirResourceDaoR4StandardQueriesLuceneTest extends BaseJpaTest {
return myTxManager;
}
@Override
public DaoRegistry getDaoRegistry() {
return myDaoRegistry;
}
@Nested
public class DateSearchTests extends BaseDateSearchDaoTests {
@Override

315
hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/ILuceneSearchR4Test.java Normal file
View File

@ -0,0 +1,315 @@
package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.model.dao.JpaPid;
import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.util.DateRangeUtil;
import org.hl7.fhir.r4.model.Patient;
import org.junit.jupiter.api.Test;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public interface ILuceneSearchR4Test {
DaoRegistry getDaoRegistry();
@SuppressWarnings("rawtypes")
private IFhirResourceDao getResourceDao(String theResourceType) {
return getDaoRegistry()
.getResourceDao(theResourceType);
}
void runInTransaction(Runnable theRunnable);
@Test
default void testNoOpUpdateDoesNotModifyLastUpdated() throws InterruptedException {
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
Patient patient = new Patient();
patient.getNameFirstRep().setFamily("graham").addGiven("gary");
patient = (Patient) patientDao.create(patient).getResource();
Date originalLastUpdated = patient.getMeta().getLastUpdated();
patient = (Patient) patientDao.update(patient).getResource();
Date newLastUpdated = patient.getMeta().getLastUpdated();
assertThat(originalLastUpdated).isEqualTo(newLastUpdated);
}
@Test
default void luceneSearch_forTagsAndLastUpdated_shouldReturn() {
// setup
SystemRequestDetails requestDeatils = new SystemRequestDetails();
String system = "http://fhir";
String code = "cv";
Date start = Date.from(Instant.now().minus(1, ChronoUnit.SECONDS).truncatedTo(ChronoUnit.SECONDS));
Date end = Date.from(Instant.now().plus(10, ChronoUnit.SECONDS).truncatedTo(ChronoUnit.SECONDS));
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
// create a patient with some tag
Patient patient = new Patient();
patient.getMeta()
.addTag(system, code, "");
patient.addName().addGiven("homer")
.setFamily("simpson");
patient.addAddress()
.setCity("springfield")
.addLine("742 evergreen terrace");
Long id = patientDao.create(patient, requestDeatils).getId().toUnqualifiedVersionless().getIdPartAsLong();
// create base search map
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
TokenOrListParam goldenRecordStatusToken = new TokenOrListParam(system, code);
map.add(Constants.PARAM_TAG, goldenRecordStatusToken);
DateRangeParam lastUpdated = DateRangeUtil.narrowDateRange(map.getLastUpdated(), start, end);
map.setLastUpdated(lastUpdated);
runInTransaction(() -> {
Stream<JpaPid> stream;
List<JpaPid> list;
Optional<JpaPid> first;
// tag search only; should return our resource
map.setLastUpdated(null);
stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
list = stream.toList();
assertEquals(1, list.size());
first = list.stream().findFirst();
assertTrue(first.isPresent());
assertEquals(id, first.get().getId());
// last updated search only; should return our resource
map.setLastUpdated(lastUpdated);
map.remove(Constants.PARAM_TAG);
stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
list = stream.toList();
assertEquals(1, list.size());
first = list.stream().findFirst();
assertTrue(first.isPresent());
assertEquals(id, first.get().getId());
// both last updated and tags; should return our resource
map.add(Constants.PARAM_TAG, goldenRecordStatusToken);
stream = patientDao.searchForIdStream(map, new SystemRequestDetails(), null);
list = stream.toList();
assertEquals(1, list.size());
first = list.stream().findFirst();
assertTrue(first.isPresent());
assertEquals(id, first.get().getId());
});
}
@Test
default void searchLuceneAndJPA_withLuceneMatchingButJpaNot_returnsNothing() {
// setup
int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
SystemRequestDetails requestDetails = new SystemRequestDetails();
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
// create resources
for (int i = 0; i < numToCreate; i++) {
Patient patient = new Patient();
patient.setActive(true);
patient.addIdentifier()
.setSystem("http://fhir.com")
.setValue("ZYX");
patient.getText().setDivAsString("<div>ABC</div>");
patientDao.create(patient, requestDetails);
}
// test
SearchParameterMap map = new SearchParameterMap();
map.setLoadSynchronous(true);
map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("ABC"));
map.add("identifier", new TokenParam(null, "not found"));
IBundleProvider provider = patientDao.search(map, requestDetails);
// verify
assertEquals(0, provider.getAllResources().size());
}
@Test
default void testLuceneNarrativeSearchQueryIntersectingJpaQuery() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
SystemRequestDetails requestDetails = new SystemRequestDetails();
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
activePatient.setActive(true);
String patientId = patientDao.create(activePatient, requestDetails).getId().toUnqualifiedVersionless().getIdPart();
expectedActivePatientIds.add(patientId);
Patient nonActivePatient = new Patient();
nonActivePatient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
nonActivePatient.setActive(false);
patientDao.create(nonActivePatient, requestDetails);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
@Test
default void testLuceneContentSearchQueryIntersectingJpaQuery() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
final String patientFamilyName = "Flanders";
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
SystemRequestDetails requestDetails = new SystemRequestDetails();
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
// create active and non-active patients with the same narrative
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient activePatient = new Patient();
activePatient.addName().setFamily(patientFamilyName);
activePatient.setActive(true);
String patientId = patientDao.create(activePatient, requestDetails).getId().toUnqualifiedVersionless().getIdPart();
expectedActivePatientIds.add(patientId);
Patient nonActivePatient = new Patient();
nonActivePatient.addName().setFamily(patientFamilyName);
nonActivePatient.setActive(false);
patientDao.create(nonActivePatient, requestDetails);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_CONTENT, new StringParam(patientFamilyName));
IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
@Test
default void searchLuceneAndJPA_withLuceneBroadAndJPASearchNarrow_returnsFoundResults() {
// setup
int numToCreate = 2 * SearchBuilder.getMaximumPageSize() + 10;
String identifierToFind = "bcde";
SystemRequestDetails requestDetails = new SystemRequestDetails();
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
// create patients
for (int i = 0; i < numToCreate; i++) {
Patient patient = new Patient();
patient.setActive(true);
String identifierVal = i == numToCreate - 10 ? identifierToFind:
"abcd";
patient.addIdentifier()
.setSystem("http://fhir.com")
.setValue(identifierVal);
patient.getText().setDivAsString(
"<div>FINDME</div>"
);
patientDao.create(patient, requestDetails);
}
// test
SearchParameterMap map = new SearchParameterMap();
map.setLoadSynchronous(true);
map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
TokenAndListParam tokenAndListParam = new TokenAndListParam();
tokenAndListParam.addAnd(new TokenOrListParam().addOr(new TokenParam().setValue("true")));
map.add("active", tokenAndListParam);
map.add(Constants.PARAM_TEXT, new StringParam("FINDME"));
map.add("identifier", new TokenParam(null, identifierToFind));
IBundleProvider provider = patientDao.search(map, requestDetails);
// verify
List<String> ids = provider.getAllResourceIds();
assertEquals(1, ids.size());
}
@Test
default void testSearchNarrativeWithLuceneSearch() {
final int numberOfPatientsToCreate = SearchBuilder.getMaximumPageSize() + 10;
List<String> expectedActivePatientIds = new ArrayList<>(numberOfPatientsToCreate);
SystemRequestDetails requestDetails = new SystemRequestDetails();
@SuppressWarnings("unchecked")
IFhirResourceDao<Patient> patientDao = getResourceDao("Patient");
for (int i = 0; i < numberOfPatientsToCreate; i++) {
Patient patient = new Patient();
patient.getText().setDivAsString("<div>AAAS<p>FOO</p> CCC </div>");
expectedActivePatientIds.add(patientDao.create(patient, requestDetails).getId().toUnqualifiedVersionless().getIdPart());
}
{
Patient patient = new Patient();
patient.getText().setDivAsString("<div>AAAB<p>FOO</p> CCC </div>");
patientDao.create(patient, requestDetails);
}
{
Patient patient = new Patient();
patient.getText().setDivAsString("<div>ZZYZXY</div>");
patientDao.create(patient, requestDetails);
}
SearchParameterMap map = new SearchParameterMap().setLoadSynchronous(true);
map.add(Constants.PARAM_TEXT, new StringParam("AAAS"));
IBundleProvider searchResultBundle = patientDao.search(map, requestDetails);
List<String> resourceIdsFromSearchResult = searchResultBundle.getAllResourceIds();
assertThat(resourceIdsFromSearchResult).containsExactlyInAnyOrderElementsOf(expectedActivePatientIds);
}
}

5
hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchIndexExtractorTest.java
View File

@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.CompositeSearchIndexData;
import ca.uhn.fhir.jpa.model.search.DateSearchIndexData;
import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
@ -55,7 +56,7 @@ class ExtendedHSearchIndexExtractorTest implements ITestDataBuilder.WithSupport
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Observation", ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
myJpaStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
ExtendedHSearchIndexData indexData = extractor.extract(new Observation(), extractedParams);
ExtendedHSearchIndexData indexData = extractor.extract(new Observation(), new ResourceTable(), extractedParams);
// validate
Set<CompositeSearchIndexData> spIndexData = indexData.getSearchParamComposites().get("component-code-value-concept");
@ -78,7 +79,7 @@ class ExtendedHSearchIndexExtractorTest implements ITestDataBuilder.WithSupport
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH);
ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor(
myJpaStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor);
ExtendedHSearchIndexData indexData = extractor.extract(new SearchParameter(), searchParams);
ExtendedHSearchIndexData indexData = extractor.extract(new SearchParameter(), new ResourceTable(), searchParams);
// validate
Set<DateSearchIndexData> dIndexData = indexData.getDateIndexData().get("Date");

29
hapi-fhir-storage-test-utilities/src/main/java/ca/uhn/fhir/storage/test/DaoTestDataBuilder.java
View File

@ -24,6 +24,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.test.utilities.ITestDataBuilder;
import ca.uhn.fhir.util.BundleBuilder;
import com.google.common.collect.HashMultimap;
@ -66,18 +67,42 @@ public class DaoTestDataBuilder implements ITestDataBuilder.WithSupport, ITestDa
}
//noinspection rawtypes
IFhirResourceDao dao = myDaoRegistry.getResourceDao(theResource.getClass());
// manipulate the transaction details to provide a fake transaction date
TransactionDetails details = null;
if (theResource.getMeta() != null && theResource.getMeta().getLastUpdated() != null) {
details = new TransactionDetails(theResource.getMeta().getLastUpdated());
} else {
details = new TransactionDetails();
}
//noinspection unchecked
IIdType id = dao.create(theResource, mySrd).getId().toUnqualifiedVersionless();
IIdType id = dao.create(theResource, null, true, mySrd, details)
.getId().toUnqualifiedVersionless();
myIds.put(theResource.fhirType(), id);
return id;
}
@Override
public IIdType doUpdateResource(IBaseResource theResource) {
// manipulate the transaction details to provdie a fake transaction date
TransactionDetails details = null;
if (theResource.getMeta() != null && theResource.getMeta().getLastUpdated() != null) {
details = new TransactionDetails(theResource.getMeta().getLastUpdated());
} else {
details = new TransactionDetails();
}
//noinspection rawtypes
IFhirResourceDao dao = myDaoRegistry.getResourceDao(theResource.getClass());
//noinspection unchecked
IIdType id = dao.update(theResource, mySrd).getId().toUnqualifiedVersionless();
IIdType id = dao.update(theResource,
null,
true,
false,
mySrd,
details)
.getId().toUnqualifiedVersionless();
myIds.put(theResource.fhirType(), id);
return id;
}