3987 group bulk export does not apply typefilter searches to members of the group (#3988)
* Create failing test * Implemented the fix, fixed existing unit tests * added changelog * added test case for no filter, exclude 1 patient Co-authored-by: Steven Li <steven@smilecdr.com>
This commit is contained in:
StevenXLi
GitHub
parent
21cce44524
commit
a4a7fd9344
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
type: fix
|
||||
issue: 3987
|
||||
jira: SMILE-4921
|
||||
title: "Previously, bulk export for Group type with _typeFilter did not apply the filter if it was for the patients, and returned all members of the group.
|
||||
This has now been fixed, and the filter will apply."
|
||||
|
|
@ -61,6 +61,7 @@ import org.hl7.fhir.instance.model.api.IBaseReference;
|
|||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.instance.model.api.IPrimitiveType;
|
||||
import org.hl7.fhir.r4.model.Patient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -76,6 +77,8 @@ import java.util.Optional;
|
|||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static ca.uhn.fhir.rest.api.Constants.PARAM_HAS;
|
||||
|
||||
public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
||||
private static final Logger ourLog = LoggerFactory.getLogger(JpaBulkExportProcessor.class);
|
||||
|
||||
|
|
@ -150,8 +153,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
|||
pids.add(resultIterator.next());
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (theParams.getExportStyle() == BulkDataExportOptions.ExportStyle.GROUP) {
|
||||
} else if (theParams.getExportStyle() == BulkDataExportOptions.ExportStyle.GROUP) {
|
||||
// Group
|
||||
if (resourceType.equalsIgnoreCase("Patient")) {
|
||||
return getExpandedPatientIterator(theParams);
|
||||
|
|
@ -168,8 +170,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
|||
queryChunker.chunk(new ArrayList<>(expandedMemberResourceIds), QUERY_CHUNK_SIZE, (idChunk) -> {
|
||||
queryResourceTypeWithReferencesToPatients(pids, idChunk, theParams, def);
|
||||
});
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// System
|
||||
List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParams);
|
||||
ISearchBuilder searchBuilder = getSearchBuilderForLocalResourceType(theParams);
|
||||
|
|
@ -256,7 +257,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
|||
* possibly expanded by MDM, and don't have to go and fetch other resource DAOs.
|
||||
*/
|
||||
private Iterator<ResourcePersistentId> getExpandedPatientIterator(ExportPIDIteratorParameters theParameters) {
|
||||
List<String> members = getMembers(theParameters.getGroupId());
|
||||
List<String> members = getMembersFromGroupWithFilter(theParameters);
|
||||
List<IIdType> ids = members.stream().map(member -> new IdDt("Patient/" + member)).collect(Collectors.toList());
|
||||
|
||||
// Are bulk exports partition aware or care about partition at all? This does
|
||||
|
|
@ -278,15 +279,36 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
|||
}
|
||||
|
||||
/**
|
||||
* Given the local myGroupId, read this group, and find all members' patient references.
|
||||
* Given the parameters, find all members' patient references in the group with the typeFilter applied.
|
||||
*
|
||||
* @return A list of strings representing the Patient IDs of the members (e.g. ["P1", "P2", "P3"]
|
||||
*/
|
||||
private List<String> getMembers(String theGroupId) {
|
||||
SystemRequestDetails requestDetails = SystemRequestDetails.newSystemRequestAllPartitions();
|
||||
IBaseResource group = myDaoRegistry.getResourceDao("Group").read(new IdDt(theGroupId), requestDetails);
|
||||
List<IPrimitiveType> evaluate = myContext.newFhirPath().evaluate(group, "member.entity.reference", IPrimitiveType.class);
|
||||
return evaluate.stream().map(IPrimitiveType::getValueAsString).collect(Collectors.toList());
|
||||
private List<String> getMembersFromGroupWithFilter(ExportPIDIteratorParameters theParameters) {
|
||||
RuntimeResourceDefinition def = myContext.getResourceDefinition(theParameters.getResourceType());
|
||||
List<String> pids = new ArrayList<>();
|
||||
|
||||
List<SearchParameterMap> maps = myBulkExportHelperSvc.createSearchParameterMapsForResourceType(def, theParameters);
|
||||
|
||||
for (SearchParameterMap map : maps) {
|
||||
//Ensure users did not monkey with the patient compartment search parameter.
|
||||
validateSearchParametersForPatient(map, theParameters);
|
||||
|
||||
ISearchBuilder searchBuilder = getSearchBuilderForLocalResourceType(theParameters);
|
||||
|
||||
// Now, further filter the query with the group id.
|
||||
HasOrListParam hasOrListParam = new HasOrListParam();
|
||||
hasOrListParam.addOr(new HasParam("Group", "member", "_id", theParameters.getGroupId()));
|
||||
map.add(PARAM_HAS, hasOrListParam);
|
||||
|
||||
IResultIterator resultIterator = searchBuilder.createQuery(map,
|
||||
new SearchRuntimeDetails(null, theParameters.getJobId()),
|
||||
null,
|
||||
RequestPartitionId.allPartitions());
|
||||
while (resultIterator.hasNext()) {
|
||||
pids.add(resultIterator.next().toString());
|
||||
}
|
||||
}
|
||||
return pids;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -435,7 +457,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor {
|
|||
|
||||
//Now manually add the members of the group (its possible even with mdm expansion that some members dont have MDM matches,
|
||||
//so would be otherwise skipped
|
||||
expandedIds.addAll(getMembers(theParams.getGroupId()));
|
||||
expandedIds.addAll(getMembersFromGroupWithFilter(theParams));
|
||||
|
||||
return expandedIds;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.bulk.export.svc;
|
|||
|
||||
import ca.uhn.fhir.context.FhirContext;
|
||||
import ca.uhn.fhir.context.RuntimeResourceDefinition;
|
||||
import ca.uhn.fhir.fhirpath.IFhirPath;
|
||||
import ca.uhn.fhir.interceptor.model.RequestPartitionId;
|
||||
import ca.uhn.fhir.jpa.api.config.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
|
||||
|
|
@ -13,7 +12,6 @@ import ca.uhn.fhir.jpa.bulk.export.model.ExportPIDIteratorParameters;
|
|||
import ca.uhn.fhir.jpa.dao.IResultIterator;
|
||||
import ca.uhn.fhir.jpa.dao.ISearchBuilder;
|
||||
import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
|
||||
import ca.uhn.fhir.jpa.dao.index.IJpaIdHelperService;
|
||||
import ca.uhn.fhir.jpa.dao.mdm.MdmExpansionCacheSvc;
|
||||
import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
|
||||
import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
|
||||
|
|
@ -24,6 +22,9 @@ import ca.uhn.fhir.mdm.model.MdmPidTuple;
|
|||
import ca.uhn.fhir.model.primitive.IdDt;
|
||||
import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions;
|
||||
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
|
||||
import ca.uhn.fhir.rest.param.HasOrListParam;
|
||||
import ca.uhn.fhir.rest.param.HasParam;
|
||||
import ch.qos.logback.classic.spi.ILoggingEvent;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.instance.model.api.IPrimitiveType;
|
||||
import org.hl7.fhir.r4.model.Group;
|
||||
|
|
@ -33,6 +34,7 @@ import org.junit.jupiter.api.Test;
|
|||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.mockito.ArgumentMatcher;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
|
|
@ -50,15 +52,17 @@ import java.util.List;
|
|||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import static ca.uhn.fhir.rest.api.Constants.PARAM_HAS;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
import static org.mockito.AdditionalMatchers.not;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
|
@ -242,9 +246,7 @@ public class JpaBulkExportProcessorTest {
|
|||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(
|
||||
booleans = { true, false }
|
||||
)
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void getResourcePidIterator_groupExportStyleWithPatientResource_returnsIterator(boolean theMdm) {
|
||||
// setup
|
||||
ExportPIDIteratorParameters parameters = createExportParameters(BulkDataExportOptions.ExportStyle.GROUP);
|
||||
|
|
@ -265,23 +267,37 @@ public class JpaBulkExportProcessorTest {
|
|||
MdmPidTuple tuple = createTuple(groupId.getIdAsLong(), groupGoldenPid);
|
||||
|
||||
IFhirResourceDao<Group> groupDao = mock(IFhirResourceDao.class);
|
||||
IFhirPath path = mock(IFhirPath.class);
|
||||
parameters.setExpandMdm(theMdm); // set mdm expansion
|
||||
|
||||
// extra mocks
|
||||
IFhirResourceDao<?> mockDao = mock(IFhirResourceDao.class);
|
||||
ISearchBuilder searchBuilder = mock(ISearchBuilder.class);
|
||||
|
||||
// when
|
||||
// getMembers
|
||||
when(myIdHelperService.getPidsOrThrowException(any(), anyList()))
|
||||
.thenReturn(pids);
|
||||
// from getMembersFromGroupWithFilter
|
||||
when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters)))
|
||||
.thenReturn(Collections.singletonList(new SearchParameterMap()));
|
||||
// from getSearchBuilderForLocalResourceType
|
||||
when(myDaoRegistry.getResourceDao(not(eq("Group"))))
|
||||
.thenReturn(mockDao);
|
||||
when(mySearchBuilderFactory.newSearchBuilder(eq(mockDao), eq(parameters.getResourceType()), any()))
|
||||
.thenReturn(searchBuilder);
|
||||
// ret
|
||||
when(searchBuilder.createQuery(
|
||||
any(SearchParameterMap.class),
|
||||
any(SearchRuntimeDetails.class),
|
||||
any(),
|
||||
eq(RequestPartitionId.allPartitions())))
|
||||
.thenReturn(new ListResultIterator(pids));
|
||||
|
||||
// mdm expansion stuff
|
||||
if (theMdm) {
|
||||
when(myDaoRegistry.getResourceDao(eq("Group")))
|
||||
.thenReturn(groupDao);
|
||||
when(groupDao.read(eq(new IdDt(parameters.getGroupId())), any(SystemRequestDetails.class)))
|
||||
.thenReturn(groupResource);
|
||||
when(myFhirContext.newFhirPath())
|
||||
.thenReturn(path);
|
||||
when(path.evaluate(eq(groupResource), anyString(), any(Class.class)))
|
||||
.thenReturn(patientTypes);
|
||||
when(myIdHelperService.getPidsOrThrowException(any(), anyList()))
|
||||
.thenReturn(pids);
|
||||
// mdm expansion stuff
|
||||
if (theMdm) {
|
||||
when(myIdHelperService.translatePidsToForcedIds(any(Set.class)))
|
||||
.thenAnswer(params -> {
|
||||
Set<ResourcePersistentId> uniqPids = params.getArgument(0);
|
||||
|
|
@ -345,17 +361,33 @@ public class JpaBulkExportProcessorTest {
|
|||
IFhirResourceDao<Group> groupDao = mock(IFhirResourceDao.class);
|
||||
IFhirResourceDao<Observation> observationDao = mock(IFhirResourceDao.class);
|
||||
parameters.setExpandMdm(theMdm); // set mdm expansion
|
||||
IFhirPath path = mock(IFhirPath.class);
|
||||
|
||||
// extra mocks
|
||||
IFhirResourceDao<?> mockDao = mock(IFhirResourceDao.class);
|
||||
ISearchBuilder searchBuilder = mock(ISearchBuilder.class);
|
||||
|
||||
// when
|
||||
// getMembers
|
||||
when(myDaoRegistry.getResourceDao(eq("Group")))
|
||||
.thenReturn(groupDao);
|
||||
when(groupDao.read(any(IIdType.class), any(SystemRequestDetails.class)))
|
||||
.thenReturn(groupResource);
|
||||
when(myIdHelperService.getPidOrNull(any(), eq(groupResource)))
|
||||
.thenReturn(groupId);
|
||||
when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), eq(parameters)))
|
||||
.thenReturn(Collections.singletonList(new SearchParameterMap()));
|
||||
when(myDaoRegistry.getResourceDao(not(eq("Group"))))
|
||||
.thenReturn(mockDao);
|
||||
when(mySearchBuilderFactory.newSearchBuilder(eq(mockDao), not(eq("Group")), any()))
|
||||
.thenReturn(searchBuilder);
|
||||
|
||||
// ret
|
||||
when(searchBuilder.createQuery(
|
||||
any(SearchParameterMap.class),
|
||||
any(SearchRuntimeDetails.class),
|
||||
any(),
|
||||
eq(RequestPartitionId.allPartitions())))
|
||||
.thenReturn(new ListResultIterator(Collections.singletonList(pid)))
|
||||
.thenReturn(resultIterator);
|
||||
|
||||
if (theMdm) {
|
||||
when(myMdmLinkDao.expandPidsFromGroupPidGivenMatchResult(any(ResourcePersistentId.class), eq(MdmMatchResultEnum.MATCH)))
|
||||
|
|
@ -370,21 +402,6 @@ public class JpaBulkExportProcessorTest {
|
|||
return new PersistentIdToForcedIdMap(answer);
|
||||
});
|
||||
}
|
||||
when(myFhirContext.newFhirPath())
|
||||
.thenReturn(path);
|
||||
when(path.evaluate(eq(groupResource), anyString(), any(Class.class)))
|
||||
.thenReturn(patientTypes);
|
||||
when(myBulkExportHelperService.createSearchParameterMapsForResourceType(any(RuntimeResourceDefinition.class), any(ExportPIDIteratorParameters.class)))
|
||||
.thenReturn(Collections.singletonList(new SearchParameterMap()));
|
||||
when(myDaoRegistry.getResourceDao(eq("Observation")))
|
||||
.thenReturn(observationDao);
|
||||
when(mySearchBuilderFactory.newSearchBuilder(any(), eq("Observation"), any()))
|
||||
.thenReturn(searchBuilder);
|
||||
when(searchBuilder.createQuery(any(SearchParameterMap.class),
|
||||
any(SearchRuntimeDetails.class),
|
||||
any(),
|
||||
eq(RequestPartitionId.allPartitions())))
|
||||
.thenReturn(resultIterator);
|
||||
|
||||
// test
|
||||
Iterator<ResourcePersistentId> pidIterator = myProcessor.getResourcePidIterator(parameters);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
package ca.uhn.fhir.jpa.bulk;
|
||||
|
||||
import ca.uhn.fhir.jpa.api.model.BulkExportJobResults;
|
||||
import ca.uhn.fhir.jpa.api.svc.IBatch2JobRunner;
|
||||
import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse;
|
||||
import ca.uhn.fhir.jpa.provider.r4.BaseResourceProviderR4Test;
|
||||
import ca.uhn.fhir.jpa.util.BulkExportUtils;
|
||||
import ca.uhn.fhir.rest.api.Constants;
|
||||
import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions;
|
||||
import ca.uhn.fhir.util.JsonUtil;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.hamcrest.Matchers;
|
||||
import org.hl7.fhir.r4.model.Binary;
|
||||
import org.hl7.fhir.r4.model.Enumerations;
|
||||
import org.hl7.fhir.r4.model.Group;
|
||||
import org.hl7.fhir.r4.model.IdType;
|
||||
import org.hl7.fhir.r4.model.Patient;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
public class BulkDataExportTest extends BaseResourceProviderR4Test {
|
||||
private static final Logger ourLog = LoggerFactory.getLogger(BulkDataExportTest.class);
|
||||
|
||||
@Autowired
|
||||
private IBatch2JobRunner myJobRunner;
|
||||
|
||||
@Test
|
||||
public void testGroupBulkExportWithTypeFilter() {
|
||||
// Create some resources
|
||||
Patient patient = new Patient();
|
||||
patient.setId("PF");
|
||||
patient.setGender(Enumerations.AdministrativeGender.FEMALE);
|
||||
patient.setActive(true);
|
||||
myClient.update().resource(patient).execute();
|
||||
|
||||
patient = new Patient();
|
||||
patient.setId("PM");
|
||||
patient.setGender(Enumerations.AdministrativeGender.MALE);
|
||||
patient.setActive(true);
|
||||
myClient.update().resource(patient).execute();
|
||||
|
||||
Group group = new Group();
|
||||
group.setId("Group/G");
|
||||
group.setActive(true);
|
||||
group.addMember().getEntity().setReference("Patient/PF");
|
||||
group.addMember().getEntity().setReference("Patient/PM");
|
||||
myClient.update().resource(group).execute();
|
||||
|
||||
varifyBulkExportResults("G", Sets.newHashSet("Patient?gender=female"), Collections.singletonList("\"PF\""), Collections.singletonList("\"PM\""));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupBulkExportNotInGroup_DoeNotShowUp() {
|
||||
// Create some resources
|
||||
Patient patient = new Patient();
|
||||
patient.setId("PING1");
|
||||
patient.setGender(Enumerations.AdministrativeGender.FEMALE);
|
||||
patient.setActive(true);
|
||||
myClient.update().resource(patient).execute();
|
||||
|
||||
patient = new Patient();
|
||||
patient.setId("PING2");
|
||||
patient.setGender(Enumerations.AdministrativeGender.MALE);
|
||||
patient.setActive(true);
|
||||
myClient.update().resource(patient).execute();
|
||||
|
||||
patient = new Patient();
|
||||
patient.setId("PNING3");
|
||||
patient.setGender(Enumerations.AdministrativeGender.MALE);
|
||||
patient.setActive(true);
|
||||
myClient.update().resource(patient).execute();
|
||||
|
||||
Group group = new Group();
|
||||
group.setId("Group/G2");
|
||||
group.setActive(true);
|
||||
group.addMember().getEntity().setReference("Patient/PING1");
|
||||
group.addMember().getEntity().setReference("Patient/PING2");
|
||||
myClient.update().resource(group).execute();
|
||||
|
||||
varifyBulkExportResults("G2", new HashSet<>(), List.of("\"PING1\"", "\"PING2\""), Collections.singletonList("\"PNING3\""));
|
||||
}
|
||||
|
||||
|
||||
private void varifyBulkExportResults(String theGroupId, HashSet<String> theFilters, List<String> theContainedList, List<String> theExcludedList) {
|
||||
BulkDataExportOptions options = new BulkDataExportOptions();
|
||||
options.setResourceTypes(Sets.newHashSet("Patient"));
|
||||
options.setGroupId(new IdType("Group", theGroupId));
|
||||
options.setFilters(theFilters);
|
||||
options.setExportStyle(BulkDataExportOptions.ExportStyle.GROUP);
|
||||
options.setOutputFormat(Constants.CT_FHIR_NDJSON);
|
||||
|
||||
Batch2JobStartResponse startResponse = myJobRunner.startNewJob(BulkExportUtils.createBulkExportJobParametersFromExportOptions(options));
|
||||
|
||||
assertNotNull(startResponse);
|
||||
|
||||
// Run a scheduled pass to build the export
|
||||
myBatch2JobHelper.awaitJobCompletion(startResponse.getJobId());
|
||||
|
||||
await().until(() -> myJobRunner.getJobInfo(startResponse.getJobId()).getReport() != null);
|
||||
|
||||
// Iterate over the files
|
||||
String report = myJobRunner.getJobInfo(startResponse.getJobId()).getReport();
|
||||
BulkExportJobResults results = JsonUtil.deserialize(report, BulkExportJobResults.class);
|
||||
for (Map.Entry<String, List<String>> file : results.getResourceTypeToBinaryIds().entrySet()) {
|
||||
List<String> binaryIds = file.getValue();
|
||||
assertEquals(1, binaryIds.size());
|
||||
for (String binaryId : binaryIds) {
|
||||
Binary binary = myBinaryDao.read(new IdType(binaryId));
|
||||
assertEquals(Constants.CT_FHIR_NDJSON, binary.getContentType());
|
||||
String contents = new String(binary.getContent(), Constants.CHARSET_UTF8);
|
||||
ourLog.info("Next contents for type {} :\n{}", binary.getResourceType(), contents);
|
||||
for (String containedString : theContainedList) {
|
||||
assertThat(contents, Matchers.containsString(containedString));
|
||||
|
||||
}
|
||||
for (String excludedString : theExcludedList) {
|
||||
assertThat(contents, not(Matchers.containsString(excludedString)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue