From a770d577cb121422b2c110060c112a8d4ff030df Mon Sep 17 00:00:00 2001 From: Tadgh Date: Tue, 29 Jun 2021 17:46:03 -0400 Subject: [PATCH] Initial quick implementation of paging with tests --- .../builder/sql/SearchQueryBuilder.java | 1 - .../uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java | 8 +- .../jpa/mdm/svc/MdmControllerSvcImpl.java | 9 +- .../fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java | 12 +-- .../provider/MdmProviderQueryLinkR4Test.java | 91 ++++++++++++++++++- .../uhn/fhir/mdm/api/IMdmControllerSvc.java | 4 +- .../ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java | 4 +- .../mdm/provider/MdmProviderDstu3Plus.java | 37 +++++++- 8 files changed, 141 insertions(+), 25 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java index 06058baed6d..c974f8a3bac 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java @@ -428,7 +428,6 @@ public class SearchQueryBuilder { startOfQueryParameterIndex = bindOffsetParameter(bindVariables, offset, limitHandler, startOfQueryParameterIndex, bindLimitParametersFirst); bindCountParameter(bindVariables, maxResultsToFetch, limitHandler, startOfQueryParameterIndex, bindLimitParametersFirst); } - } } diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java index 0573e95ffaf..0399bdcbc1b 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java @@ -33,6 +33,9 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Example; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -286,8 +289,9 @@ public class MdmLinkDaoSvc { * @param theExampleLink The MDM link containing the data we would like to search for. * @return a list of {@link MdmLink} entities which match the example. */ - public List findMdmLinkByExample(Example theExampleLink) { - return myMdmLinkDao.findAll(theExampleLink); + public Page findMdmLinkByExample(Example theExampleLink, int theOffset, int theCount) { + PageRequest of = PageRequest.of(theOffset / theCount, theCount); + return myMdmLinkDao.findAll(theExampleLink, of); } /** diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java index bc02474ac35..97b6015320d 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java @@ -66,18 +66,17 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc { } @Override - public Stream queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext) { + public Stream queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount) { IIdType goldenResourceId = MdmControllerUtil.extractGoldenResourceIdDtOrNull(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId); IIdType sourceId = MdmControllerUtil.extractSourceIdDtOrNull(ProviderConstants.MDM_QUERY_LINKS_RESOURCE_ID, theSourceResourceId); MdmMatchResultEnum matchResult = MdmControllerUtil.extractMatchResultOrNull(theMatchResult); MdmLinkSourceEnum linkSource = MdmControllerUtil.extractLinkSourceOrNull(theLinkSource); - - return myMdmLinkQuerySvc.queryLinks(goldenResourceId, sourceId, matchResult, linkSource, theMdmTransactionContext); + return myMdmLinkQuerySvc.queryLinks(goldenResourceId, sourceId, matchResult, linkSource, theMdmTransactionContext, theOffset, theCount); } @Override - public Stream getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext) { - return myMdmLinkQuerySvc.getDuplicateGoldenResources(theMdmTransactionContext); + public Stream getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount) { + return myMdmLinkQuerySvc.getDuplicateGoldenResources(theMdmTransactionContext, theOffset, theCount); } @Override diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java index d191482a7ed..7fd03fb26fd 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Example; +import org.springframework.data.domain.Page; import java.util.stream.Stream; @@ -46,17 +47,16 @@ public class MdmLinkQuerySvcImpl implements IMdmLinkQuerySvc { MdmLinkDaoSvc myMdmLinkDaoSvc; @Override - public Stream queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext) { + public Stream queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext, int theOffset, int theCount) { Example exampleLink = exampleLinkFromParameters(theGoldenResourceId, theSourceResourceId, theMatchResult, theLinkSource); - return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink).stream() - .filter(mdmLink -> mdmLink.getMatchResult() != MdmMatchResultEnum.POSSIBLE_DUPLICATE) - .map(this::toJson); + Page mdmLinkByExample = myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink, theOffset, theCount); + return mdmLinkByExample.stream().map(this::toJson); } @Override - public Stream getDuplicateGoldenResources(MdmTransactionContext theMdmContext) { + public Stream getDuplicateGoldenResources(MdmTransactionContext theMdmContext, int theOffset, int theCount) { Example exampleLink = exampleLinkFromParameters(null, null, MdmMatchResultEnum.POSSIBLE_DUPLICATE, null); - return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink).stream().map(this::toJson); + return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink, theOffset, theCount).stream().map(this::toJson); } private MdmLinkJson toJson(MdmLink theLink) { diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderQueryLinkR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderQueryLinkR4Test.java index a41dc5e99cb..418d511538a 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderQueryLinkR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderQueryLinkR4Test.java @@ -1,10 +1,15 @@ package ca.uhn.fhir.jpa.mdm.provider; import ca.uhn.fhir.jpa.entity.MdmLink; +import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum; import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.model.primitive.IdDt; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.util.StopWatch; +import org.apache.commons.lang3.StringUtils; +import org.hl7.fhir.dstu3.model.UnsignedIntType; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.BooleanType; @@ -16,12 +21,16 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import java.util.List; +import java.util.stream.Collectors; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -31,6 +40,8 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test { private StringType myLinkSource; private StringType myGoldenResource1Id; private StringType myGoldenResource2Id; + @Autowired + protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; @Override @BeforeEach @@ -55,7 +66,7 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test { @Test public void testQueryLinkOneMatch() { - Parameters result = (Parameters) myMdmProvider.queryLinks(mySourcePatientId, myPatientId, null, null, myRequestDetails); + Parameters result = (Parameters) myMdmProvider.queryLinks(mySourcePatientId, myPatientId, null, null, new UnsignedIntType(0), new UnsignedIntType(10), myRequestDetails); ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(result)); List list = result.getParameter(); assertThat(list, hasSize(1)); @@ -63,6 +74,82 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test { assertMdmLink(7, part, mySourcePatientId.getValue(), myPatientId.getValue(), MdmMatchResultEnum.POSSIBLE_MATCH, "false", "true", null); } + @Test + public void testQueryLinkPages() { + for (int i = 0; i < 10; i++) { + createPatientAndUpdateLinks(buildJanePatient()); + } + + int offset = 0; + int count = 2; + StopWatch sw = new StopWatch(); + while (true) { + Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, new UnsignedIntType(offset), new UnsignedIntType(count), myRequestDetails); + List parameter = result.getParameter(); + String sourceResourceIds = parameter.stream().flatMap(p -> p.getPart().stream()).filter(part -> part.getName().equals("sourceResourceId")).map(part -> part.getValue().toString()).collect(Collectors.joining(",")); + ourLog.warn("Search at offset {} took {}ms",offset, sw.getMillisAndRestart()); + ourLog.warn("Found source resource IDs: {}", sourceResourceIds); + offset += count; + assertThat(parameter.size(), is(lessThanOrEqualTo(2))); + + //We have stopped finding patients. + if (StringUtils.isEmpty(sourceResourceIds)) { + break; + } + } + } + + @Test + public void testQueryWithIllegalPagingValuesFails() { + //Given + int count = 0; + int offset = 0; + try { + //When + myMdmProvider.queryLinks( + null, null, + null, myLinkSource, + new UnsignedIntType(offset), + new UnsignedIntType(count), + myRequestDetails); + } catch (InvalidRequestException e) { + //Then + assertThat(e.getMessage(), is(equalTo("_count must be greater than 0."))); + } + + //Given + count = 1; + offset= -1; + try { + //When + myMdmProvider.queryLinks( + null, null, + null, myLinkSource, + new UnsignedIntType(offset), + new UnsignedIntType(count), + myRequestDetails); + } catch (InvalidRequestException e) { + //Then + assertThat(e.getMessage(), is(equalTo("_offset must be greater than or equal to 0. "))); + } + + //Given + count = 0; + offset= -1; + try { + //When + myMdmProvider.queryLinks( + null, null, + null, myLinkSource, + new UnsignedIntType(offset), + new UnsignedIntType(count), + myRequestDetails); + } catch (InvalidRequestException e) { + //Then + assertThat(e.getMessage(), is(equalTo("_offset must be greater than or equal to 0. _count must be greater than 0."))); + } + } + @Test public void testQueryLinkThreeMatches() { // Add a third patient @@ -71,7 +158,7 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test { IAnyResource goldenResource = getGoldenResourceFromTargetResource(patient); IIdType goldenResourceId = goldenResource.getIdElement().toVersionless(); - Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, myRequestDetails); + Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, new UnsignedIntType(0), new UnsignedIntType(10), myRequestDetails); ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(result)); List list = result.getParameter(); assertThat(list, hasSize(3)); diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java index fb0e9ebe981..5ff3c62570f 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java @@ -28,9 +28,9 @@ import java.util.stream.Stream; public interface IMdmControllerSvc { - Stream queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext); + Stream queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount); - Stream getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext); + Stream getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount); void notDuplicateGoldenResource(String theGoldenResourceId, String theTargetGoldenResourceId, MdmTransactionContext theMdmTransactionContext); diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java index 8d5fe42e254..28aefbaf550 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java @@ -29,6 +29,6 @@ import java.util.stream.Stream; * This service supports the MDM operation providers for those services that return multiple MDM links. */ public interface IMdmLinkQuerySvc { - Stream queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext); - Stream getDuplicateGoldenResources(MdmTransactionContext theMdmContext); + Stream queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext, int theOffset, int theCount); + Stream getDuplicateGoldenResources(MdmTransactionContext theMdmContext, int theOffset, int theCount); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java index 45f9d2031a4..df6fe73ffc5 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java @@ -29,9 +29,11 @@ import ca.uhn.fhir.mdm.api.MatchedTarget; import ca.uhn.fhir.mdm.api.MdmConstants; import ca.uhn.fhir.mdm.api.MdmLinkJson; import ca.uhn.fhir.mdm.model.MdmTransactionContext; +import ca.uhn.fhir.model.api.annotation.Description; import ca.uhn.fhir.rest.annotation.IdParam; import ca.uhn.fhir.rest.annotation.Operation; import ca.uhn.fhir.rest.annotation.OperationParam; +import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.provider.ProviderConstants; @@ -39,6 +41,7 @@ import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.util.BundleBuilder; import ca.uhn.fhir.util.ParametersUtil; import org.apache.commons.lang3.StringUtils; +import org.hl7.fhir.dstu3.model.UnsignedIntType; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBaseBackboneElement; @@ -58,6 +61,9 @@ import java.util.List; import java.util.UUID; import java.util.stream.Stream; +import static ca.uhn.fhir.rest.api.Constants.PARAM_COUNT; +import static ca.uhn.fhir.rest.api.Constants.PARAM_OFFSET; + public class MdmProviderDstu3Plus extends BaseMdmProvider { private final IMdmControllerSvc myMdmControllerSvc; @@ -189,22 +195,43 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { public IBaseParameters queryLinks(@OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, min = 0, max = 1, typeName = "string") IPrimitiveType theGoldenResourceId, @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_RESOURCE_ID, min = 0, max = 1, typeName = "string") IPrimitiveType theResourceId, @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_MATCH_RESULT, min = 0, max = 1, typeName = "string") IPrimitiveType theMatchResult, - @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_LINK_SOURCE, min = 0, max = 1, typeName = "string") IPrimitiveType theLinkSource, + @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_LINK_SOURCE, min = 0, max = 1, typeName = "string") + IPrimitiveType theLinkSource, + + @Description(formalDefinition="Results from this method are returned across multiple pages. This parameter controls the offset when fetching a page.") + @OperationParam(name = PARAM_OFFSET) + UnsignedIntType theOffset, + @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.") + @OperationParam(name = Constants.PARAM_COUNT) + UnsignedIntType theCount, + ServletRequestDetails theRequestDetails) { + validatePagingParameters(theOffset, theCount); Stream mdmLinkJson = myMdmControllerSvc.queryLinks(extractStringOrNull(theGoldenResourceId), extractStringOrNull(theResourceId), extractStringOrNull(theMatchResult), extractStringOrNull(theLinkSource), createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.QUERY_LINKS, - getResourceType(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId)) + getResourceType(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId)), theOffset.getValue(), theCount.getValue() ); return parametersFromMdmLinks(mdmLinkJson, true); } + private void validatePagingParameters(UnsignedIntType theOffset, UnsignedIntType theCount) { + String errorMessage = ""; + if (theOffset!= null && theOffset.getValue() < 0) { + errorMessage += PARAM_OFFSET + " must be greater than or equal to 0. "; + } + if (theCount != null && theCount.getValue() <= 0 ) { + errorMessage += PARAM_COUNT + " must be greater than 0."; + } + if (StringUtils.isNotEmpty(errorMessage)) { + throw new InvalidRequestException(errorMessage); + } + } + @Operation(name = ProviderConstants.MDM_DUPLICATE_GOLDEN_RESOURCES, idempotent = true) public IBaseParameters getDuplicateGoldenResources(ServletRequestDetails theRequestDetails) { - Stream possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources( - createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, (String) null) - ); + Stream possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources(createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, (String) null),0,10); return parametersFromMdmLinks(possibleDuplicates, false); }