Initial quick implementation of paging with tests

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java

@@ -428,7 +428,6 @@ public class SearchQueryBuilder {
 					startOfQueryParameterIndex = bindOffsetParameter(bindVariables, offset, limitHandler, startOfQueryParameterIndex, bindLimitParametersFirst);
 					bindCountParameter(bindVariables, maxResultsToFetch, limitHandler, startOfQueryParameterIndex, bindLimitParametersFirst);
 				}
-
 			}
 		}
 

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java

@@ -33,6 +33,9 @@ import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.slf4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Example;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -286,8 +289,9 @@ public class MdmLinkDaoSvc {
 	 * @param theExampleLink The MDM link containing the data we would like to search for.
 	 * @return a list of {@link MdmLink} entities which match the example.
 	 */
-	public List<MdmLink> findMdmLinkByExample(Example<MdmLink> theExampleLink) {
+	public Page<MdmLink> findMdmLinkByExample(Example<MdmLink> theExampleLink, int theOffset, int theCount) {
-		return myMdmLinkDao.findAll(theExampleLink);
+		PageRequest of = PageRequest.of(theOffset / theCount, theCount);
+		return myMdmLinkDao.findAll(theExampleLink, of);
 	}
 
 	/**

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java

@@ -66,18 +66,17 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc {
 	}
 
 	@Override
-	public Stream<MdmLinkJson> queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext) {
+	public Stream<MdmLinkJson> queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount) {
 		IIdType goldenResourceId = MdmControllerUtil.extractGoldenResourceIdDtOrNull(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId);
 		IIdType sourceId = MdmControllerUtil.extractSourceIdDtOrNull(ProviderConstants.MDM_QUERY_LINKS_RESOURCE_ID, theSourceResourceId);
 		MdmMatchResultEnum matchResult = MdmControllerUtil.extractMatchResultOrNull(theMatchResult);
 		MdmLinkSourceEnum linkSource = MdmControllerUtil.extractLinkSourceOrNull(theLinkSource);
-
+		return myMdmLinkQuerySvc.queryLinks(goldenResourceId, sourceId, matchResult, linkSource, theMdmTransactionContext, theOffset, theCount);
-		return myMdmLinkQuerySvc.queryLinks(goldenResourceId, sourceId, matchResult, linkSource, theMdmTransactionContext);
 	}
 
 	@Override
-	public Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext) {
+	public Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount) {
-		return myMdmLinkQuerySvc.getDuplicateGoldenResources(theMdmTransactionContext);
+		return myMdmLinkQuerySvc.getDuplicateGoldenResources(theMdmTransactionContext, theOffset, theCount);
 	}
 
 	@Override

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmLinkQuerySvcImpl.java

@@ -33,6 +33,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Example;
+import org.springframework.data.domain.Page;
 
 import java.util.stream.Stream;
 
@@ -46,17 +47,16 @@ public class MdmLinkQuerySvcImpl implements IMdmLinkQuerySvc {
 	MdmLinkDaoSvc myMdmLinkDaoSvc;
 
 	@Override
-	public Stream<MdmLinkJson> queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext) {
+	public Stream<MdmLinkJson> queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext, int theOffset, int theCount) {
 		Example<MdmLink> exampleLink = exampleLinkFromParameters(theGoldenResourceId, theSourceResourceId, theMatchResult, theLinkSource);
-		return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink).stream()
+		Page<MdmLink> mdmLinkByExample = myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink, theOffset, theCount);
-			.filter(mdmLink -> mdmLink.getMatchResult() != MdmMatchResultEnum.POSSIBLE_DUPLICATE)
+		return mdmLinkByExample.stream().map(this::toJson);
-			.map(this::toJson);
 	}
 
 	@Override
-	public Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmContext) {
+	public Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmContext, int theOffset, int theCount) {
 		Example<MdmLink> exampleLink = exampleLinkFromParameters(null, null, MdmMatchResultEnum.POSSIBLE_DUPLICATE, null);
-		return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink).stream().map(this::toJson);
+		return myMdmLinkDaoSvc.findMdmLinkByExample(exampleLink, theOffset, theCount).stream().map(this::toJson);
 	}
 
 	private MdmLinkJson toJson(MdmLink theLink) {

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderQueryLinkR4Test.java

@@ -1,10 +1,15 @@
 package ca.uhn.fhir.jpa.mdm.provider;
 
 import ca.uhn.fhir.jpa.entity.MdmLink;
+import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
 import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum;
 import ca.uhn.fhir.mdm.api.MdmMatchResultEnum;
 import ca.uhn.fhir.model.primitive.IdDt;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
+import ca.uhn.fhir.util.StopWatch;
+import org.apache.commons.lang3.StringUtils;
+import org.hl7.fhir.dstu3.model.UnsignedIntType;
 import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.r4.model.BooleanType;
@@ -16,12 +21,16 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -31,6 +40,8 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test {
 	private StringType myLinkSource;
 	private StringType myGoldenResource1Id;
 	private StringType myGoldenResource2Id;
+	@Autowired
+	protected CircularQueueCaptureQueriesListener myCaptureQueriesListener;
 
 	@Override
 	@BeforeEach
@@ -55,7 +66,7 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test {
 
 	@Test
 	public void testQueryLinkOneMatch() {
-		Parameters result = (Parameters) myMdmProvider.queryLinks(mySourcePatientId, myPatientId, null, null, myRequestDetails);
+		Parameters result = (Parameters) myMdmProvider.queryLinks(mySourcePatientId, myPatientId, null, null, new UnsignedIntType(0), new UnsignedIntType(10), myRequestDetails);
 		ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(result));
 		List<Parameters.ParametersParameterComponent> list = result.getParameter();
 		assertThat(list, hasSize(1));
@@ -63,6 +74,82 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test {
 		assertMdmLink(7, part, mySourcePatientId.getValue(), myPatientId.getValue(), MdmMatchResultEnum.POSSIBLE_MATCH, "false", "true", null);
 	}
 
+	@Test
+	public void testQueryLinkPages() {
+		for (int i = 0; i < 10; i++) {
+		 	createPatientAndUpdateLinks(buildJanePatient());
+		}
+
+		int offset = 0;
+		int count = 2;
+		StopWatch sw = new StopWatch();
+		while (true)  {
+			Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, new UnsignedIntType(offset), new UnsignedIntType(count), myRequestDetails);
+			List<Parameters.ParametersParameterComponent> parameter = result.getParameter();
+			String sourceResourceIds = parameter.stream().flatMap(p -> p.getPart().stream()).filter(part -> part.getName().equals("sourceResourceId")).map(part -> part.getValue().toString()).collect(Collectors.joining(","));
+			ourLog.warn("Search at offset {} took {}ms",offset, sw.getMillisAndRestart());
+			ourLog.warn("Found source resource IDs: {}", sourceResourceIds);
+			offset += count;
+			assertThat(parameter.size(), is(lessThanOrEqualTo(2)));
+
+			//We have stopped finding patients.
+			if (StringUtils.isEmpty(sourceResourceIds)) {
+				break;
+			}
+		}
+	}
+
+	@Test
+	public void testQueryWithIllegalPagingValuesFails() {
+		//Given
+		int count = 0;
+		int offset = 0;
+		try {
+		 //When
+		 myMdmProvider.queryLinks(
+				null, null,
+				null, myLinkSource,
+				new UnsignedIntType(offset),
+				new UnsignedIntType(count),
+				myRequestDetails);
+		} catch (InvalidRequestException e) {
+			//Then
+			assertThat(e.getMessage(), is(equalTo("_count must be greater than 0.")));
+		}
+
+		//Given
+		count = 1;
+		offset= -1;
+		try {
+			//When
+			myMdmProvider.queryLinks(
+				null, null,
+				null, myLinkSource,
+				new UnsignedIntType(offset),
+				new UnsignedIntType(count),
+				myRequestDetails);
+		} catch (InvalidRequestException e) {
+			//Then
+			assertThat(e.getMessage(), is(equalTo("_offset must be greater than or equal to 0. ")));
+		}
+
+		//Given
+		count = 0;
+		offset= -1;
+		try {
+			//When
+			myMdmProvider.queryLinks(
+				null, null,
+				null, myLinkSource,
+				new UnsignedIntType(offset),
+				new UnsignedIntType(count),
+				myRequestDetails);
+		} catch (InvalidRequestException e) {
+			//Then
+			assertThat(e.getMessage(), is(equalTo("_offset must be greater than or equal to 0. _count must be greater than 0.")));
+		}
+	}
+
 	@Test
 	public void testQueryLinkThreeMatches() {
 		// Add a third patient
@@ -71,7 +158,7 @@ public class MdmProviderQueryLinkR4Test extends BaseLinkR4Test {
 		IAnyResource goldenResource = getGoldenResourceFromTargetResource(patient);
 		IIdType goldenResourceId = goldenResource.getIdElement().toVersionless();
 
-		Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, myRequestDetails);
+		Parameters result = (Parameters) myMdmProvider.queryLinks(null, null, null, myLinkSource, new UnsignedIntType(0), new UnsignedIntType(10), myRequestDetails);
 		ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(result));
 		List<Parameters.ParametersParameterComponent> list = result.getParameter();
 		assertThat(list, hasSize(3));

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java

@@ -28,9 +28,9 @@ import java.util.stream.Stream;
 
 public interface IMdmControllerSvc {
 
-	Stream<MdmLinkJson> queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext);
+	Stream<MdmLinkJson> queryLinks(@Nullable String theGoldenResourceId, @Nullable String theSourceResourceId, @Nullable String theMatchResult, @Nullable String theLinkSource, MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount);
 
-	Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext);
+	Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmTransactionContext, int theOffset, int theCount);
 
 	void notDuplicateGoldenResource(String theGoldenResourceId, String theTargetGoldenResourceId, MdmTransactionContext theMdmTransactionContext);
 

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkQuerySvc.java

@@ -29,6 +29,6 @@ import java.util.stream.Stream;
  * This service supports the MDM operation providers for those services that return multiple MDM links.
  */
 public interface IMdmLinkQuerySvc {
-	Stream<MdmLinkJson> queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext);
+	Stream<MdmLinkJson> queryLinks(IIdType theGoldenResourceId, IIdType theSourceResourceId, MdmMatchResultEnum theMatchResult, MdmLinkSourceEnum theLinkSource, MdmTransactionContext theMdmContext, int theOffset, int theCount);
-	Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmContext);
+	Stream<MdmLinkJson> getDuplicateGoldenResources(MdmTransactionContext theMdmContext, int theOffset, int theCount);
 }

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java

@@ -29,9 +29,11 @@ import ca.uhn.fhir.mdm.api.MatchedTarget;
 import ca.uhn.fhir.mdm.api.MdmConstants;
 import ca.uhn.fhir.mdm.api.MdmLinkJson;
 import ca.uhn.fhir.mdm.model.MdmTransactionContext;
+import ca.uhn.fhir.model.api.annotation.Description;
 import ca.uhn.fhir.rest.annotation.IdParam;
 import ca.uhn.fhir.rest.annotation.Operation;
 import ca.uhn.fhir.rest.annotation.OperationParam;
+import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.provider.ProviderConstants;
@@ -39,6 +41,7 @@ import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.util.BundleBuilder;
 import ca.uhn.fhir.util.ParametersUtil;
 import org.apache.commons.lang3.StringUtils;
+import org.hl7.fhir.dstu3.model.UnsignedIntType;
 import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
@@ -58,6 +61,9 @@ import java.util.List;
 import java.util.UUID;
 import java.util.stream.Stream;
 
+import static ca.uhn.fhir.rest.api.Constants.PARAM_COUNT;
+import static ca.uhn.fhir.rest.api.Constants.PARAM_OFFSET;
+
 public class MdmProviderDstu3Plus extends BaseMdmProvider {
 
 	private final IMdmControllerSvc myMdmControllerSvc;
@@ -189,22 +195,43 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider {
 	public IBaseParameters queryLinks(@OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theGoldenResourceId,
 												 @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_RESOURCE_ID, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theResourceId,
 												 @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_MATCH_RESULT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theMatchResult,
-												 @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_LINK_SOURCE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theLinkSource,
+												 @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_LINK_SOURCE, min = 0, max = 1, typeName = "string")
+														 IPrimitiveType<String> theLinkSource,
+
+												 @Description(formalDefinition="Results from this method are returned across multiple pages. This parameter controls the offset when fetching a page.")
+												 @OperationParam(name = PARAM_OFFSET)
+														 UnsignedIntType theOffset,
+												 @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.")
+												 @OperationParam(name = Constants.PARAM_COUNT)
+														 UnsignedIntType theCount,
+
 												 ServletRequestDetails theRequestDetails) {
+		validatePagingParameters(theOffset, theCount);
 
 		Stream<MdmLinkJson> mdmLinkJson = myMdmControllerSvc.queryLinks(extractStringOrNull(theGoldenResourceId),
 			extractStringOrNull(theResourceId), extractStringOrNull(theMatchResult), extractStringOrNull(theLinkSource),
 			createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.QUERY_LINKS,
-				getResourceType(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId))
+				getResourceType(ProviderConstants.MDM_QUERY_LINKS_GOLDEN_RESOURCE_ID, theGoldenResourceId)), theOffset.getValue(), theCount.getValue()
 		);
 		return parametersFromMdmLinks(mdmLinkJson, true);
 	}
 
+	private void validatePagingParameters(UnsignedIntType theOffset, UnsignedIntType theCount) {
+		String errorMessage = "";
+		if (theOffset!= null && theOffset.getValue() < 0) {
+				errorMessage += PARAM_OFFSET + " must be greater than or equal to 0. ";
+		}
+		if (theCount != null && theCount.getValue() <= 0 ) {
+			errorMessage += PARAM_COUNT + " must be greater than 0.";
+		}
+		if (StringUtils.isNotEmpty(errorMessage)) {
+			throw new InvalidRequestException(errorMessage);
+		}
+	}
+
 	@Operation(name = ProviderConstants.MDM_DUPLICATE_GOLDEN_RESOURCES, idempotent = true)
 	public IBaseParameters getDuplicateGoldenResources(ServletRequestDetails theRequestDetails) {
-		Stream<MdmLinkJson> possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources(
+		Stream<MdmLinkJson> possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources(createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, (String) null),0,10);
-			createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, (String) null)
-		);
 		return parametersFromMdmLinks(possibleDuplicates, false);
 	}