5237 fixing empty last page if even results (#5506)

paging will not return empty pages if no results left

hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/BulkImportCommandIT.java

@@ -8,6 +8,8 @@ import ca.uhn.fhir.batch2.model.JobInstanceStartRequest;
 import ca.uhn.fhir.batch2.model.StatusEnum;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse;
+import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
+import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.interceptor.LoggingInterceptor;
 import ca.uhn.fhir.system.HapiSystemProperties;
@@ -66,6 +68,8 @@ public class BulkImportCommandIT {
 	private IJobCoordinator myJobCoordinator;
 	private final BulkDataImportProvider myProvider = new BulkDataImportProvider();
 	private final FhirContext myCtx = FhirContext.forR4Cached();
+	@Mock
+	private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
 	@RegisterExtension
 	public RestfulServerExtension myRestfulServerExtension = new RestfulServerExtension(myCtx, myProvider)
 		.registerInterceptor(new LoggingInterceptor());
@@ -77,6 +81,7 @@ public class BulkImportCommandIT {
 	public void beforeEach() throws IOException {
 		myProvider.setFhirContext(myCtx);
 		myProvider.setJobCoordinator(myJobCoordinator);
+		myProvider.setRequestPartitionHelperService(myRequestPartitionHelperSvc);
 		myTempDir = Files.createTempDirectory("hapifhir");
 		ourLog.info("Created temp directory: {}", myTempDir);
 	}
@@ -123,7 +128,7 @@ public class BulkImportCommandIT {
 		await().until(() -> myRestfulServerExtension.getRequestContentTypes().size(), equalTo(2));
 		ourLog.info("Initiation requests complete");
 
-		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(myStartCaptor.capture());
+		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(any(RequestDetails.class), myStartCaptor.capture());
 
 		JobInstanceStartRequest startRequest = myStartCaptor.getValue();
 		BulkImportJobParameters jobParameters = startRequest.getParameters(BulkImportJobParameters.class);
@@ -165,7 +170,7 @@ public class BulkImportCommandIT {
 		await().until(() -> myRestfulServerExtension.getRequestContentTypes().size(), equalTo(2));
 		ourLog.info("Initiation requests complete");
 
-		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(myStartCaptor.capture());
+		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(any(RequestDetails.class), myStartCaptor.capture());
 
 		JobInstanceStartRequest startRequest = myStartCaptor.getValue();
 		BulkImportJobParameters jobParameters = startRequest.getParameters(BulkImportJobParameters.class);
@@ -206,7 +211,7 @@ public class BulkImportCommandIT {
 		await().until(() -> myRestfulServerExtension.getRequestContentTypes().size(), equalTo(2));
 		ourLog.info("Initiation requests complete");
 
-		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(myStartCaptor.capture());
+		verify(myJobCoordinator, timeout(10000).times(1)).startInstance(any(RequestDetails.class), myStartCaptor.capture());
 
 		try{
 			JobInstanceStartRequest startRequest = myStartCaptor.getValue();

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_0_0/5192-include-in-search-paging-fix.yaml

@@ -0,0 +1,7 @@
+---
+type: fix
+issue: 5192
+title: "Fixed a bug where search Bundles with `include` entries from an _include query parameter might
+      trigger a 'next' link to blank pages when
+      no more results `match` results are available.
+"

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java

@@ -125,7 +125,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider {
 	 * of this class, since it's a prototype
 	 */
 	private Search mySearchEntity;
-	private String myUuid;
+	private final String myUuid;
 	private SearchCacheStatusEnum myCacheStatus;
 	private RequestPartitionId myRequestPartitionId;
 
@@ -259,13 +259,21 @@ public class PersistedJpaBundleProvider implements IBundleProvider {
 		final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(dao, resourceName, resourceType);
 
 		RequestPartitionId requestPartitionId = getRequestPartitionId();
-		final List<JpaPid> pidsSubList =
+		// we request 1 more resource than we need
-				mySearchCoordinatorSvc.getResources(myUuid, theFromIndex, theToIndex, myRequest, requestPartitionId);
+		// this is so we can be sure of when we hit the last page
+		// (when doing offset searches)
+		final List<JpaPid> pidsSubList = mySearchCoordinatorSvc.getResources(
+				myUuid, theFromIndex, theToIndex + 1, myRequest, requestPartitionId);
+		// max list size should be either the entire list, or from - to length
+		int maxSize = Math.min(theToIndex - theFromIndex, pidsSubList.size());
+		theResponsePageBuilder.setTotalRequestedResourcesFetched(pidsSubList.size());
+
+		List<JpaPid> firstBatchOfPids = pidsSubList.subList(0, maxSize);
 		List<IBaseResource> resources = myTxService
 				.withRequest(myRequest)
 				.withRequestPartitionId(requestPartitionId)
 				.execute(() -> {
-					return toResourceList(sb, pidsSubList, theResponsePageBuilder);
+					return toResourceList(sb, firstBatchOfPids, theResponsePageBuilder);
 				});
 
 		return resources;
@@ -541,8 +549,8 @@ public class PersistedJpaBundleProvider implements IBundleProvider {
 		// this can (potentially) change the results being returned.
 		int precount = resources.size();
 		resources = ServerInterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
-		// we only care about omitted results from *this* page
+		// we only care about omitted results from this page
-		theResponsePageBuilder.setToOmittedResourceCount(precount - resources.size());
+		theResponsePageBuilder.setOmittedResourceCount(precount - resources.size());
 		theResponsePageBuilder.setResources(resources);
 		theResponsePageBuilder.setIncludedResourceCount(includedPidList.size());
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java

@@ -73,16 +73,23 @@ public class PersistedJpaSearchFirstPageBundleProvider extends PersistedJpaBundl
 
 		mySearchTask.awaitInitialSync();
 
+		// request 1 more than we need to, in order to know if there are extra values
 		ourLog.trace("Fetching search resource PIDs from task: {}", mySearchTask.getClass());
-		final List<JpaPid> pids = mySearchTask.getResourcePids(theFromIndex, theToIndex);
+		final List<JpaPid> pids = mySearchTask.getResourcePids(theFromIndex, theToIndex + 1);
 		ourLog.trace("Done fetching search resource PIDs");
 
+		int countOfPids = pids.size();
+		;
+		int maxSize = Math.min(theToIndex - theFromIndex, countOfPids);
+		thePageBuilder.setTotalRequestedResourcesFetched(countOfPids);
+
 		RequestPartitionId requestPartitionId = getRequestPartitionId();
 
+		List<JpaPid> firstBatch = pids.subList(0, maxSize);
 		List<IBaseResource> retVal = myTxService
 				.withRequest(myRequest)
 				.withRequestPartitionId(requestPartitionId)
-				.execute(() -> toResourceList(mySearchBuilder, pids, thePageBuilder));
+				.execute(() -> toResourceList(mySearchBuilder, firstBatch, thePageBuilder));
 
 		long totalCountWanted = theToIndex - theFromIndex;
 		long totalCountMatch = (int) retVal.stream().filter(t -> !isInclude(t)).count();
@@ -103,12 +110,15 @@ public class PersistedJpaSearchFirstPageBundleProvider extends PersistedJpaBundl
 
 				long remainingWanted = totalCountWanted - totalCountMatch;
 				long fromIndex = theToIndex - remainingWanted;
-				List<IBaseResource> remaining = super.getResources((int) fromIndex, theToIndex, thePageBuilder);
+				ResponsePage.ResponsePageBuilder pageBuilder = new ResponsePage.ResponsePageBuilder();
+				pageBuilder.setBundleProvider(this);
+				List<IBaseResource> remaining = super.getResources((int) fromIndex, theToIndex, pageBuilder);
 				remaining.forEach(t -> {
 					if (!existingIds.contains(t.getIdElement().getValue())) {
 						retVal.add(t);
 					}
 				});
+				thePageBuilder.combineWith(pageBuilder);
 			}
 		}
 		ourLog.trace("Loaded resources to return");

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java

@@ -115,7 +115,7 @@ public class SynchronousSearchSvcImpl implements ISynchronousSearchSvc {
 				.execute(() -> {
 
 					// Load the results synchronously
-					final List<JpaPid> pids = new ArrayList<>();
+					List<JpaPid> pids = new ArrayList<>();
 
 					Long count = 0L;
 					if (wantCount) {
@@ -145,8 +145,17 @@ public class SynchronousSearchSvcImpl implements ISynchronousSearchSvc {
 						return bundleProvider;
 					}
 
+					// if we have a count, we'll want to request
+					// additional resources
+					SearchParameterMap clonedParams = theParams.clone();
+					Integer requestedCount = clonedParams.getCount();
+					boolean hasACount = requestedCount != null;
+					if (hasACount) {
+						clonedParams.setCount(requestedCount.intValue() + 1);
+					}
+
 					try (IResultIterator<JpaPid> resultIter = theSb.createQuery(
-							theParams, searchRuntimeDetails, theRequestDetails, theRequestPartitionId)) {
+							clonedParams, searchRuntimeDetails, theRequestDetails, theRequestPartitionId)) {
 						while (resultIter.hasNext()) {
 							pids.add(resultIter.next());
 							if (theLoadSynchronousUpTo != null && pids.size() >= theLoadSynchronousUpTo) {
@@ -162,6 +171,15 @@ public class SynchronousSearchSvcImpl implements ISynchronousSearchSvc {
 						throw new InternalErrorException(Msg.code(1164) + e);
 					}
 
+					// truncate the list we retrieved - if needed
+					int receivedResourceCount = -1;
+					if (hasACount) {
+						// we want the accurate received resource count
+						receivedResourceCount = pids.size();
+						int resourcesToReturn = Math.min(theParams.getCount(), pids.size());
+						pids = pids.subList(0, resourcesToReturn);
+					}
+
 					JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(pids, () -> theSb);
 					HookParams params = new HookParams()
 							.add(IPreResourceAccessDetails.class, accessDetails)
@@ -228,6 +246,9 @@ public class SynchronousSearchSvcImpl implements ISynchronousSearchSvc {
 							resources, theRequestDetails, myInterceptorBroadcaster);
 
 					SimpleBundleProvider bundleProvider = new SimpleBundleProvider(resources);
+					if (hasACount) {
+						bundleProvider.setTotalResourcesRequestedReturned(receivedResourceCount);
+					}
 					if (theParams.isOffsetQuery()) {
 						bundleProvider.setCurrentPageOffset(theParams.getOffset());
 						bundleProvider.setCurrentPageSize(theParams.getCount());

hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImplTest.java

@@ -42,14 +42,17 @@ public class SynchronousSearchSvcImplTest extends BaseSearchSvc {
 
 	@Test
 	public void testSynchronousSearch() {
-		when(mySearchBuilderFactory.newSearchBuilder(any(), any(), any())).thenReturn(mySearchBuilder);
+		when(mySearchBuilderFactory.newSearchBuilder(any(), any(), any()))
+			.thenReturn(mySearchBuilder);
 
 		SearchParameterMap params = new SearchParameterMap();
 
 		List<JpaPid> pids = createPidSequence(800);
-		when(mySearchBuilder.createQuery(same(params), any(), any(), nullable(RequestPartitionId.class))).thenReturn(new BaseSearchSvc.ResultIterator(pids.iterator()));
+		when(mySearchBuilder.createQuery(any(SearchParameterMap.class), any(), any(), nullable(RequestPartitionId.class)))
+			.thenReturn(new BaseSearchSvc.ResultIterator(pids.iterator()));
 
-		doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any());
+		doAnswer(loadPids()).when(mySearchBuilder)
+			.loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any());
 
 		IBundleProvider result = mySynchronousSearchSvc.executeQuery( "Patient", params, RequestPartitionId.allPartitions());
 		assertNull(result.getUuid());
@@ -71,8 +74,8 @@ public class SynchronousSearchSvcImplTest extends BaseSearchSvc {
 		params.setSearchTotalMode(SearchTotalModeEnum.ACCURATE);
 
 		List<JpaPid> pids = createPidSequence(30);
-		when(mySearchBuilder.createCountQuery(same(params), any(String.class),nullable(RequestDetails.class), nullable(RequestPartitionId.class))).thenReturn(20L);
+		when(mySearchBuilder.createCountQuery(any(SearchParameterMap.class), any(String.class),nullable(RequestDetails.class), nullable(RequestPartitionId.class))).thenReturn(20L);
-		when(mySearchBuilder.createQuery(same(params), any(), nullable(RequestDetails.class), nullable(RequestPartitionId.class))).thenReturn(new BaseSearchSvc.ResultIterator(pids.subList(10, 20).iterator()));
+		when(mySearchBuilder.createQuery(any(SearchParameterMap.class), any(), nullable(RequestDetails.class), nullable(RequestPartitionId.class))).thenReturn(new BaseSearchSvc.ResultIterator(pids.subList(10, 20).iterator()));
 
 		doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any());
 
@@ -92,7 +95,8 @@ public class SynchronousSearchSvcImplTest extends BaseSearchSvc {
 		params.setLoadSynchronousUpTo(100);
 
 		List<JpaPid> pids = createPidSequence(800);
-		when(mySearchBuilder.createQuery(same(params), any(), nullable(RequestDetails.class), nullable(RequestPartitionId.class))).thenReturn(new BaseSearchSvc.ResultIterator(pids.iterator()));
+		when(mySearchBuilder.createQuery(any(SearchParameterMap.class), any(), nullable(RequestDetails.class), nullable(RequestPartitionId.class)))
+			.thenReturn(new BaseSearchSvc.ResultIterator(pids.iterator()));
 
 		pids = createPidSequence(110);
 		List<JpaPid> finalPids = pids;

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java

@@ -29,11 +29,11 @@ import org.slf4j.LoggerFactory;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import javax.servlet.ServletException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.apache.commons.lang3.StringUtils.leftPad;
@@ -54,7 +54,7 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 	private List<String> myPatientIds;
 	private List<String> myObservationIdsOddOnly;
 	private List<String> myObservationIdsEvenOnly;
-	private List<String> myObservationIdsWithVersions;
+	private List<String> myObservationIdsWithoutVersions;
 	private List<String> myPatientIdsEvenOnly;
 
 	@AfterEach
@@ -64,13 +64,16 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 	}
 
 	@BeforeEach
-	public void before() throws ServletException {
+	@Override
+	public void beforeInitMocks() throws Exception {
+		super.beforeInitMocks();
 		RestfulServer restfulServer = new RestfulServer();
 		restfulServer.setPagingProvider(myPagingProvider);
 
 		when(mySrd.getServer()).thenReturn(restfulServer);
 
 		myStorageSettings.setSearchPreFetchThresholds(Arrays.asList(20, 50, 190));
+		restfulServer.setDefaultPageSize(null);
 	}
 
 	@Test
@@ -147,6 +150,7 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 
 	@Test
 	public void testSearchAndBlockSome_LoadSynchronous() {
+		// setup
 		create50Observations();
 
 		AtomicInteger hitCount = new AtomicInteger(0);
@@ -281,6 +285,7 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 
 	@Test
 	public void testSearchAndBlockSomeOnIncludes_LoadSynchronous() {
+		// setup
 		create50Observations();
 
 		AtomicInteger hitCount = new AtomicInteger(0);
@@ -328,9 +333,8 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 		 * returned results because we create it then update it in create50Observations()
 		 */
 		assertEquals(1, hitCount.get());
-		assertEquals(myObservationIdsWithVersions.subList(90, myObservationIdsWithVersions.size()), sort(interceptedResourceIds));
+		assertEquals(sort(myObservationIdsWithoutVersions.subList(90, myObservationIdsWithoutVersions.size())), sort(interceptedResourceIds));
 		returnedIdValues.forEach(t -> assertTrue(new IdType(t).getIdPartAsLong() % 2 == 0));
-
 	}
 
 	@Test
@@ -363,7 +367,7 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 	private void create50Observations() {
 		myPatientIds = new ArrayList<>();
 		myObservationIds = new ArrayList<>();
-		myObservationIdsWithVersions = new ArrayList<>();
+		myObservationIdsWithoutVersions = new ArrayList<>();
 
 		Patient p = new Patient();
 		p.setActive(true);
@@ -383,9 +387,9 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 			final Observation obs1 = new Observation();
 			obs1.setStatus(Observation.ObservationStatus.FINAL);
 			obs1.addIdentifier().setSystem("urn:system").setValue("I" + leftPad("" + i, 5, '0'));
-			IIdType obs1id = myObservationDao.create(obs1).getId().toUnqualifiedVersionless();
+			IIdType obs1id = myObservationDao.create(obs1).getId();
 			myObservationIds.add(obs1id.toUnqualifiedVersionless().getValue());
-			myObservationIdsWithVersions.add(obs1id.toUnqualifiedVersionless().getValue());
+			myObservationIdsWithoutVersions.add(obs1id.toUnqualifiedVersionless().getValue());
 
 			obs1.setId(obs1id);
 			if (obs1id.getIdPartAsLong() % 2 == 0) {
@@ -394,7 +398,7 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 				obs1.getSubject().setReference(oddPid);
 			}
 			myObservationDao.update(obs1);
-			myObservationIdsWithVersions.add(obs1id.toUnqualifiedVersionless().getValue());
+			myObservationIdsWithoutVersions.add(obs1id.toUnqualifiedVersionless().getValue());
 
 		}
 
@@ -483,14 +487,24 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest {
 		}
 	}
 
-	private static List<String> sort(List<String>... theLists) {
+	private List<String> sort(List<String>... theLists) {
+		return sort(id -> {
+			String idParsed = id.substring(id.indexOf("/") + 1);
+			if (idParsed.contains("/_history")) {
+				idParsed = idParsed.substring(0, idParsed.indexOf("/"));
+			}
+			return Long.parseLong(idParsed);
+		}, theLists);
+	}
+
+	private List<String> sort(Function<String, Long> theParser, List<String>... theLists) {
 		ArrayList<String> retVal = new ArrayList<>();
 		for (List<String> next : theLists) {
 			retVal.addAll(next);
 		}
 		retVal.sort((o0, o1) -> {
-			long i0 = Long.parseLong(o0.substring(o0.indexOf('/') + 1));
+			long i0 = theParser.apply(o0);
-			long i1 = Long.parseLong(o1.substring(o1.indexOf('/') + 1));
+			long i1 = theParser.apply(o1);
 			return (int) (i0 - i1);
 		});
 		return retVal;

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java

@@ -114,6 +114,7 @@ import static org.hamcrest.Matchers.not;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.eq;
@@ -1229,6 +1230,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 			nextChunk.forEach(t -> foundIds.add(t.getIdElement().toUnqualifiedVersionless().getValue()));
 		}
 
+		assertEquals(ids.size(), foundIds.size());
 		ids.sort(new ComparableComparator<>());
 		foundIds.sort(new ComparableComparator<>());
 		assertEquals(ids, foundIds);
@@ -1327,7 +1329,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '6'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
 		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
@@ -1343,7 +1345,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '6'"));
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("offset '5'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
@@ -1351,22 +1353,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 		assertEquals(1, myCaptureQueriesListener.countCommits());
 		assertEquals(0, myCaptureQueriesListener.countRollbacks());
 
-		assertThat(outcome.getLink("next").getUrl(), containsString("Patient?_count=5&_offset=10&active=true"));
+		assertNull(outcome.getLink("next"));
-
-		// Third page (no results)
-
-		myCaptureQueriesListener.clear();
-		outcome = myClient.search().forResource("Patient").where(Patient.ACTIVE.exactly().code("true")).offset(10).count(5).returnBundle(Bundle.class).execute();
-		assertThat(toUnqualifiedVersionlessIdValues(outcome).toString(), toUnqualifiedVersionlessIdValues(outcome), empty());
-		myCaptureQueriesListener.logSelectQueries();
-		assertEquals(1, myCaptureQueriesListener.countSelectQueries());
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("offset '10'"));
-		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
-		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
-		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
-
 	}
 
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/ForceOffsetSearchModeInterceptorTest.java

@@ -13,10 +13,7 @@ import org.junit.jupiter.api.Test;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
@@ -66,7 +63,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '6'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
 		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
@@ -91,7 +88,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '6'"));
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("offset '5'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
@@ -99,31 +96,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 		assertEquals(1, myCaptureQueriesListener.countCommits());
 		assertEquals(0, myCaptureQueriesListener.countRollbacks());
 
-		assertThat(outcome.getLink("next").getUrl(), containsString("Patient?_count=5&_offset=10&active=true"));
+		assertNull(outcome.getLink("next"));
-
-		// Third page (no results)
-
-		myCaptureQueriesListener.clear();
-		Bundle outcome3 = myClient
-			.search()
-			.forResource("Patient")
-			.where(Patient.ACTIVE.exactly().code("true"))
-			.offset(10)
-			.count(5)
-			.returnBundle(Bundle.class)
-			.execute();
-		assertThat(toUnqualifiedVersionlessIdValues(outcome3).toString(), toUnqualifiedVersionlessIdValues(outcome3), empty());
-		myCaptureQueriesListener.logSelectQueries();
-		assertEquals(1, myCaptureQueriesListener.countSelectQueries());
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '5'"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("offset '10'"));
-		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
-		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
-		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
-
-		assertNull(outcome3.getLink("next"), () -> outcome3.getLink("next").getUrl());
-
 	}
 
 	@Test
@@ -148,11 +121,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 
 		assertThat(secondPageBundle.getEntry(), hasSize(5));
 
-		Bundle thirdPageBundle = myClient.loadPage().next(secondPageBundle).execute();
+		assertNull(secondPageBundle.getLink("next"));
-
-		assertThat(thirdPageBundle.getEntry(), hasSize(0));
-		assertNull(thirdPageBundle.getLink("next"), () -> thirdPageBundle.getLink("next").getUrl());
-
 	}
 
 
@@ -180,7 +149,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '7'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '8'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
 		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());
@@ -203,7 +172,7 @@ public class ForceOffsetSearchModeInterceptorTest extends BaseResourceProviderR4
 		myCaptureQueriesListener.logSelectQueries();
 		assertEquals(2, myCaptureQueriesListener.countSelectQueries());
 		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0"));
-		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '7'"));
+		assertThat(myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false), containsString("limit '8'"));
 		assertEquals(0, myCaptureQueriesListener.countInsertQueries());
 		assertEquals(0, myCaptureQueriesListener.countUpdateQueries());
 		assertEquals(0, myCaptureQueriesListener.countDeleteQueries());

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderCustomSearchParamR4Test.java

@@ -484,7 +484,6 @@ public class ResourceProviderCustomSearchParamR4Test extends BaseResourceProvide
 	 */
 	@Test
 	public void testCustomParameterMatchingManyValues() {
-
 		List<String> found = new ArrayList<>();
 
 		class Interceptor {
@@ -496,7 +495,6 @@ public class ResourceProviderCustomSearchParamR4Test extends BaseResourceProvide
 		Interceptor interceptor = new Interceptor();
 		myInterceptorRegistry.registerInterceptor(interceptor);
 		try {
-
 			int textIndex = 0;
 			List<Long> ids = new ArrayList<>();
 			for (int i = 0; i < 200; i++) {
@@ -549,9 +547,8 @@ public class ResourceProviderCustomSearchParamR4Test extends BaseResourceProvide
 			ourLog.info("Found: {}", found);
 
 			runInTransaction(() -> {
-
+				List<?> currentResults = myEntityManager.createNativeQuery("select distinct resourceta0_.RES_ID as col_0_0_ from HFJ_RESOURCE resourceta0_ left outer join HFJ_SPIDX_STRING myparamsst1_ on resourceta0_.RES_ID=myparamsst1_.RES_ID where myparamsst1_.HASH_NORM_PREFIX='5901791607832193956' and (myparamsst1_.SP_VALUE_NORMALIZED like 'SECTION%') limit '500'").getResultList();
-				List currentResults = myEntityManager.createNativeQuery("select distinct resourceta0_.RES_ID as col_0_0_ from HFJ_RESOURCE resourceta0_ left outer join HFJ_SPIDX_STRING myparamsst1_ on resourceta0_.RES_ID=myparamsst1_.RES_ID where myparamsst1_.HASH_NORM_PREFIX='5901791607832193956' and (myparamsst1_.SP_VALUE_NORMALIZED like 'SECTION%') limit '500'").getResultList();
+				List<?> currentResources = myEntityManager.createNativeQuery("select resourceta0_.RES_ID as col_0_0_ from HFJ_RESOURCE resourceta0_").getResultList();
-				List currentResources = myEntityManager.createNativeQuery("select resourceta0_.RES_ID as col_0_0_ from HFJ_RESOURCE resourceta0_").getResultList();
 
 				List<Search> searches = mySearchEntityDao.findAll();
 				assertEquals(1, searches.size());

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4EverythingTest.java

@@ -1012,7 +1012,6 @@ public class ResourceProviderR4EverythingTest extends BaseResourceProviderR4Test
 		assertThat(ids, containsInAnyOrder("Patient/FOO", "Observation/BAZ"));
 	}
 
-
 	@Test
 	public void testPagingOverEverythingSet() throws InterruptedException {
 		Patient p = new Patient();

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java

@@ -25,11 +25,13 @@ import ca.uhn.fhir.model.primitive.InstantDt;
 import ca.uhn.fhir.model.primitive.UriDt;
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.parser.StrictErrorHandler;
+import ca.uhn.fhir.rest.api.CacheControlDirective;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.MethodOutcome;
 import ca.uhn.fhir.rest.api.PreferReturnEnum;
 import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
 import ca.uhn.fhir.rest.api.SummaryEnum;
+import ca.uhn.fhir.rest.api.server.IRestfulServer;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.client.apache.ResourceEntity;
 import ca.uhn.fhir.rest.client.api.IClientInterceptor;
@@ -42,6 +44,7 @@ import ca.uhn.fhir.rest.gclient.NumberClientParam;
 import ca.uhn.fhir.rest.gclient.StringClientParam;
 import ca.uhn.fhir.rest.param.DateRangeParam;
 import ca.uhn.fhir.rest.param.ParamPrefixEnum;
+import ca.uhn.fhir.rest.server.IPagingProvider;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
@@ -159,8 +162,10 @@ import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Spy;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.util.AopTestUtils;
 import org.springframework.transaction.TransactionStatus;
@@ -220,6 +225,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 @SuppressWarnings("Duplicates")
 public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
@@ -255,6 +263,8 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		myStorageSettings.setUpdateWithHistoryRewriteEnabled(false);
 		myStorageSettings.setPreserveRequestIdInResourceBody(false);
 
+		when(myPagingProvider.canStoreSearchResults())
+			.thenCallRealMethod();
 	}
 
 	@BeforeEach
@@ -2718,6 +2728,90 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		assertEquals(total + 1, ids.size());
 	}
 
+
+	@ParameterizedTest
+	@CsvSource({
+		"true,19,10",
+		"false,19,10",
+		"true,20,0",
+		"false,20,0"
+	})
+	public void testPagingWithIncludesReturnsConsistentValues(
+		boolean theAllowStoringSearchResults,
+		int theResourceCount,
+		int theOrgCount
+	) {
+		// setup
+
+		// create resources
+		{
+			Coding tagCode = new Coding();
+			tagCode.setCode("test");
+			tagCode.setSystem("http://example.com");
+			int orgCount = theOrgCount;
+			for (int i = 0; i < theResourceCount; i++) {
+				Task t = new Task();
+				t.getMeta()
+					.addTag(tagCode);
+				t.setStatus(Task.TaskStatus.REQUESTED);
+				if (orgCount > 0) {
+					Organization org = new Organization();
+					org.setName("ORG");
+					IIdType orgId = myOrganizationDao.create(org).getId().toUnqualifiedVersionless();
+
+					orgCount--;
+					t.getOwner().setReference(orgId.getValue());
+				}
+				myTaskDao.create(t);
+			}
+		}
+
+		// when
+		if (!theAllowStoringSearchResults) {
+			// we don't actually allow this in our current
+			// pagingProvider implementations (except for history).
+			// But we will test with it because our ResponsePage
+			// is what's under test here
+			when(myPagingProvider.canStoreSearchResults())
+				.thenReturn(false);
+		}
+
+		int requestedAmount = 10;
+		Bundle bundle = myClient
+			.search()
+			.byUrl("Task?_count=10&_tag=test&status=requested&_include=Task%3Aowner&_sort=status")
+			.returnBundle(Bundle.class)
+			.execute();
+		int count = bundle.getEntry().size();
+		assertFalse(bundle.getEntry().isEmpty());
+
+		String nextUrl = null;
+		do {
+			Bundle.BundleLinkComponent nextLink = bundle.getLink("next");
+			if (nextLink != null) {
+				nextUrl = nextLink.getUrl();
+
+				// make sure we're always requesting 10
+				assertTrue(nextUrl.contains(String.format("_count=%d", requestedAmount)));
+
+				// get next batch
+				bundle = myClient.fetchResourceFromUrl(Bundle.class, nextUrl);
+				int received = bundle.getEntry().size();
+
+				// every next result should produce results
+				assertFalse(bundle.getEntry().isEmpty());
+				count += received;
+			} else {
+				nextUrl = null;
+			}
+		} while (nextUrl != null);
+
+		// verify
+		// we should receive all resources and linked resources
+		assertEquals(theResourceCount + theOrgCount, count);
+	}
+
+
 	@Test
 	public void testPagingWithIncludesReturnsConsistentValues() {
 		// setup
@@ -3204,7 +3298,11 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
 		});
 		myCaptureQueriesListener.logAllQueriesForCurrentThread();
 
-		Bundle bundle = myClient.search().forResource("Patient").returnBundle(Bundle.class).execute();
+		Bundle bundle = myClient
+			.search()
+			.forResource("Patient")
+			.returnBundle(Bundle.class)
+			.execute();
 		ourLog.debug("Result: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(bundle));
 		assertEquals(2, bundle.getTotal());
 		assertEquals(1, bundle.getEntry().size());

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java

@@ -215,7 +215,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 
 @ExtendWith(SpringExtension.class)
-@ContextConfiguration(classes = {TestR4Config.class})
+@ContextConfiguration(classes = {
+	TestR4Config.class
+})
 public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuilder {
 	public static final String MY_VALUE_SET = "my-value-set";
 	public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set";
@@ -398,6 +400,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
 	@Autowired
 	@Qualifier("myOrganizationAffiliationDaoR4")
 	protected IFhirResourceDao<OrganizationAffiliation> myOrganizationAffiliationDao;
+
 	@Autowired
 	protected DatabaseBackedPagingProvider myPagingProvider;
 	@Autowired

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestHapiJpaConfig.java

@@ -0,0 +1,21 @@
+package ca.uhn.fhir.jpa.test.config;
+
+import ca.uhn.fhir.jpa.config.HapiJpaConfig;
+import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import static org.mockito.Mockito.spy;
+
+/**
+ * This is a Test configuration class that allows spying underlying JpaConfigs beans
+ */
+@Configuration
+public class TestHapiJpaConfig extends HapiJpaConfig {
+
+	@Override
+	@Bean
+	public DatabaseBackedPagingProvider databaseBackedPagingProvider() {
+		return spy(super.databaseBackedPagingProvider());
+	}
+}

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java

@@ -24,7 +24,6 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.batch2.JpaBatch2Config;
 import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
 import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl;
-import ca.uhn.fhir.jpa.config.HapiJpaConfig;
 import ca.uhn.fhir.jpa.config.PackageLoaderConfig;
 import ca.uhn.fhir.jpa.config.r4.JpaR4Config;
 import ca.uhn.fhir.jpa.config.util.HapiEntityManagerFactoryUtil;
@@ -65,7 +64,7 @@ import static org.junit.jupiter.api.Assertions.fail;
 @Import({
 	JpaR4Config.class,
 	PackageLoaderConfig.class,
-	HapiJpaConfig.class,
+	TestHapiJpaConfig.class,
 	TestJPAConfig.class,
 	TestHSearchAddInConfig.DefaultLuceneHeap.class,
 	JpaBatch2Config.class,

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/IBundleProvider.java

@@ -119,6 +119,7 @@ public interface IBundleProvider {
 	 * server's processing rules (e.g. _include'd resources, OperationOutcome, etc.). For example,
 	 * if the method is invoked with index 0,10 the method might return 10 search results, plus an
 	 * additional 20 resources which matched a client's _include specification.
+	 * </p>
 	 * <p>
 	 * Note that if this bundle provider was loaded using a
 	 * page ID (i.e. via {@link ca.uhn.fhir.rest.server.IPagingProvider#retrieveResultList(RequestDetails, String, String)}

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/SimpleBundleProvider.java

@@ -43,6 +43,15 @@ public class SimpleBundleProvider implements IBundleProvider {
 	private Integer myCurrentPageSize;
 	private ResponsePage.ResponsePageBuilder myPageBuilder;
 
+	/**
+	 * The actual number of resources we have tried to fetch.
+	 * This value will only be populated if there is a
+	 * _count query parameter provided.
+	 * In which case, it will be the total number of resources
+	 * we tried to fetch (should be _count + 1 for accurate paging)
+	 */
+	private int myTotalResourcesRequestedReturned = -1;
+
 	/**
 	 * Constructor
 	 */
@@ -144,6 +153,7 @@ public class SimpleBundleProvider implements IBundleProvider {
 	@Override
 	public List<IBaseResource> getResources(
 			int theFromIndex, int theToIndex, @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
+		theResponsePageBuilder.setTotalRequestedResourcesFetched(myTotalResourcesRequestedReturned);
 		return (List<IBaseResource>)
 				myList.subList(Math.min(theFromIndex, myList.size()), Math.min(theToIndex, myList.size()));
 	}
@@ -153,6 +163,10 @@ public class SimpleBundleProvider implements IBundleProvider {
 		return myUuid;
 	}
 
+	public void setTotalResourcesRequestedReturned(int theAmount) {
+		myTotalResourcesRequestedReturned = theAmount;
+	}
+
 	/**
 	 * Defaults to null
 	 */

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/ResponseBundleBuilder.java

@@ -105,8 +105,11 @@ public class ResponseBundleBuilder {
 			pageSize = pagingCalculatePageSize(requestedPage, server.getPagingProvider());
 
 			Integer size = bundleProvider.size();
-			numToReturn =
+			if (size == null) {
-					(size == null) ? pageSize : Math.min(pageSize, size.intValue() - theResponseBundleRequest.offset);
+				numToReturn = pageSize;
+			} else {
+				numToReturn = Math.min(pageSize, size.intValue() - theResponseBundleRequest.offset);
+			}
 
 			resourceList =
 					pagingBuildResourceList(theResponseBundleRequest, bundleProvider, numToReturn, responsePageBuilder);
@@ -252,6 +255,7 @@ public class ResponseBundleBuilder {
 				RestfulServerUtils.prettyPrintResponse(server, theResponseBundleRequest.requestDetails),
 				theResponseBundleRequest.bundleType);
 
+		// set self link
 		retval.setSelf(theResponseBundleRequest.linkSelf);
 
 		// determine if we are using offset / uncached pages

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/ResponsePage.java

@@ -71,6 +71,16 @@ public class ResponsePage {
 	 * even though it will change number of resources returned.
 	 */
 	private final int myOmittedResourceCount;
+	/**
+	 * This is the total count of requested resources
+	 * (ie, non-omitted, non-_include'd resource count).
+	 * We typically fetch (for offset queries) 1 more than
+	 * we need so we know if there is an additional page
+	 * to fetch.
+	 * But this is determined by the implementers of
+	 * IBundleProvider.
+	 */
+	private final int myTotalRequestedResourcesFetched;
 
 	/**
 	 * The bundle provider.
@@ -109,6 +119,7 @@ public class ResponsePage {
 			int theNumToReturn,
 			int theIncludedResourceCount,
 			int theOmittedResourceCount,
+			int theTotalRequestedResourcesFetched,
 			IBundleProvider theBundleProvider) {
 		mySearchId = theSearchId;
 		myResourceList = theResourceList;
@@ -116,6 +127,7 @@ public class ResponsePage {
 		myNumToReturn = theNumToReturn;
 		myIncludedResourceCount = theIncludedResourceCount;
 		myOmittedResourceCount = theOmittedResourceCount;
+		myTotalRequestedResourcesFetched = theTotalRequestedResourcesFetched;
 		myBundleProvider = theBundleProvider;
 
 		myNumTotalResults = myBundleProvider.size();
@@ -190,24 +202,16 @@ public class ResponsePage {
 				return StringUtils.isNotBlank(myBundleProvider.getNextPageId());
 			case NONCACHED_OFFSET:
 				if (myNumTotalResults == null) {
-					/*
+					if (hasNextPageWithoutKnowingTotal()) {
-					 * Having a null total results is synonymous with
-					 * having a next link. Once our results are exhausted,
-					 * we will always have a myNumTotalResults value.
-					 *
-					 * Alternatively, if _total=accurate is provided,
-					 * we'll also have a myNumTotalResults value.
-					 */
 						return true;
+					}
 				} else if (myNumTotalResults > myNumToReturn + ObjectUtils.defaultIfNull(myRequestedPage.offset, 0)) {
 					return true;
 				}
 				break;
 			case SAVED_SEARCH:
 				if (myNumTotalResults == null) {
-					if (myPageSize == myResourceList.size() + myOmittedResourceCount - myIncludedResourceCount) {
+					if (hasNextPageWithoutKnowingTotal()) {
-						// if the size of the resource list - included resources + omitted resources == pagesize
-						// we have more pages
 						return true;
 					}
 				} else if (myResponseBundleRequest.offset + myNumToReturn < myNumTotalResults) {
@@ -220,6 +224,53 @@ public class ResponsePage {
 		return false;
 	}
 
+	/**
+	 * If myNumTotalResults is null, it typically means we don't
+	 * have an accurate total.
+	 *
+	 * Ie, we're in the middle of a set of pages (of non-named page results),
+	 * and _total=accurate was not passed.
+	 *
+	 * This typically always means that a
+	 * 'next' link definitely exists.
+	 *
+	 * But there are cases where this might not be true:
+	 * * the last page of a search that also has an _include
+	 * 	query parameter where the total of resources + _include'd
+	 * 	resources is > the page size expected to be returned.
+	 * * the last page of a search that returns the exact number
+	 * 	of resources requested
+	 *
+	 * In these case, we must check to see if the returned
+	 * number of *requested* resources.
+	 * If our bundleprovider has fetched > requested,
+	 * we'll know that there are more resources already.
+	 * But if it hasn't, we'll have to check pagesize compared to
+	 * _include'd count, omitted count, and resource count.
+	 */
+	private boolean hasNextPageWithoutKnowingTotal() {
+		// if we have totalRequestedResource count, and it's not equal to pagesize,
+		// then we can use this, alone, to determine if there are more pages
+		if (myTotalRequestedResourcesFetched >= 0) {
+			if (myPageSize < myTotalRequestedResourcesFetched) {
+				return true;
+			}
+		} else {
+			// otherwise we'll try and determine if there are next links based on the following
+			// calculation:
+			// resourceList.size - included resources + omitted resources == pagesize
+			// -> we (most likely) have more resources
+			if (myPageSize == myResourceList.size() - myIncludedResourceCount + myOmittedResourceCount) {
+				ourLog.warn(
+						"Returning a next page based on calculated resource count."
+								+ " This could be inaccurate if the exact number of resources were fetched is equal to the pagesize requested. "
+								+ " Consider setting ResponseBundleBuilder.setTotalResourcesFetchedRequest after fetching resources.");
+				return true;
+			}
+		}
+		return false;
+	}
+
 	public void setNextPageIfNecessary(BundleLinks theLinks) {
 		if (hasNextPage()) {
 			String next;
@@ -356,9 +407,10 @@ public class ResponsePage {
 		private int myIncludedResourceCount;
 		private int myOmittedResourceCount;
 		private IBundleProvider myBundleProvider;
+		private int myTotalRequestedResourcesFetched = -1;
 
-		public ResponsePageBuilder setToOmittedResourceCount(int theOmittedResourcesCountToAdd) {
+		public ResponsePageBuilder setOmittedResourceCount(int theOmittedResourceCount) {
-			myOmittedResourceCount = theOmittedResourcesCountToAdd;
+			myOmittedResourceCount = theOmittedResourceCount;
 			return this;
 		}
 
@@ -392,6 +444,36 @@ public class ResponsePage {
 			return this;
 		}
 
+		public ResponsePageBuilder setTotalRequestedResourcesFetched(int theTotalRequestedResourcesFetched) {
+			myTotalRequestedResourcesFetched = theTotalRequestedResourcesFetched;
+			return this;
+		}
+
+		/**
+		 * Combine this builder with a second buider.
+		 * Useful if a second page is requested, but you do not wish to
+		 * overwrite the current values.
+		 *
+		 * Will not replace searchId, nor IBundleProvider (which should be
+		 * the exact same for any subsequent searches anyways).
+		 *
+		 * Will also not copy pageSize nor numToReturn, as these should be
+		 * the same for any single search result set.
+		 *
+		 * @param theSecondBuilder - a second builder (cannot be this one)
+		 */
+		public void combineWith(ResponsePageBuilder theSecondBuilder) {
+			assert theSecondBuilder != this; // don't want to combine with itself
+
+			if (myTotalRequestedResourcesFetched != -1 && theSecondBuilder.myTotalRequestedResourcesFetched != -1) {
+				myTotalRequestedResourcesFetched += theSecondBuilder.myTotalRequestedResourcesFetched;
+			}
+
+			// primitives can always be added
+			myIncludedResourceCount += theSecondBuilder.myIncludedResourceCount;
+			myOmittedResourceCount += theSecondBuilder.myOmittedResourceCount;
+		}
+
 		public ResponsePage build() {
 			return new ResponsePage(
 					mySearchId, // search id
@@ -400,6 +482,7 @@ public class ResponsePage {
 					myNumToReturn, // num to return
 					myIncludedResourceCount, // included count
 					myOmittedResourceCount, // omitted resources
+					myTotalRequestedResourcesFetched, // total count of requested resources
 					myBundleProvider // the bundle provider
 					);
 		}

hapi-fhir-server/src/test/java/ca/uhn/fhir/rest/api/server/method/ResponsePageTest.java

@@ -161,17 +161,24 @@ public class ResponsePageTest {
 	 */
 	@ParameterizedTest
 	@CsvSource({
-		"true,false,true",
+		"true,false,true,true",
-		"true,true,true",
+		"true,true,true,true",
-		"false,false,false",
+		"false,false,false,true",
-		"false,true,false",
+		"false,true,false,true",
-		"false,false,true",
+		"false,false,true,true",
-		"false,true,true"
+		"false,true,true,true",
+		"true,false,true,false",
+		"true,true,true,false",
+		"false,false,false,false",
+		"false,true,false,false",
+		"false,false,true,false",
+		"false,true,true,false"
 	})
 	public void nonCachedOffsetPaging_setsNextPreviousLinks_test(
 		boolean theNumTotalResultsIsNull,
 		boolean theHasPreviousBoolean,
-		boolean theHasNextBoolean
+		boolean theHasNextBoolean,
+		boolean theHasTotalRequestedCountBool
 	) {
 		// setup
 		myBundleBuilder
@@ -193,6 +200,11 @@ public class ResponsePageTest {
 		} else {
 			when(myBundleProvider.size())
 				.thenReturn(null);
+			if (theHasTotalRequestedCountBool) {
+				myBundleBuilder.setTotalRequestedResourcesFetched(11); // 1 more than pagesize
+			} else {
+				myBundleBuilder.setPageSize(10);
+			}
 		}
 
 		RequestedPage requestedPage = new RequestedPage(
@@ -215,19 +227,28 @@ public class ResponsePageTest {
 
 	@ParameterizedTest
 	@CsvSource({
-		"true,false,false",
+		"true,false,false,true",
-		"true,true,false",
+		"true,true,false,true",
-		"true,false,true",
+		"true,false,true,true",
-		"true,true,true",
+		"true,true,true,true",
-		"false,false,false",
+		"false,false,false,true",
-		"false,true,false",
+		"false,true,false,true",
-		"false,false,true",
+		"false,false,true,true",
-		"false,true,true"
+		"false,true,true,true",
+		"true,false,false,false",
+		"true,true,false,false",
+		"true,false,true,false",
+		"true,true,true,false",
+		"false,false,false,false",
+		"false,true,false,false",
+		"false,false,true,false",
+		"false,true,true,false"
 	})
 	public void savedSearch_setsNextPreviousLinks_test(
 		boolean theNumTotalResultsIsNull,
 		boolean theHasPreviousBoolean,
-		boolean theHasNextBoolean
+		boolean theHasNextBoolean,
+		boolean theHasTotalRequestedFetched
 	) {
 		// setup
 		int pageSize = myList.size();
@@ -255,6 +276,12 @@ public class ResponsePageTest {
 			if (!theHasNextBoolean) {
 				myBundleBuilder.setNumToReturn(pageSize + offset + includeResourceCount);
 			}
+		} else if (theHasTotalRequestedFetched) {
+			if (theHasNextBoolean) {
+				myBundleBuilder.setTotalRequestedResourcesFetched(pageSize + 1); // 1 more than page size
+			} else {
+				myBundleBuilder.setTotalRequestedResourcesFetched(pageSize);
+			}
 		}
 
 		// when

hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/imprt/BulkDataImportProvider.java

@@ -76,13 +76,10 @@ public class BulkDataImportProvider {
 	public static final String PARAM_INPUT_TYPE = "type";
 	private static final Logger ourLog = LoggerFactory.getLogger(BulkDataImportProvider.class);
 
-	@Autowired
 	private IJobCoordinator myJobCoordinator;
 
-	@Autowired
 	private FhirContext myFhirCtx;
 
-	@Autowired
 	private IRequestPartitionHelperSvc myRequestPartitionHelperService;
 
 	private volatile List<String> myResourceTypeOrder;
@@ -94,14 +91,17 @@ public class BulkDataImportProvider {
 		super();
 	}
 
+	@Autowired
 	public void setJobCoordinator(IJobCoordinator theJobCoordinator) {
 		myJobCoordinator = theJobCoordinator;
 	}
 
+	@Autowired
 	public void setFhirContext(FhirContext theCtx) {
 		myFhirCtx = theCtx;
 	}
 
+	@Autowired
 	public void setRequestPartitionHelperService(IRequestPartitionHelperSvc theRequestPartitionHelperSvc) {
 		myRequestPartitionHelperService = theRequestPartitionHelperSvc;
 	}