More perf work

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -696,6 +696,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
 		search.setResourceType(resourceName);
 		search.setResourceId(theId);
 		search.setSearchType(SearchTypeEnum.HISTORY);
+		search.setStatus(SearchStatusEnum.FINISHED);
 		
 		if (theSince != null) {
 			if (resourceName == null) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -593,87 +593,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 		return retVal;
 	}
 
-	// @Override
-	// public IBundleProvider everything(IIdType theId) {
-	// Search search = new Search();
-	// search.setUuid(UUID.randomUUID().toString());
-	// search.setCreated(new Date());
-	// myEntityManager.persist(search);
-	//
-	// List<SearchResult> results = new ArrayList<SearchResult>();
-	// if (theId != null) {
-	// Long pid = translateForcedIdToPid(theId);
-	// ResourceTable entity = myEntityManager.find(ResourceTable.class, pid);
-	// validateGivenIdIsAppropriateToRetrieveResource(theId, entity);
-	// SearchResult res = new SearchResult(search);
-	// res.setResourcePid(pid);
-	// results.add(res);
-	// } else {
-	// TypedQuery<Tuple> query = createSearchAllByTypeQuery();
-	// for (Tuple next : query.getResultList()) {
-	// SearchResult res = new SearchResult(search);
-	// res.setResourcePid(next.get(0, Long.class));
-	// results.add(res);
-	// }
-	// }
-	//
-	// int totalCount = results.size();
-	// mySearchResultDao.save(results);
-	// mySearchResultDao.flush();
-	//
-	// CriteriaBuilder builder = myEntityManager.getCriteriaBuilder();
-	//
-	// // Load _revincludes
-	// CriteriaQuery<Long> cq = builder.createQuery(Long.class);
-	// Root<ResourceLink> from = cq.from(ResourceLink.class);
-	// cq.select(from.get("mySourceResourcePid").as(Long.class));
-	//
-	// Subquery<Long> pidsSubquery = cq.subquery(Long.class);
-	// Root<SearchResult> pidsSubqueryFrom = pidsSubquery.from(SearchResult.class);
-	// pidsSubquery.select(pidsSubqueryFrom.get("myResourcePid").as(Long.class));
-	// pidsSubquery.where(pidsSubqueryFrom.get("mySearch").in(search));
-	//
-	// cq.where(from.get("myTargetResourceId").in(pidsSubquery));
-	// TypedQuery<Long> query = myEntityManager.createQuery(cq);
-	//
-	// results = new ArrayList<SearchResult>();
-	// for (Long next : query.getResultList()) {
-	// SearchResult res = new SearchResult(search);
-	// res.setResourcePid(next);
-	// results.add(res);
-	// }
-	//
-	// // Save _revincludes
-	// totalCount += results.size();
-	// mySearchResultDao.save(results);
-	// mySearchResultDao.flush();
-	//
-	// final int finalTotalCount = totalCount;
-	// return new IBundleProvider() {
-	//
-	// @Override
-	// public int size() {
-	// return finalTotalCount;
-	// }
-	//
-	// @Override
-	// public Integer preferredPageSize() {
-	// return null;
-	// }
-	//
-	// @Override
-	// public List<IBaseResource> getResources(int theFromIndex, int theToIndex) {
-	// // TODO Auto-generated method stub
-	// return null;
-	// }
-	//
-	// @Override
-	// public InstantDt getPublished() {
-	// // TODO Auto-generated method stub
-	// return null;
-	// }
-	// };
-	// }
 
 	@Override
 	public <MT extends IBaseMetaType> MT metaGetOperation(Class<MT> theType, IIdType theId, RequestDetails theRequestDetails) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java

@@ -1315,8 +1315,10 @@ public class SearchBuilder implements ISearchBuilder {
 			@Override
 			public Long next() {
 				fetchNext();
-				Validate.isTrue(myNext != NO_MORE, "No more elements");
+				Long retVal = myNext;
-				return myNext;
+				myNext = null;
+				Validate.isTrue(retVal != NO_MORE, "No more elements");
+				return retVal;
 			}
 		};
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java

@@ -268,6 +268,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 
 		@Override
 		public Void call() throws Exception {
+			StopWatch sw = new StopWatch();
+			
 			try {
 				TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
 				txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
@@ -277,14 +279,18 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
 						doSearch();
 					}
 				});
+				
+				ourLog.info("Completed search for {} resources in {}ms", mySyncedPids.size(), sw.getMillis());
+				
 			} catch (Throwable t) {
-				ourLog.error("Failed during search loading", t);
+				ourLog.error("Failed during search loading after {}ms", t, sw.getMillis());
 				myUnsyncedPids.clear();
 
 				mySearch.setStatus(SearchStatusEnum.FAILED);
 				String failureMessage = ExceptionUtils.getRootCauseMessage(t);
 				mySearch.setFailureMessage(failureMessage);
 				saveSearch();
+				
 			}
 
 			myIdToSearchTask.remove(mySearch.getUuid());

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java

@@ -308,9 +308,6 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
 
 	@Test
 	public void testCountParam() throws Exception {
-		// NB this does not get used- The paging provider has its own limits built in
-		myDaoConfig.setHardSearchLimit(100);
-
 		List<IBaseResource> resources = new ArrayList<IBaseResource>();
 		for (int i = 0; i < 100; i++) {
 			Organization org = new Organization();
@@ -1611,6 +1608,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
 		assertEquals(77, ids.size());
 	}
 
+	@SuppressWarnings("unused")
 	@Test
 	public void testFullTextSearch() throws RuntimeException, Exception {
 		Observation obs1 = new Observation();
@@ -1920,6 +1918,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
 		}
 	}
 
+	@SuppressWarnings("unused")
 	@Test
 	public void testMetadataSuperParamsAreIncluded() throws IOException {
 		StructureDefinition p = new StructureDefinition();
@@ -2622,6 +2621,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
 
 	}
 
+	@SuppressWarnings("unused")
 	@Test
 	public void testSearchPagingKeepsOldSearches() throws Exception {
 		String methodName = "testSearchPagingKeepsOldSearches";

hapi-fhir-structures-dstu/src/main/java/ca/uhn/fhir/rest/server/Dstu1BundleFactory.java

@@ -60,6 +60,23 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 		myContext = theContext;
 	}
 
+	private void addProfileIfNeeded(IRestfulServer<?> theServer, String theServerBase, IBaseResource nextRes) {
+		RuntimeResourceDefinition def = theServer.getFhirContext().getResourceDefinition(nextRes);
+		if (theServer.getAddProfileTag() == AddProfileTagEnum.ALWAYS || !def.isStandardType()) {
+			TagList tl = ResourceMetadataKeyEnum.TAG_LIST.get((IResource) nextRes);
+			if (tl == null) {
+				tl = new TagList();
+				ResourceMetadataKeyEnum.TAG_LIST.put((IResource) nextRes, tl);
+			}
+
+			RuntimeResourceDefinition nextDef = myContext.getResourceDefinition(nextRes);
+			String profile = nextDef.getResourceProfile(theServerBase);
+			if (isNotBlank(profile)) {
+				tl.add(new Tag(Tag.HL7_ORG_PROFILE_TAG, profile, null));
+			}
+		}
+	}
+
 	@Override
 	public void addResourcesToBundle(List<IBaseResource> theResult, BundleTypeEnum theBundleType, String theServerBase, BundleInclusionRule theBundleInclusionRule, Set<Include> theIncludes) {
 		if (myBundle == null) {
@@ -149,7 +166,7 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 			myBundle.getAuthorName().setValue(theAuthor);
 		}
 
-		if (myBundle.getUpdated().isEmpty() && isNotBlank(theLastUpdated.getValueAsString())) {
+		if (theLastUpdated != null && myBundle.getUpdated().isEmpty() && isNotBlank(theLastUpdated.getValueAsString())) {
 			myBundle.getUpdated().setValueAsString(theLastUpdated.getValueAsString());
 		}
 
@@ -190,27 +207,31 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 		int numToReturn;
 		String searchId = null;
 		List<IBaseResource> resourceList;
+		Integer numTotalResults = theResult.size();
 		if (theServer.getPagingProvider() == null) {
-			numToReturn = theResult.size();
+			numToReturn = numTotalResults;
 			resourceList = theResult.getResources(0, numToReturn);
 			RestfulServerUtils.validateResourceListNotNull(resourceList);
 
 		} else {
 			IPagingProvider pagingProvider = theServer.getPagingProvider();
-			if (theLimit == null) {
+			if (theLimit == null || theLimit.equals(Integer.valueOf(0))) {
 				numToReturn = pagingProvider.getDefaultPageSize();
 			} else {
 				numToReturn = Math.min(pagingProvider.getMaximumPageSize(), theLimit);
 			}
 
-			numToReturn = Math.min(numToReturn, theResult.size() - theOffset);
+			if (numTotalResults != null) {
+				numToReturn = Math.min(numToReturn, numTotalResults - theOffset);
+			}
+			
 			resourceList = theResult.getResources(theOffset, numToReturn + theOffset);
 			RestfulServerUtils.validateResourceListNotNull(resourceList);
 
 			if (theSearchId != null) {
 				searchId = theSearchId;
 			} else {
-				if (theResult.size() > numToReturn) {
+				if (numTotalResults == null || numTotalResults > numToReturn) {
 					searchId = pagingProvider.storeResultList(theResult);
 					Validate.notNull(searchId, "Paging provider returned null searchId");
 				}
@@ -230,7 +251,7 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 
 
 		addResourcesToBundle(new ArrayList<IBaseResource>(resourceList), theBundleType, theServerBase, theServer.getBundleInclusionRule(), theIncludes);
-		addRootPropertiesToBundle(null, theServerBase, theCompleteUrl, theResult.size(), theBundleType, theResult.getPublished());
+		addRootPropertiesToBundle(null, theServerBase, theCompleteUrl, numTotalResults, theBundleType, theResult.getPublished());
 
 		if (theServer.getPagingProvider() != null) {
 			int limit;
@@ -238,7 +259,7 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 			limit = Math.min(limit, theServer.getPagingProvider().getMaximumPageSize());
 
 			if (searchId != null) {
-				if (theOffset + numToReturn < theResult.size()) {
+				if (numTotalResults == null || theOffset + numToReturn < numTotalResults) {
 					myBundle.getLinkNext()
 							.setValue(RestfulServerUtils.createPagingLink(theIncludes, theServerBase, searchId, theOffset + numToReturn, numToReturn, theResponseEncoding, thePrettyPrint, theBundleType));
 				}
@@ -250,23 +271,6 @@ public class Dstu1BundleFactory implements IVersionSpecificBundleFactory {
 		}
 	}
 
-	private void addProfileIfNeeded(IRestfulServer<?> theServer, String theServerBase, IBaseResource nextRes) {
-		RuntimeResourceDefinition def = theServer.getFhirContext().getResourceDefinition(nextRes);
-		if (theServer.getAddProfileTag() == AddProfileTagEnum.ALWAYS || !def.isStandardType()) {
-			TagList tl = ResourceMetadataKeyEnum.TAG_LIST.get((IResource) nextRes);
-			if (tl == null) {
-				tl = new TagList();
-				ResourceMetadataKeyEnum.TAG_LIST.put((IResource) nextRes, tl);
-			}
-
-			RuntimeResourceDefinition nextDef = myContext.getResourceDefinition(nextRes);
-			String profile = nextDef.getResourceProfile(theServerBase);
-			if (isNotBlank(profile)) {
-				tl.add(new Tag(Tag.HL7_ORG_PROFILE_TAG, profile, null));
-			}
-		}
-	}
-
 	@Override
 	public void initializeBundleFromResourceList(String theAuthor, List<? extends IBaseResource> theResult, String theServerBase, String theCompleteUrl, int theTotalResults,
 			BundleTypeEnum theBundleType) {

hapi-fhir-structures-dstu/src/test/java/ca/uhn/fhir/rest/server/SearchBundleProviderWithNoSizeDstu1Test.java

@@ -0,0 +1,186 @@
+package ca.uhn.fhir.rest.server;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.collect.Lists;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.model.api.Bundle;
+import ca.uhn.fhir.model.dstu.resource.Patient;
+import ca.uhn.fhir.model.primitive.StringDt;
+import ca.uhn.fhir.rest.annotation.RequiredParam;
+import ca.uhn.fhir.rest.annotation.Search;
+import ca.uhn.fhir.rest.param.TokenAndListParam;
+import ca.uhn.fhir.util.PortUtil;
+import ca.uhn.fhir.util.TestUtil;
+
+public class SearchBundleProviderWithNoSizeDstu1Test {
+
+	private static CloseableHttpClient ourClient;
+	private static FhirContext ourCtx = FhirContext.forDstu1();
+	private static TokenAndListParam ourIdentifiers;
+	private static IBundleProvider ourLastBundleProvider;
+	private static String ourLastMethod;
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBundleProviderWithNoSizeDstu1Test.class);
+	private static int ourPort;
+
+	private static Server ourServer;
+
+	@Before
+	public void before() {
+		ourLastMethod = null;
+		ourIdentifiers = null;
+	}
+
+	@Test
+	public void testBundleProviderReturnsNoSize() throws Exception {
+		Bundle respBundle;
+		
+		ourLastBundleProvider = mock(IBundleProvider.class); 
+		when(ourLastBundleProvider.size()).thenReturn(null);
+		when(ourLastBundleProvider.getResources(any(int.class), any(int.class))).then(new Answer<List<IBaseResource>>() {
+			@Override
+			public List<IBaseResource> answer(InvocationOnMock theInvocation) throws Throwable {
+				int from =(Integer)theInvocation.getArguments()[0]; 
+				int to =(Integer)theInvocation.getArguments()[1];
+				ArrayList<IBaseResource> retVal = Lists.newArrayList();
+				for (int i = from; i < to; i++) {
+					Patient p = new Patient();
+					p.setId(Integer.toString(i));
+					retVal.add(p);
+				}
+				return retVal;
+			}});
+		
+		HttpGet httpGet;
+		CloseableHttpResponse status = null;
+		StringDt linkNext;
+		
+		try {
+			httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?_format=json");
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseBundle(responseContent);
+			
+			assertEquals(10, respBundle.getEntries().size());
+			assertEquals("Patient/0", respBundle.getEntries().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLinkNext();
+			assertNotNull(linkNext.getValue());
+			
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		
+		when(ourLastBundleProvider.size()).thenReturn(25);
+
+		try {
+			httpGet = new HttpGet(linkNext.getValue());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseBundle(responseContent);
+			
+			assertEquals(10, respBundle.getEntries().size());
+			assertEquals("Patient/10", respBundle.getEntries().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLinkNext();
+			assertNotNull(linkNext.getValue());
+
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		try {
+			httpGet = new HttpGet(linkNext.getValue());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseBundle(responseContent);
+			
+			assertEquals(5, respBundle.getEntries().size());
+			assertEquals("Patient/20", respBundle.getEntries().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLinkNext();
+			assertNull(linkNext.getValue());
+
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+	}
+
+	@AfterClass
+	public static void afterClassClearContext() throws Exception {
+		ourServer.stop();
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		ourPort = PortUtil.findFreePort();
+		ourServer = new Server(ourPort);
+
+		DummyPatientResourceProvider patientProvider = new DummyPatientResourceProvider();
+
+		ServletHandler proxyHandler = new ServletHandler();
+		RestfulServer servlet = new RestfulServer(ourCtx);
+		servlet.setPagingProvider(new FifoMemoryPagingProvider(10));
+
+		servlet.setResourceProviders(patientProvider);
+		ServletHolder servletHolder = new ServletHolder(servlet);
+		proxyHandler.addServletWithMapping(servletHolder, "/*");
+		ourServer.setHandler(proxyHandler);
+		ourServer.start();
+
+		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+		HttpClientBuilder builder = HttpClientBuilder.create();
+		builder.setConnectionManager(connectionManager);
+		ourClient = builder.build();
+
+	}
+
+	public static class DummyPatientResourceProvider implements IResourceProvider {
+
+		@Override
+		public Class<? extends IBaseResource> getResourceType() {
+			return Patient.class;
+		}
+
+		@Search()
+		public IBundleProvider searchAll() {
+			ourLastMethod = "searchAll";
+			return ourLastBundleProvider;
+		}
+
+	}
+
+}

hapi-fhir-structures-dstu2/src/main/java/ca/uhn/fhir/rest/server/provider/dstu2/Dstu2BundleFactory.java

@@ -306,8 +306,9 @@ public class Dstu2BundleFactory implements IVersionSpecificBundleFactory {
 		int numToReturn;
 		String searchId = null;
 		List<IBaseResource> resourceList;
+		Integer numTotalResults = theResult.size();
 		if (theServer.getPagingProvider() == null) {
-			numToReturn = theResult.size();
+			numToReturn = numTotalResults;
 			if (numToReturn > 0) {
 				resourceList = theResult.getResources(0, numToReturn);
 			} else {
@@ -317,13 +318,16 @@ public class Dstu2BundleFactory implements IVersionSpecificBundleFactory {
 
 		} else {
 			IPagingProvider pagingProvider = theServer.getPagingProvider();
-			if (theLimit == null) {
+			if (theLimit == null || theLimit.equals(Integer.valueOf(0))) {
 				numToReturn = pagingProvider.getDefaultPageSize();
 			} else {
 				numToReturn = Math.min(pagingProvider.getMaximumPageSize(), theLimit);
 			}
 
-			numToReturn = Math.min(numToReturn, theResult.size() - theOffset);
+			if (numTotalResults != null) {
+				numToReturn = Math.min(numToReturn, numTotalResults - theOffset);
+			}
+			
 			if (numToReturn > 0) {
 				resourceList = theResult.getResources(theOffset, numToReturn + theOffset);
 			} else {
@@ -334,7 +338,7 @@ public class Dstu2BundleFactory implements IVersionSpecificBundleFactory {
 			if (theSearchId != null) {
 				searchId = theSearchId;
 			} else {
-				if (theResult.size() > numToReturn) {
+				if (numTotalResults == null || numTotalResults > numToReturn) {
 					searchId = pagingProvider.storeResultList(theResult);
 					Validate.notNull(searchId, "Paging provider returned null searchId");
 				}
@@ -350,7 +354,7 @@ public class Dstu2BundleFactory implements IVersionSpecificBundleFactory {
 		}
 
 		addResourcesToBundle(new ArrayList<IBaseResource>(resourceList), theBundleType, theServerBase, theServer.getBundleInclusionRule(), theIncludes);
-		addRootPropertiesToBundle(null, theServerBase, theCompleteUrl, theResult.size(), theBundleType, theResult.getPublished());
+		addRootPropertiesToBundle(null, theServerBase, theCompleteUrl, numTotalResults, theBundleType, theResult.getPublished());
 
 		if (theServer.getPagingProvider() != null) {
 			int limit;
@@ -358,7 +362,7 @@ public class Dstu2BundleFactory implements IVersionSpecificBundleFactory {
 			limit = Math.min(limit, theServer.getPagingProvider().getMaximumPageSize());
 
 			if (searchId != null) {
-				if (theOffset + numToReturn < theResult.size()) {
+				if (numTotalResults == null || theOffset + numToReturn < numTotalResults) {
 					myBundle.addLink().setRelation(Constants.LINK_NEXT)
 							.setUrl(RestfulServerUtils.createPagingLink(theIncludes, theServerBase, searchId, theOffset + numToReturn, numToReturn, theResponseEncoding, thePrettyPrint, theBundleType));
 				}

hapi-fhir-structures-dstu2/src/test/java/ca/uhn/fhir/rest/server/SearchBundleProviderWithNoSizeDstu2Test.java

@@ -0,0 +1,186 @@
+package ca.uhn.fhir.rest.server;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.collect.Lists;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.model.dstu2.resource.Bundle;
+import ca.uhn.fhir.model.dstu2.resource.Bundle.Link;
+import ca.uhn.fhir.model.dstu2.resource.Patient;
+import ca.uhn.fhir.rest.annotation.RequiredParam;
+import ca.uhn.fhir.rest.annotation.Search;
+import ca.uhn.fhir.rest.param.TokenAndListParam;
+import ca.uhn.fhir.util.PortUtil;
+import ca.uhn.fhir.util.TestUtil;
+
+public class SearchBundleProviderWithNoSizeDstu2Test {
+
+	private static CloseableHttpClient ourClient;
+	private static FhirContext ourCtx = FhirContext.forDstu2();
+	private static TokenAndListParam ourIdentifiers;
+	private static IBundleProvider ourLastBundleProvider;
+	private static String ourLastMethod;
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBundleProviderWithNoSizeDstu2Test.class);
+	private static int ourPort;
+
+	private static Server ourServer;
+
+	@Before
+	public void before() {
+		ourLastMethod = null;
+		ourIdentifiers = null;
+	}
+
+	@Test
+	public void testBundleProviderReturnsNoSize() throws Exception {
+		Bundle respBundle;
+		
+		ourLastBundleProvider = mock(IBundleProvider.class); 
+		when(ourLastBundleProvider.size()).thenReturn(null);
+		when(ourLastBundleProvider.getResources(any(int.class), any(int.class))).then(new Answer<List<IBaseResource>>() {
+			@Override
+			public List<IBaseResource> answer(InvocationOnMock theInvocation) throws Throwable {
+				int from =(Integer)theInvocation.getArguments()[0]; 
+				int to =(Integer)theInvocation.getArguments()[1];
+				ArrayList<IBaseResource> retVal = Lists.newArrayList();
+				for (int i = from; i < to; i++) {
+					Patient p = new Patient();
+					p.setId(Integer.toString(i));
+					retVal.add(p);
+				}
+				return retVal;
+			}});
+		
+		HttpGet httpGet;
+		CloseableHttpResponse status = null;
+		Link linkNext;
+		
+		try {
+			httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?_format=json");
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(10, respBundle.getEntry().size());
+			assertEquals("Patient/0", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNotNull(linkNext);
+			
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		
+		when(ourLastBundleProvider.size()).thenReturn(25);
+
+		try {
+			httpGet = new HttpGet(linkNext.getUrl());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(10, respBundle.getEntry().size());
+			assertEquals("Patient/10", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNotNull(linkNext);
+
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		try {
+			httpGet = new HttpGet(linkNext.getUrl());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(5, respBundle.getEntry().size());
+			assertEquals("Patient/20", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNull(linkNext);
+
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+	}
+
+	@AfterClass
+	public static void afterClassClearContext() throws Exception {
+		ourServer.stop();
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		ourPort = PortUtil.findFreePort();
+		ourServer = new Server(ourPort);
+
+		DummyPatientResourceProvider patientProvider = new DummyPatientResourceProvider();
+
+		ServletHandler proxyHandler = new ServletHandler();
+		RestfulServer servlet = new RestfulServer(ourCtx);
+		servlet.setPagingProvider(new FifoMemoryPagingProvider(10));
+
+		servlet.setResourceProviders(patientProvider);
+		ServletHolder servletHolder = new ServletHolder(servlet);
+		proxyHandler.addServletWithMapping(servletHolder, "/*");
+		ourServer.setHandler(proxyHandler);
+		ourServer.start();
+
+		PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(5000, TimeUnit.MILLISECONDS);
+		HttpClientBuilder builder = HttpClientBuilder.create();
+		builder.setConnectionManager(connectionManager);
+		ourClient = builder.build();
+
+	}
+
+	public static class DummyPatientResourceProvider implements IResourceProvider {
+
+		@Override
+		public Class<? extends IBaseResource> getResourceType() {
+			return Patient.class;
+		}
+
+		@Search()
+		public IBundleProvider searchAll() {
+			ourLastMethod = "searchAll";
+			return ourLastBundleProvider;
+		}
+
+	}
+
+}

hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/hapi/rest/server/Dstu3BundleFactory.java

@@ -302,8 +302,9 @@ public class Dstu3BundleFactory implements IVersionSpecificBundleFactory {
     int numToReturn;
     String searchId = null;
     List<IBaseResource> resourceList;
+    Integer numTotalResults = theResult.size();
     if (theServer.getPagingProvider() == null) {
-			numToReturn = theResult.size();
+      numToReturn = numTotalResults;
       if (numToReturn > 0) {
         resourceList = theResult.getResources(0, numToReturn);
       } else {
@@ -313,13 +314,16 @@ public class Dstu3BundleFactory implements IVersionSpecificBundleFactory {
 
     } else {
       IPagingProvider pagingProvider = theServer.getPagingProvider();
-			if (theLimit == null) {
+      if (theLimit == null || theLimit.equals(Integer.valueOf(0))) {
         numToReturn = pagingProvider.getDefaultPageSize();
       } else {
         numToReturn = Math.min(pagingProvider.getMaximumPageSize(), theLimit);
       }
 
-			numToReturn = Math.min(numToReturn, theResult.size() - theOffset);
+      if (numTotalResults != null) {
+        numToReturn = Math.min(numToReturn, numTotalResults - theOffset);
+      }
+
       if (numToReturn > 0) {
         resourceList = theResult.getResources(theOffset, numToReturn + theOffset);
       } else {
@@ -330,7 +334,7 @@ public class Dstu3BundleFactory implements IVersionSpecificBundleFactory {
       if (theSearchId != null) {
         searchId = theSearchId;
       } else {
-				if (theResult.size() > numToReturn) {
+        if (numTotalResults == null || numTotalResults > numToReturn) {
           searchId = pagingProvider.storeResultList(theResult);
           Validate.notNull(searchId, "Paging provider returned null searchId");
         }
@@ -354,7 +358,7 @@ public class Dstu3BundleFactory implements IVersionSpecificBundleFactory {
       limit = Math.min(limit, theServer.getPagingProvider().getMaximumPageSize());
 
       if (searchId != null) {
-				if (theOffset + numToReturn < theResult.size()) {
+        if (numTotalResults == null || theOffset + numToReturn < numTotalResults) {
           myBundle.addLink().setRelation(Constants.LINK_NEXT)
               .setUrl(RestfulServerUtils.createPagingLink(theIncludes, theServerBase, searchId, theOffset + numToReturn, numToReturn, theResponseEncoding, thePrettyPrint, theBundleType));
         }

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/rest/server/SearchBundleProviderWithNoSizeDstu3Test.java

@@ -1,8 +1,7 @@
 package ca.uhn.fhir.rest.server;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
-import static org.mockito.Mockito.when;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -19,6 +18,8 @@ import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.hl7.fhir.dstu3.model.Bundle;
+import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.dstu3.model.Bundle.BundleLinkComponent;
 import org.hl7.fhir.dstu3.model.HumanName;
 import org.hl7.fhir.dstu3.model.OperationOutcome;
 import org.hl7.fhir.dstu3.model.Patient;
@@ -27,6 +28,10 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.collect.Lists;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.annotation.RequiredParam;
@@ -58,16 +63,78 @@ public class SearchBundleProviderWithNoSizeDstu3Test {
 		Bundle respBundle;
 		
 		ourLastBundleProvider = mock(IBundleProvider.class); 
-		when(ourLastBundleProvider.size()).thenReturn(-1);
+		when(ourLastBundleProvider.size()).thenReturn(null);
+		when(ourLastBundleProvider.getResources(any(int.class), any(int.class))).then(new Answer<List<IBaseResource>>() {
+			@Override
+			public List<IBaseResource> answer(InvocationOnMock theInvocation) throws Throwable {
+				int from =(Integer)theInvocation.getArguments()[0]; 
+				int to =(Integer)theInvocation.getArguments()[1];
+				ArrayList<IBaseResource> retVal = Lists.newArrayList();
+				for (int i = from; i < to; i++) {
+					Patient p = new Patient();
+					p.setId(Integer.toString(i));
+					retVal.add(p);
+				}
+				return retVal;
+			}});
+		
+		HttpGet httpGet;
+		CloseableHttpResponse status = null;
+		BundleLinkComponent linkNext;
 		
-		HttpGet httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?_format=json");
-		CloseableHttpResponse status = ourClient.execute(httpGet);
 		try {
+			httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?_format=json");
+			status = ourClient.execute(httpGet);
 			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(responseContent);
 			assertEquals(200, status.getStatusLine().getStatusCode());
 			assertEquals("searchAll", ourLastMethod);
 			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(10, respBundle.getEntry().size());
+			assertEquals("Patient/0", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNotNull(linkNext);
+			
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		
+		when(ourLastBundleProvider.size()).thenReturn(25);
+
+		try {
+			httpGet = new HttpGet(linkNext.getUrl());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(10, respBundle.getEntry().size());
+			assertEquals("Patient/10", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNotNull(linkNext);
+
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+		try {
+			httpGet = new HttpGet(linkNext.getUrl());
+			status = ourClient.execute(httpGet);
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(200, status.getStatusLine().getStatusCode());
+			assertEquals("searchAll", ourLastMethod);
+			respBundle = ourCtx.newJsonParser().parseResource(Bundle.class, responseContent);
+			
+			assertEquals(5, respBundle.getEntry().size());
+			assertEquals("Patient/20", respBundle.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless().getValue());
+			linkNext = respBundle.getLink("next");
+			assertNull(linkNext);
+
 		} finally {
 			IOUtils.closeQuietly(status.getEntity().getContent());
 		}