honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optionally require JPA server to collect count information even for 

large searches
This commit is contained in:
James Agnew 2017-10-13 06:31:37 -04:00
parent 4b8a8e8af1
commit 290894557d
7 changed files with 158 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View File

@ -108,6 +108,7 @@ public class DaoConfig {
private Set<String> myTreatReferencesAsLogical = new HashSet<String>(DEFAULT_LOGICAL_BASE_URLS);
private boolean myAutoCreatePlaceholderReferenceTargets;
private Integer myCacheControlNoStoreMaxResultsUpperLimit = 1000;
private Integer myCountSearchResultsUpTo = null;
/**
* Constructor
@ -152,6 +153,56 @@ public class DaoConfig {
myCacheControlNoStoreMaxResultsUpperLimit = theCacheControlNoStoreMaxResults;
}
/**
* When searching, if set to a non-null value (default is <code>null</code>) the
* search coordinator will attempt to find at least this many results
* before returning a response to the client. This parameter mainly affects
* whether a "total count" is included in the response bundle for searches that
* return large amounts of data.
* <p>
* For a search that returns 10000 results, if this value is set to
* 10000 the search coordinator will find all 10000 results
* prior to returning, so the initial response bundle will have the
* total set to 10000. If this value is null (or less than 10000)
* the response bundle will likely return slightly faster, but will
* not include the total. Subsequent page requests will likely
* include the total however, if they are performed after the
* search coordinator has found all results.
* </p>
* <p>
* Set this value to <code>0</code> to always load all
* results before returning.
* </p>
*/
public Integer getCountSearchResultsUpTo() {
return myCountSearchResultsUpTo;
}
/**
* When searching, if set to a non-null value (default is <code>null</code>) the
* search coordinator will attempt to find at least this many results
* before returning a response to the client. This parameter mainly affects
* whether a "total count" is included in the response bundle for searches that
* return large amounts of data.
* <p>
* For a search that returns 10000 results, if this value is set to
* 10000 the search coordinator will find all 10000 results
* prior to returning, so the initial response bundle will have the
* total set to 10000. If this value is null (or less than 10000)
* the response bundle will likely return slightly faster, but will
* not include the total. Subsequent page requests will likely
* include the total however, if they are performed after the
* search coordinator has found all results.
* </p>
* <p>
* Set this value to <code>0</code> to always load all
* results before returning.
* </p>
*/
public void setCountSearchResultsUpTo(Integer theCountSearchResultsUpTo) {
myCountSearchResultsUpTo = theCountSearchResultsUpTo;
}
/**
* When a code system is added that contains more than this number of codes,
* the code system will be indexed later in an incremental process in order to
@ -357,11 +408,8 @@ public class DaoConfig {
/**
* This may be used to optionally register server interceptors directly against the DAOs.
*/
public void setInterceptors(IServerInterceptor... theInterceptor) {
setInterceptors(new ArrayList<IServerInterceptor>());
if (theInterceptor != null && theInterceptor.length != 0) {
getInterceptors().addAll(Arrays.asList(theInterceptor));
}
public void setInterceptors(List<IServerInterceptor> theInterceptors) {
myInterceptors = theInterceptors;
}
/**
@ -959,8 +1007,11 @@ public class DaoConfig {
/**
* This may be used to optionally register server interceptors directly against the DAOs.
*/
public void setInterceptors(List<IServerInterceptor> theInterceptors) {
myInterceptors = theInterceptors;
public void setInterceptors(IServerInterceptor... theInterceptor) {
setInterceptors(new ArrayList<IServerInterceptor>());
if (theInterceptor != null && theInterceptor.length != 0) {
getInterceptors().addAll(Arrays.asList(theInterceptor));
}
}
/**

14
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java
View File

@ -509,6 +509,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
}
myIdToSearchTask.remove(mySearch.getUuid());
myInitialCollectionLatch.countDown();
myCompletionLatch.countDown();
return null;
}
@ -574,7 +575,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
}
}
ArrayList<Long> retVal = new ArrayList<Long>();
ArrayList<Long> retVal = new ArrayList<>();
synchronized (mySyncedPids) {
verifySearchHasntFailedOrThrowInternalErrorException(mySearch);
@ -645,7 +646,16 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
}
});
myInitialCollectionLatch.countDown();
int numSynced;
synchronized (mySyncedPids) {
numSynced = mySyncedPids.size();
}
if (myDaoConfig.getCountSearchResultsUpTo() == null ||
myDaoConfig.getCountSearchResultsUpTo() <= 0 ||
myDaoConfig.getCountSearchResultsUpTo() <= numSynced) {
myInitialCollectionLatch.countDown();
}
}
}

71
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java
View File

@ -3194,6 +3194,77 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
assertEquals(SearchEntryMode.INCLUDE, found.getEntry().get(1).getSearch().getMode());
}
@Test
public void testSearchWithCountNotSet() throws Exception {
mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(1);
mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(100);
for (int i =0; i < 10; i++) {
Patient pat = new Patient();
pat.addIdentifier().setSystem("urn:system:rpdstu2").setValue("test" + i);
ourClient.create().resource(pat).execute();
}
Bundle found = ourClient
.search()
.forResource(Patient.class)
.returnBundle(Bundle.class)
.count(1)
.execute();
// If this fails under load, try increasing the throttle above
assertEquals(null, found.getTotalElement().getValue());
assertEquals(1, found.getEntry().size());
}
@Test
public void testSearchWithCountSearchResultsUpTo5() throws Exception {
mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(1);
mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(100);
myDaoConfig.setCountSearchResultsUpTo(5);
for (int i =0; i < 10; i++) {
Patient pat = new Patient();
pat.addIdentifier().setSystem("urn:system:rpdstu2").setValue("test" + i);
ourClient.create().resource(pat).execute();
}
Bundle found = ourClient
.search()
.forResource(Patient.class)
.returnBundle(Bundle.class)
.count(1)
.execute();
// If this fails under load, try increasing the throttle above
assertEquals(null, found.getTotalElement().getValue());
assertEquals(1, found.getEntry().size());
}
@Test
public void testSearchWithCountSearchResultsUpTo20() throws Exception {
mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(1);
mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(100);
myDaoConfig.setCountSearchResultsUpTo(20);
for (int i =0; i < 10; i++) {
Patient pat = new Patient();
pat.addIdentifier().setSystem("urn:system:rpdstu2").setValue("test" + i);
ourClient.create().resource(pat).execute();
}
Bundle found = ourClient
.search()
.forResource(Patient.class)
.returnBundle(Bundle.class)
.count(1)
.execute();
// If this fails under load, try increasing the throttle above
assertEquals(10, found.getTotalElement().getValue().intValue());
assertEquals(1, found.getEntry().size());
}
@Test()
public void testSearchWithInvalidNumberPrefix() throws Exception {
try {

1
hapi-fhir-jpaserver-uhnfhirtest/src/main/java/ca/uhn/fhirtest/config/TdlDstu2Config.java
View File

@ -67,6 +67,7 @@ public class TdlDstu2Config extends BaseJavaConfigDstu2 {
retVal.getTreatBaseUrlsAsLocal().add("http://fhirtest.uhn.ca/testDataLibraryDstu2");
retVal.getTreatBaseUrlsAsLocal().add("https://fhirtest.uhn.ca/testDataLibraryDstu2");
retVal.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED);
retVal.setCountSearchResultsUpTo(TestR4Config.COUNT_SEARCH_RESULTS_UP_TO);
return retVal;
}

1
hapi-fhir-jpaserver-uhnfhirtest/src/main/java/ca/uhn/fhirtest/config/TdlDstu3Config.java
View File

@ -57,6 +57,7 @@ public class TdlDstu3Config extends BaseJavaConfigDstu3 {
retVal.getTreatBaseUrlsAsLocal().add("http://fhirtest.uhn.ca/testDataLibraryStu3");
retVal.getTreatBaseUrlsAsLocal().add("https://fhirtest.uhn.ca/testDataLibraryStu3");
retVal.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED);
retVal.setCountSearchResultsUpTo(TestR4Config.COUNT_SEARCH_RESULTS_UP_TO);
return retVal;
}

2
hapi-fhir-jpaserver-uhnfhirtest/src/main/java/ca/uhn/fhirtest/config/TestR4Config.java
View File

@ -34,6 +34,7 @@ public class TestR4Config extends BaseJavaConfigR4 {
public static final String FHIR_DB_USERNAME = "${fhir.db.username}";
public static final String FHIR_DB_PASSWORD = "${fhir.db.password}";
public static final String FHIR_LUCENE_LOCATION_R4 = "${fhir.lucene.location.r4}";
public static final Integer COUNT_SEARCH_RESULTS_UP_TO = 20000;
@Value(TestR4Config.FHIR_DB_USERNAME)
private String myDbUsername;
@ -56,6 +57,7 @@ public class TestR4Config extends BaseJavaConfigR4 {
retVal.getTreatBaseUrlsAsLocal().add("http://fhirtest.uhn.ca/baseR4");
retVal.getTreatBaseUrlsAsLocal().add("https://fhirtest.uhn.ca/baseR4");
retVal.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED);
retVal.setCountSearchResultsUpTo(TestR4Config.COUNT_SEARCH_RESULTS_UP_TO);
return retVal;
}

13
src/changes/changes.xml
View File

@ -69,6 +69,19 @@
specify a charset. This log line often showed up any time a server was not supplying
a response, making client logs quite noisy
</action>
<action type="add">
A new configuration item has been added to the JPA server DaoConfig
called
<![CDATA[<code>getCountSearchResultsUpTo()</code>]]>.
This setting governs how many search results the search
coordinator should try to find before returning an initial
search response to the user, which has an effect on whether
the
<![CDATA[<code>Bundle.total</code>]]>
field is always populated in search responses. This has now
been set to 20000 on out public server (fhirtest.uhn.ca)
so most search results should now include a total.
</action>
</release>
<release version="3.0.0" date="2017-09-27">
<action type="add">