From d40c5fa5e3ad3b93ed2552209371603954a9eda9 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 16 Jun 2017 09:41:45 -0400 Subject: [PATCH] Add DAO setting to specify maximum query size --- .../fhir/jpa/dao/BaseHapiFhirSystemDao.java | 13 +++- .../java/ca/uhn/fhir/jpa/dao/DaoConfig.java | 73 ++++++++++++++----- .../FhirResourceDaoDstu3SearchNoFtTest.java | 20 ++++- src/changes/changes.xml | 7 ++ 4 files changed, 88 insertions(+), 25 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java index 5ed0a47b7f0..3e0ae8af35b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java @@ -104,7 +104,7 @@ public abstract class BaseHapiFhirSystemDao extends BaseHapiFhirDao q = myEntityManager.createQuery("SELECT t.myId FROM ResourceTable t WHERE t.myIndexStatus IS NULL", Long.class); - ourLog.info("Beginning indexing query with maximum {}", maxResult); + ourLog.debug("Beginning indexing query with maximum {}", maxResult); q.setMaxResults(maxResult); Collection resources = q.getResultList(); @@ -145,9 +145,14 @@ public abstract class BaseHapiFhirSystemDao extends BaseHapiFhirDao 0 ? (delay / count) : 0; - ourLog.info("Indexed {} resources in {}ms - Avg {}ms / resource", new Object[] { count, delay, avg }); - + long avg; + if (count > 0) { + avg = (delay / count); + ourLog.info("Indexed {} resources in {}ms - Avg {}ms / resource", new Object[] { count, delay, avg }); + } else { + ourLog.debug("Indexed 0 resources in {}ms", delay); + } + return count; } }); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java index 067a07e32a1..56aee54fbd7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java @@ -64,21 +64,21 @@ public class DaoConfig { */ public static final Long DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS = DateUtils.MILLIS_PER_MINUTE; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private boolean myAllowExternalReferences = false; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private boolean myAllowInlineMatchUrlReferences = true; private boolean myAllowMultipleDelete; private boolean myDefaultSearchParamsCanBeOverridden = false; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private int myDeferIndexingForCodesystemsOfSize = 2000; private boolean myDeleteStaleSearches = true; @@ -87,27 +87,34 @@ public class DaoConfig { private boolean myEnforceReferentialIntegrityOnWrite = true; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR; - private int myHardTagListLimit = 1000; + /** + * update setter javadoc if default changes + */ + private Integer myFetchSizeDefaultMaximum = null; + private int myHardTagListLimit = 1000; private int myIncludeLimit = 2000; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private boolean myIndexContainedResources = true; private List myInterceptors; - // *** - // update setter javadoc if default changes - // *** + /** + * update setter javadoc if default changes + */ private int myMaximumExpansionSize = 5000; private int myMaximumSearchResultCountInTransaction = DEFAULT_MAXIMUM_SEARCH_RESULT_COUNT_IN_TRANSACTION; private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC; private Long myReuseCachedSearchResultsForMillis = DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS; private boolean mySchedulingDisabled; private boolean mySubscriptionEnabled; + /** + * update setter javadoc if default changes + */ private long mySubscriptionPollDelay = 1000; private Long mySubscriptionPurgeInactiveAfterMillis; private boolean mySuppressUpdatesWithNoChange = true; @@ -160,6 +167,20 @@ public class DaoConfig { return myExpireSearchResultsAfterMillis; } + /** + * Gets the default maximum number of results to load in a query. + *

+ * For example, if the database has a million Patient resources in it, and + * the client requests GET /Patient, if this value is set + * to a non-null value (default is null) only this number + * of results will be fetched. Setting this value appropriately + * can be useful to improve performance in some situations. + *

+ */ + public Integer getFetchSizeDefaultMaximum() { + return myFetchSizeDefaultMaximum; + } + /** * Gets the maximum number of results to return in a GetTags query (DSTU1 only) */ @@ -549,6 +570,20 @@ public class DaoConfig { myExpireSearchResultsAfterMillis = theExpireSearchResultsAfterMillis; } + /** + * Gets the default maximum number of results to load in a query. + *

+ * For example, if the database has a million Patient resources in it, and + * the client requests GET /Patient, if this value is set + * to a non-null value (default is null) only this number + * of results will be fetched. Setting this value appropriately + * can be useful to improve performance in some situations. + *

+ */ + public void setFetchSizeDefaultMaximum(Integer theFetchSizeDefaultMaximum) { + myFetchSizeDefaultMaximum = theFetchSizeDefaultMaximum; + } + /** * Do not call this method, it exists only for legacy reasons. It * will be removed in a future version. Configure the page size on your diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java index 152f6479d89..e9cb2b3080c 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java @@ -82,8 +82,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; -import ca.uhn.fhir.jpa.dao.SearchParameterMap; +import ca.uhn.fhir.jpa.dao.*; import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum; import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate; import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber; @@ -130,6 +129,7 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { @Before public void beforeDisableResultReuse() { myDaoConfig.setReuseCachedSearchResultsForMillis(null); + myDaoConfig.setFetchSizeDefaultMaximum(new DaoConfig().getFetchSizeDefaultMaximum()); } /** @@ -1852,6 +1852,22 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { assertThat(patients, not(containsInAnyOrder(pid2))); } + @Test + public void testSearchWithFetchSizeDefaultMaximum() { + myDaoConfig.setFetchSizeDefaultMaximum(5); + + for (int i = 0; i < 10; i++) { + Patient p = new Patient(); + p.addName().setFamily("PT" + i); + myPatientDao.create(p); + } + + SearchParameterMap map = new SearchParameterMap(); + map.setLoadSynchronous(true); + IBundleProvider values = myPatientDao.search(map); + assertEquals(5, values.size().intValue()); + assertEquals(5, values.getResources(0, 1000).size()); + } @Test public void testSearchStringParam() throws Exception { diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 93ce40e6fa1..8c12874f002 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -28,6 +28,13 @@ Optimize queries in JPA server remove a few redundant select columns when performing searches. This provides a slight speed increase in some cases. + + Add configuration to JPA server DaoConfig that allows a maximum + number of search results to be specified. Queries will never return + more than this number, which can be good for avoiding accidental + performance problems in situations where lare queries should not be + needed +