honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge c7a6cd9813 into 061390d76b

This commit is contained in:
TipzCM 2024-11-27 08:24:34 -05:00 committed by GitHub
parent 061390d76b c7a6cd9813
commit 23805dca18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 254 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6469-reduce-memory-overhead-for-searches.yaml Normal file
View File

@ -0,0 +1,8 @@
---
type: perf
issue: 6469
title: "Searching for a large number of resources can use a lot of
memory, due to the nature of deduplication of results in memory.
We will instead push this responsibility to the db to save
reduce this overhead.
"

46
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java
View File

@ -200,7 +200,28 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
private String mySearchUuid; private String mySearchUuid;
private int myFetchSize; private int myFetchSize;
private Integer myMaxResultsToFetch; private Integer myMaxResultsToFetch;
/**
* Set of PIDs of results that have already been returned in a search.
*
* Searches use pre-fetch thresholds to avoid returning every result in the db
* (see {@link JpaStorageSettings mySearchPreFetchThresholds}). These threshold values
* dictate the usage of this set.
*
* Results from searches returning *less* than a prefetch threshold are put into this set
* for 2 purposes:
* 1) skipping already seen resources. ie, client requesting next "page" of
* results should skip previously returned results
* 2) deduplication of returned results. ie, searches can return duplicate resources (due to
* sort and filter criteria), so this set will be used to avoid returning duplicate results.
*
* NOTE: if a client requests *more* resources than *all* prefetch thresholds,
* we push the work of "deduplication" to the database. No newly seen resource
* will be stored in this set (to avoid this set exploding in size and the JVM running out memory).
* We will, however, still use it to skip previously seen results.
*/
private Set<JpaPid> myPidSet; private Set<JpaPid> myPidSet;
private boolean myHasNextIteratorQuery = false; private boolean myHasNextIteratorQuery = false;
private RequestPartitionId myRequestPartitionId; private RequestPartitionId myRequestPartitionId;
@ -732,9 +753,12 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
} }
/* /*
* If offset is present, we want deduplicate the results by using GROUP BY * If offset is present, we want to deduplicate the results by using GROUP BY;
* OR
* if the MaxResultsToFetch is null, we are requesting "everything",
* so we'll let the db do the deduplication (instead of in-memory)
*/ */
if (theOffset != null) { if (theOffset != null || (myMaxResultsToFetch == null && !theCountOnlyFlag)) {
queryStack3.addGrouping(); queryStack3.addGrouping();
queryStack3.setUseAggregate(true); queryStack3.setUseAggregate(true);
} }
@ -2429,10 +2453,22 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
if (nextLong != null) { if (nextLong != null) {
JpaPid next = JpaPid.fromId(nextLong); JpaPid next = JpaPid.fromId(nextLong);
if (myPidSet.add(next) && doNotSkipNextPidForEverything()) {
if (!myPidSet.contains(next)) {
if (myMaxResultsToFetch != null) {
/*
* We only add to the map if we aren't fetching "everything";
* otherwise, we let the de-duplication happen in the database
* (see createChunkedQueryNormalSearch above), because it
* saves memory that way.
*/
myPidSet.add(next);
}
if (doNotSkipNextPidForEverything()) {
myNext = next; myNext = next;
myNonSkipCount++; myNonSkipCount++;
break; break;
}
} else { } else {
mySkipCount++; mySkipCount++;
} }
@ -2475,7 +2511,11 @@ public class SearchBuilder implements ISearchBuilder<JpaPid> {
} }
} }
if (myMaxResultsToFetch == null) {
mySearchRuntimeDetails.setFoundIndexMatchesCount(myNonSkipCount);
} else {
mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
}
} finally { } finally {
// search finished - fire hooks // search finished - fire hooks

31
hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java
View File

@ -1,10 +1,5 @@
package ca.uhn.fhir.jpa.provider.dstu3; package ca.uhn.fhir.jpa.provider.dstu3;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.data.ISearchDao;
@ -30,9 +25,7 @@ import ca.uhn.fhir.rest.gclient.StringClientParam;
import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum; import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.StringAndListParam; import ca.uhn.fhir.rest.server.IPagingProvider;
import ca.uhn.fhir.rest.param.StringOrListParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
@ -133,6 +126,7 @@ import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
@ -161,7 +155,11 @@ import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;
@ -184,10 +182,8 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null);
mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE);
mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false);
} }
@Test @Test
public void testSearchBySourceTransactionId() { public void testSearchBySourceTransactionId() {
@ -1485,6 +1481,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
ourLog.info(ids.toString()); ourLog.info(ids.toString());
} }
@Test @Test
public void testEverythingInstanceWithContentFilter() { public void testEverythingInstanceWithContentFilter() {
Patient pt1 = new Patient(); Patient pt1 = new Patient();
@ -1503,6 +1500,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
dev2.setManufacturer("Some Manufacturer 2"); dev2.setManufacturer("Some Manufacturer 2");
myDeviceDao.create(dev2, mySrd).getId().toUnqualifiedVersionless(); myDeviceDao.create(dev2, mySrd).getId().toUnqualifiedVersionless();
// create an observation that links to Dev1 and Patient1
Observation obs1 = new Observation(); Observation obs1 = new Observation();
obs1.getText().setDivAsString("<div>OBSTEXT1</div>"); obs1.getText().setDivAsString("<div>OBSTEXT1</div>");
obs1.getSubject().setReferenceElement(ptId1); obs1.getSubject().setReferenceElement(ptId1);
@ -1524,13 +1522,9 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
IIdType obsId3 = myObservationDao.create(obs3, mySrd).getId().toUnqualifiedVersionless(); IIdType obsId3 = myObservationDao.create(obs3, mySrd).getId().toUnqualifiedVersionless();
List<IIdType> actual; List<IIdType> actual;
StringAndListParam param;
ourLog.info("Pt1:{} Pt2:{} Obs1:{} Obs2:{} Obs3:{}", ptId1.getIdPart(), ptId2.getIdPart(), obsId1.getIdPart(), obsId2.getIdPart(), obsId3.getIdPart()); ourLog.info("Pt1:{} Pt2:{} Obs1:{} Obs2:{} Obs3:{}", ptId1.getIdPart(), ptId2.getIdPart(), obsId1.getIdPart(), obsId2.getIdPart(), obsId3.getIdPart());
param = new StringAndListParam();
param.addAnd(new StringOrListParam().addOr(new StringParam("obsvalue1")));
//@formatter:off //@formatter:off
Parameters response = myClient Parameters response = myClient
.operation() .operation()
@ -1542,7 +1536,6 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
actual = toUnqualifiedVersionlessIds((Bundle) response.getParameter().get(0).getResource()); actual = toUnqualifiedVersionlessIds((Bundle) response.getParameter().get(0).getResource());
assertThat(actual).containsExactlyInAnyOrder(ptId1, obsId1, devId1); assertThat(actual).containsExactlyInAnyOrder(ptId1, obsId1, devId1);
} }
/** /**
@ -2497,6 +2490,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
assertEquals(1, resp.getTotal()); assertEquals(1, resp.getTotal());
} }
// @Disabled
@Test @Test
public void testMetaOperations() { public void testMetaOperations() {
String methodName = "testMetaOperations"; String methodName = "testMetaOperations";
@ -2627,6 +2621,8 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
@Test @Test
public void testEverythingWithNoPagingProvider() { public void testEverythingWithNoPagingProvider() {
IPagingProvider pagingProvider = myRestServer.getPagingProvider();
try {
myRestServer.setPagingProvider(null); myRestServer.setPagingProvider(null);
Patient p = new Patient(); Patient p = new Patient();
@ -2656,6 +2652,9 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
assertThat(response.getEntry()).hasSize(10); assertThat(response.getEntry()).hasSize(10);
assertNull(response.getTotalElement().getValue()); assertNull(response.getTotalElement().getValue());
assertNull(response.getLink("next")); assertNull(response.getLink("next"));
} finally {
myRestServer.setPagingProvider(pagingProvider);
}
} }
@Test @Test

17
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java
View File

@ -165,7 +165,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue());
assertThat(myCaptureQueriesListener.getSelectQueries().stream().map(t -> t.getSql(true, false)).toList()).contains( assertThat(myCaptureQueriesListener.getSelectQueries().stream().map(t -> t.getSql(true, false)).toList()).contains(
"SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '-2634469377090377342')" "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '-2634469377090377342') GROUP BY t0.RES_ID"
); );
logCapturedMessages(); logCapturedMessages();
@ -291,7 +291,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue());
String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false);
String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND (((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_LOW_DATE_ORDINAL <= '20210202')) AND ((t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL >= '20210202')))))"; String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND (((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_LOW_DATE_ORDINAL <= '20210202')) AND ((t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL >= '20210202'))))) GROUP BY t1.RES_ID";
assertEquals(expected, sql); assertEquals(expected, sql);
logCapturedMessages(); logCapturedMessages();
@ -323,7 +323,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue());
String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false);
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '7545664593829342272') AND ((t1.HASH_NORM_PREFIX = '6206712800146298788') AND (t1.SP_VALUE_NORMALIZED LIKE 'JAY%')))"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '7545664593829342272') AND ((t1.HASH_NORM_PREFIX = '6206712800146298788') AND (t1.SP_VALUE_NORMALIZED LIKE 'JAY%'))) GROUP BY t0.RES_ID";
assertEquals(expected, sql); assertEquals(expected, sql);
logCapturedMessages(); logCapturedMessages();
@ -363,7 +363,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
myCaptureQueriesListener.logSelectQueries(); myCaptureQueriesListener.logSelectQueries();
assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue()); assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue());
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '7196518367857292879')"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '7196518367857292879') GROUP BY t0.RES_ID";
assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false));
logCapturedMessages(); logCapturedMessages();
@ -398,7 +398,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
myCaptureQueriesListener.logSelectQueries(); myCaptureQueriesListener.logSelectQueries();
assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue()); assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue());
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '2591238402961312979')"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '2591238402961312979') GROUP BY t0.RES_ID";
assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false));
} }
@ -461,9 +461,8 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
myCaptureQueriesListener.logSelectQueries(); myCaptureQueriesListener.logSelectQueries();
assertThat(actual).contains("Patient/A"); assertThat(actual).contains("Patient/A");
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_IDX_CMB_TOK_NU t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND (t1.HASH_COMPLETE = '-8088946700286918311'))"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_IDX_CMB_TOK_NU t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND (t1.HASH_COMPLETE = '-8088946700286918311')) GROUP BY t0.RES_ID";
assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false));
} }
/** /**
@ -497,7 +496,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
myCaptureQueriesListener.logSelectQueries(); myCaptureQueriesListener.logSelectQueries();
assertThat(actual).contains("Patient/A"); assertThat(actual).contains("Patient/A");
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND ((t1.HASH_NORM_PREFIX = '-3664262414674370905') AND (t1.SP_VALUE_NORMALIZED LIKE 'JONES%')))"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND ((t1.HASH_NORM_PREFIX = '-3664262414674370905') AND (t1.SP_VALUE_NORMALIZED LIKE 'JONES%'))) GROUP BY t0.RES_ID";
assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false));
} }
@ -520,7 +519,7 @@ public class FhirResourceDaoR4ComboNonUniqueParamTest extends BaseComboParamsR4T
myCaptureQueriesListener.logSelectQueries(); myCaptureQueriesListener.logSelectQueries();
assertThat(actual).contains("Observation/O1"); assertThat(actual).contains("Observation/O1");
String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE IN ('2445648980345828396','-6884698528022589694','-8034948665712960724') )"; String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE IN ('2445648980345828396','-6884698528022589694','-8034948665712960724') ) GROUP BY t0.RES_ID";
assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false));
logCapturedMessages(); logCapturedMessages();

69
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java
View File

@ -15,6 +15,7 @@ import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
import ca.uhn.fhir.jpa.util.QueryParameterUtils; import ca.uhn.fhir.jpa.util.QueryParameterUtils;
import ca.uhn.fhir.jpa.util.SqlQuery;
import ca.uhn.fhir.rest.api.SearchTotalModeEnum; import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.api.SummaryEnum; import ca.uhn.fhir.rest.api.SummaryEnum;
@ -33,6 +34,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.BaseResource;
import org.hl7.fhir.r4.model.BodyStructure; import org.hl7.fhir.r4.model.BodyStructure;
import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Coding;
@ -65,14 +67,18 @@ import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.leftPad; import static org.apache.commons.lang3.StringUtils.leftPad;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.fail;
import static org.awaitility.Awaitility.await; import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;
@ -556,6 +562,67 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test {
} }
/**
* We want to use the db to deduplicate in the "fetch everything"
* case because it's more memory efficient.
*/
@Test
public void search_whenPastPreFetchLimit_usesDBToDeduplicate() {
// setup
IBundleProvider results;
List<SqlQuery> queries;
List<String> ids;
create200Patients();
myCaptureQueriesListener.clear();
// set the prefetch thresholds low so we don't need to
// search for tons of resources
myStorageSettings.setSearchPreFetchThresholds(List.of(5, 10, -1));
// basic search map
SearchParameterMap map = new SearchParameterMap();
map.setSort(new SortSpec(BaseResource.SP_RES_LAST_UPDATED));
// test
results = myPatientDao.search(map, null);
String uuid = results.getUuid();
ourLog.debug("** Search returned UUID: {}", uuid);
assertNotNull(results);
ids = toUnqualifiedVersionlessIdValues(results, 0, 9, true);
assertEquals(9, ids.size());
// first search was < 10 (our max pre-fetch value); so we should
// expect no "group by" queries (we deduplicate in memory)
queries = findGroupByQueries();
assertTrue(queries.isEmpty());
myCaptureQueriesListener.clear();
ids = toUnqualifiedVersionlessIdValues(results, 10, 100, true);
assertEquals(90, ids.size());
// we are now requesting > 10 results, meaning we should be using the
// database to deduplicate any values not fetched yet;
// so we *do* expect to see a "group by" query
queries = findGroupByQueries();
assertFalse(queries.isEmpty());
assertEquals(1, queries.size());
SqlQuery query = queries.get(0);
String sql = query.getSql(true, false);
// we expect a "GROUP BY t0.RES_ID" (but we'll be ambiguous about the table
// name, just in case)
Pattern p = Pattern.compile("GROUP BY .+\\.RES_ID");
Matcher m = p.matcher(sql);
assertTrue(m.find());
}
private List<SqlQuery> findGroupByQueries() {
List<SqlQuery> queries = myCaptureQueriesListener.getSelectQueries();
queries = queries.stream().filter(q -> q.getSql(true, false).toLowerCase().contains("group by"))
.collect(Collectors.toList());
return queries;
}
@Test @Test
public void testFetchMoreThanFirstPageSizeInFirstPage() { public void testFetchMoreThanFirstPageSizeInFirstPage() {
create200Patients(); create200Patients();

90
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java
View File

@ -1289,7 +1289,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
// Only the read columns should be used, no criteria use partition // Only the read columns should be used, no criteria use partition
assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'");
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
} }
// Read in null Partition // Read in null Partition
@ -1342,7 +1342,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
// Only the read columns should be used, no criteria use partition // Only the read columns should be used, no criteria use partition
assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'");
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // If this switches to 2 that would be fine assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(4); // If this switches to 2 that would be fine
} }
// Read in null Partition // Read in null Partition
@ -1397,7 +1397,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
// Only the read columns should be used, no criteria use partition // Only the read columns should be used, no criteria use partition
assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'");
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
} }
// Read in null Partition // Read in null Partition
@ -1475,10 +1475,8 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
IBundleProvider searchOutcome = myPatientDao.search(map, mySrd); IBundleProvider searchOutcome = myPatientDao.search(map, mySrd);
assertEquals(0, searchOutcome.size()); assertEquals(0, searchOutcome.size());
} }
} }
@Test @Test
public void testSearch_MissingParamString_SearchAllPartitions() { public void testSearch_MissingParamString_SearchAllPartitions() {
myPartitionSettings.setIncludePartitionInSearchHashes(false); myPartitionSettings.setIncludePartitionInSearchHashes(false);
@ -1500,7 +1498,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'true'")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'true'"));
} }
@ -1517,7 +1515,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'false'")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'false'"));
} }
} }
@ -1626,7 +1624,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")); assertEquals(1, StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT"));
assertEquals(1, StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")); assertEquals(1, StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'"));
} }
@ -1653,7 +1651,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true));
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '-3438137196820602023'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '-3438137196820602023'")).as(searchSql).isEqualTo(1);
@ -1681,7 +1679,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true));
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1);
@ -1707,7 +1705,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1);
@ -1732,7 +1730,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
} }
@Test @Test
@ -1752,7 +1750,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
} }
@Test @Test
@ -1822,7 +1820,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date OR param // Date OR param
@ -1838,7 +1836,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param // Date AND param
@ -1854,7 +1852,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam // DateRangeParam
@ -1870,14 +1868,12 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
} }
@Test @Test
public void testSearch_DateParam_SearchSpecificPartitions() { public void testSearch_DateParam_SearchSpecificPartitions() {
myPartitionSettings.setIncludePartitionInSearchHashes(false); myPartitionSettings.setIncludePartitionInSearchHashes(false);
@ -1907,7 +1903,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(2);
// Date OR param // Date OR param
@ -1923,7 +1919,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param // Date AND param
@ -1939,7 +1935,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam // DateRangeParam
@ -1955,7 +1951,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
@ -1987,7 +1983,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date OR param // Date OR param
@ -2003,7 +1999,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// Date AND param // Date AND param
@ -2019,7 +2015,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
// DateRangeParam // DateRangeParam
@ -2035,7 +2031,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
// NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH"));
@ -2105,7 +2101,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'");
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
} }
@ -2129,7 +2125,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED"));
} }
@ -2152,7 +2148,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
searchSql = searchSql.toUpperCase(); searchSql = searchSql.toUpperCase();
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED"));
} }
@ -2176,7 +2172,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED"));
} }
@ -2209,7 +2205,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true));
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
assertThat(searchSql).contains("PARTITION_ID IN ('1','2')"); assertThat(searchSql).contains("PARTITION_ID IN ('1','2')");
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
} }
// Match two partitions including null // Match two partitions including null
@ -2226,7 +2222,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
assertThat(searchSql).contains("PARTITION_ID IS NULL"); assertThat(searchSql).contains("PARTITION_ID IS NULL");
assertThat(searchSql).contains("PARTITION_ID = '1'"); assertThat(searchSql).contains("PARTITION_ID = '1'");
assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "PARTITION_ID"));
} }
} }
@ -2287,7 +2283,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
searchSql = searchSql.toUpperCase(); searchSql = searchSql.toUpperCase();
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED"));
} }
@ -2315,7 +2311,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED"));
} }
@ -2372,7 +2368,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
// And with another param // And with another param
@ -2389,7 +2385,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2);
assertThat(StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, ".HASH_SYS_AND_VALUE =")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, ".HASH_SYS_AND_VALUE =")).as(searchSql).isEqualTo(1);
@ -2413,7 +2409,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
@ -2441,7 +2437,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
} }
@ -2463,7 +2459,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
} }
@ -2489,7 +2485,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "JOIN")); assertEquals(2, StringUtils.countMatches(searchSql, "JOIN"));
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
} }
@ -2514,7 +2510,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
} }
@ -2539,7 +2535,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'"));
} }
@ -2687,7 +2683,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
// Same query, different partition // Same query, different partition
addReadPartition(2); addReadPartition(2);
@ -2724,7 +2720,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
assertEquals(1, StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL"));
assertEquals(1, StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'"));
assertEquals(1, StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'"));
assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID"));
// Same query, different partition // Same query, different partition
addReadPartition(2); addReadPartition(2);
@ -2759,7 +2755,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true));
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false);
assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID = '1'")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
// Same query, different partition // Same query, different partition
addReadPartition(2); addReadPartition(2);
@ -2829,7 +2825,7 @@ public class PartitioningSqlR4Test extends BasePartitioningR4Test {
String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true);
ourLog.info("Search SQL:\n{}", searchSql); ourLog.info("Search SQL:\n{}", searchSql);
assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1);
assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3);
// Same query, different partition // Same query, different partition
addReadPartition(2); addReadPartition(2);

5
hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java
View File

@ -188,8 +188,9 @@ public class JpaStorageSettings extends StorageSettings {
// start with a tiny number so our first page always loads quickly. // start with a tiny number so our first page always loads quickly.
// If they fetch the second page, fetch more. // If they fetch the second page, fetch more.
// Use prime sizes to avoid empty next links. // we'll only fetch (by default) up to 1 million records, because after that, deduplication in local memory is
private List<Integer> mySearchPreFetchThresholds = Arrays.asList(13, 503, 2003, -1); // prohibitive
private List<Integer> mySearchPreFetchThresholds = Arrays.asList(13, 503, 2003, 1000003, -1);
private List<WarmCacheEntry> myWarmCacheEntries = new ArrayList<>(); private List<WarmCacheEntry> myWarmCacheEntries = new ArrayList<>();
private boolean myEnforceReferenceTargetTypes = true; private boolean myEnforceReferenceTargetTypes = true;
private ClientIdStrategyEnum myResourceClientIdStrategy = ClientIdStrategyEnum.ALPHANUMERIC; private ClientIdStrategyEnum myResourceClientIdStrategy = ClientIdStrategyEnum.ALPHANUMERIC;