honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge history table prefetch with resource table.

This commit is contained in:
Michael Buckley 2024-04-04 15:30:38 -04:00
parent fcf0a85584
commit 6958f0eb43
3 changed files with 66 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5824-fix-history-prefetch.yaml Normal file
View File

@ -0,0 +1,4 @@
---
type: fix
issue: 5824
title: "We now avoid a query during reindex and transaction processing that was very slow on Sql Server."

81
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java
View File

@ -38,7 +38,6 @@ import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory;
import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
import ca.uhn.fhir.jpa.util.QueryChunker;
import ca.uhn.fhir.jpa.util.ResourceCountCache;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
@ -51,12 +50,9 @@ import jakarta.annotation.Nullable;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import jakarta.persistence.PersistenceContextType;
import jakarta.persistence.Query;
import jakarta.persistence.TypedQuery;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaQuery;
import jakarta.persistence.criteria.JoinType;
import jakarta.persistence.criteria.Predicate;
import jakarta.persistence.criteria.Root;
import org.hl7.fhir.instance.model.api.IBaseBundle;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
@ -206,8 +202,25 @@ public abstract class BaseHapiFhirSystemDao<T extends IBaseBundle, MT> extends B
new QueryChunker<Long>()
.chunk(
ids,
nextChunk ->
loadedResourceTableEntries.addAll(myResourceTableDao.findAllById(nextChunk)));
nextChunk -> {
//List<ResourceTable> allById = myResourceTableDao.findAllById(nextChunk);
Query query = myEntityManager.createQuery(
"select r, h FROM ResourceTable r " +
" LEFT JOIN fetch ResourceHistoryTable h on r.myVersion = h.myResourceVersion and r.id = h.myResourceId" +
" left join fetch h.myProvenance" +
" WHERE r.myId IN ( :IDS )");
query.setParameter("IDS", ids);
@SuppressWarnings("unchecked")
List<Object[]> allById = query.getResultList();
for (Object[] nextPair : allById) {
ResourceTable r = (ResourceTable) nextPair[0];
ResourceHistoryTable h = (ResourceHistoryTable) nextPair[1];
r.setCurrentVersionEntity(h);
loadedResourceTableEntries.add(r);
}
});
List<Long> entityIds;
@ -269,33 +282,33 @@ public abstract class BaseHapiFhirSystemDao<T extends IBaseBundle, MT> extends B
}
}
new QueryChunker<ResourceTable>()
.chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> {
Map<Long, ResourceTable> entities =
entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t));
CriteriaBuilder b = myEntityManager.getCriteriaBuilder();
CriteriaQuery<ResourceHistoryTable> q = b.createQuery(ResourceHistoryTable.class);
Root<ResourceHistoryTable> from = q.from(ResourceHistoryTable.class);
from.fetch("myProvenance", JoinType.LEFT);
List<Predicate> orPredicates = new ArrayList<>();
for (ResourceTable next : entries) {
Predicate resId = b.equal(from.get("myResourceId"), next.getId());
Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion());
orPredicates.add(b.and(resId, resVer));
}
q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY)));
List<ResourceHistoryTable> resultList =
myEntityManager.createQuery(q).getResultList();
for (ResourceHistoryTable next : resultList) {
ResourceTable nextEntity = entities.get(next.getResourceId());
if (nextEntity != null) {
nextEntity.setCurrentVersionEntity(next);
}
}
});
// new QueryChunker<ResourceTable>()
// .chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> {
// Map<Long, ResourceTable> entities =
// entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t));
//
// CriteriaBuilder b = myEntityManager.getCriteriaBuilder();
// CriteriaQuery<ResourceHistoryTable> q = b.createQuery(ResourceHistoryTable.class);
// Root<ResourceHistoryTable> from = q.from(ResourceHistoryTable.class);
//
// from.fetch("myProvenance", JoinType.LEFT);
//
// List<Predicate> orPredicates = new ArrayList<>();
// for (ResourceTable next : entries) {
// Predicate resId = b.equal(from.get("myResourceId"), next.getId());
// Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion());
// orPredicates.add(b.and(resId, resVer));
// }
// q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY)));
// List<ResourceHistoryTable> resultList =
// myEntityManager.createQuery(q).getResultList();
// for (ResourceHistoryTable next : resultList) {
// ResourceTable nextEntity = entities.get(next.getResourceId());
// if (nextEntity != null) {
// nextEntity.setCurrentVersionEntity(next);
// }
// }
// });
}
});
}

30
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java
View File

@ -827,7 +827,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?active=true", new SystemRequestDetails());
// Validate
assertEquals(13, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(12, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(10, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
assertEquals(10, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
assertEquals(30, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
@ -1026,10 +1026,10 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
@ParameterizedTest
@CsvSource({
// OptimisticLock OptimizeMode ExpectedSelect ExpectedUpdate
" false, CURRENT_VERSION, 2, 0",
" true, CURRENT_VERSION, 12, 0",
" false, ALL_VERSIONS, 12, 0",
" true, ALL_VERSIONS, 22, 0",
" false, CURRENT_VERSION, 1, 0",
" true, CURRENT_VERSION, 11, 0",
" false, ALL_VERSIONS, 11, 0",
" true, ALL_VERSIONS, 21, 0",
})
public void testReindexJob_OptimizeStorage(boolean theOptimisticLock, ReindexParameters.OptimizeStorageModeEnum theOptimizeStorageModeEnum, int theExpectedSelectCount, int theExpectedUpdateCount) {
// Setup
@ -1841,7 +1841,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
assertEquals(4, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -1857,7 +1857,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
assertEquals(4, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -1927,7 +1927,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(9, myCaptureQueriesListener.countSelectQueries());
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(7, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -1943,7 +1943,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(7, myCaptureQueriesListener.countSelectQueries());
assertEquals(6, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(5, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -2239,7 +2239,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(9, myCaptureQueriesListener.countSelectQueries());
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -2256,7 +2256,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
assertEquals(7, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -2271,7 +2271,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
outcome = mySystemDao.transaction(mySrd, input.get());
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
myCaptureQueriesListener.logSelectQueries();
assertEquals(6, myCaptureQueriesListener.countSelectQueries());
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
myCaptureQueriesListener.logInsertQueries();
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
myCaptureQueriesListener.logUpdateQueries();
@ -3365,7 +3365,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
myCaptureQueriesListener.clear();
Bundle outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
myCaptureQueriesListener.logInsertQueries();
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
@ -3388,7 +3388,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
myCaptureQueriesListener.clear();
outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
myCaptureQueriesListener.logInsertQueries();
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
@ -3449,7 +3449,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
myCaptureQueriesListener.clear();
mySystemDao.transaction(new SystemRequestDetails(), loadResourceFromClasspath(Bundle.class, "r4/transaction-perf-bundle-smallchanges.json"));
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
assertEquals(5, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());