Merge history table prefetch with resource table.
This commit is contained in:
parent
fcf0a85584
commit
6958f0eb43
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
type: fix
|
||||||
|
issue: 5824
|
||||||
|
title: "We now avoid a query during reindex and transaction processing that was very slow on Sql Server."
|
|
@ -38,7 +38,6 @@ import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
|
||||||
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
|
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
|
||||||
import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
|
import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
|
||||||
import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory;
|
import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory;
|
||||||
import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
|
|
||||||
import ca.uhn.fhir.jpa.util.QueryChunker;
|
import ca.uhn.fhir.jpa.util.QueryChunker;
|
||||||
import ca.uhn.fhir.jpa.util.ResourceCountCache;
|
import ca.uhn.fhir.jpa.util.ResourceCountCache;
|
||||||
import ca.uhn.fhir.rest.api.server.IBundleProvider;
|
import ca.uhn.fhir.rest.api.server.IBundleProvider;
|
||||||
|
@ -51,12 +50,9 @@ import jakarta.annotation.Nullable;
|
||||||
import jakarta.persistence.EntityManager;
|
import jakarta.persistence.EntityManager;
|
||||||
import jakarta.persistence.PersistenceContext;
|
import jakarta.persistence.PersistenceContext;
|
||||||
import jakarta.persistence.PersistenceContextType;
|
import jakarta.persistence.PersistenceContextType;
|
||||||
|
import jakarta.persistence.Query;
|
||||||
import jakarta.persistence.TypedQuery;
|
import jakarta.persistence.TypedQuery;
|
||||||
import jakarta.persistence.criteria.CriteriaBuilder;
|
|
||||||
import jakarta.persistence.criteria.CriteriaQuery;
|
|
||||||
import jakarta.persistence.criteria.JoinType;
|
|
||||||
import jakarta.persistence.criteria.Predicate;
|
import jakarta.persistence.criteria.Predicate;
|
||||||
import jakarta.persistence.criteria.Root;
|
|
||||||
import org.hl7.fhir.instance.model.api.IBaseBundle;
|
import org.hl7.fhir.instance.model.api.IBaseBundle;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.context.ApplicationContext;
|
import org.springframework.context.ApplicationContext;
|
||||||
|
@ -206,8 +202,25 @@ public abstract class BaseHapiFhirSystemDao<T extends IBaseBundle, MT> extends B
|
||||||
new QueryChunker<Long>()
|
new QueryChunker<Long>()
|
||||||
.chunk(
|
.chunk(
|
||||||
ids,
|
ids,
|
||||||
nextChunk ->
|
nextChunk -> {
|
||||||
loadedResourceTableEntries.addAll(myResourceTableDao.findAllById(nextChunk)));
|
//List<ResourceTable> allById = myResourceTableDao.findAllById(nextChunk);
|
||||||
|
Query query = myEntityManager.createQuery(
|
||||||
|
"select r, h FROM ResourceTable r " +
|
||||||
|
" LEFT JOIN fetch ResourceHistoryTable h on r.myVersion = h.myResourceVersion and r.id = h.myResourceId" +
|
||||||
|
" left join fetch h.myProvenance" +
|
||||||
|
" WHERE r.myId IN ( :IDS )");
|
||||||
|
query.setParameter("IDS", ids);
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<Object[]> allById = query.getResultList();
|
||||||
|
|
||||||
|
for (Object[] nextPair : allById) {
|
||||||
|
ResourceTable r = (ResourceTable) nextPair[0];
|
||||||
|
ResourceHistoryTable h = (ResourceHistoryTable) nextPair[1];
|
||||||
|
r.setCurrentVersionEntity(h);
|
||||||
|
loadedResourceTableEntries.add(r);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
List<Long> entityIds;
|
List<Long> entityIds;
|
||||||
|
|
||||||
|
@ -269,33 +282,33 @@ public abstract class BaseHapiFhirSystemDao<T extends IBaseBundle, MT> extends B
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
new QueryChunker<ResourceTable>()
|
// new QueryChunker<ResourceTable>()
|
||||||
.chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> {
|
// .chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> {
|
||||||
Map<Long, ResourceTable> entities =
|
// Map<Long, ResourceTable> entities =
|
||||||
entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t));
|
// entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t));
|
||||||
|
//
|
||||||
CriteriaBuilder b = myEntityManager.getCriteriaBuilder();
|
// CriteriaBuilder b = myEntityManager.getCriteriaBuilder();
|
||||||
CriteriaQuery<ResourceHistoryTable> q = b.createQuery(ResourceHistoryTable.class);
|
// CriteriaQuery<ResourceHistoryTable> q = b.createQuery(ResourceHistoryTable.class);
|
||||||
Root<ResourceHistoryTable> from = q.from(ResourceHistoryTable.class);
|
// Root<ResourceHistoryTable> from = q.from(ResourceHistoryTable.class);
|
||||||
|
//
|
||||||
from.fetch("myProvenance", JoinType.LEFT);
|
// from.fetch("myProvenance", JoinType.LEFT);
|
||||||
|
//
|
||||||
List<Predicate> orPredicates = new ArrayList<>();
|
// List<Predicate> orPredicates = new ArrayList<>();
|
||||||
for (ResourceTable next : entries) {
|
// for (ResourceTable next : entries) {
|
||||||
Predicate resId = b.equal(from.get("myResourceId"), next.getId());
|
// Predicate resId = b.equal(from.get("myResourceId"), next.getId());
|
||||||
Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion());
|
// Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion());
|
||||||
orPredicates.add(b.and(resId, resVer));
|
// orPredicates.add(b.and(resId, resVer));
|
||||||
}
|
// }
|
||||||
q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY)));
|
// q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY)));
|
||||||
List<ResourceHistoryTable> resultList =
|
// List<ResourceHistoryTable> resultList =
|
||||||
myEntityManager.createQuery(q).getResultList();
|
// myEntityManager.createQuery(q).getResultList();
|
||||||
for (ResourceHistoryTable next : resultList) {
|
// for (ResourceHistoryTable next : resultList) {
|
||||||
ResourceTable nextEntity = entities.get(next.getResourceId());
|
// ResourceTable nextEntity = entities.get(next.getResourceId());
|
||||||
if (nextEntity != null) {
|
// if (nextEntity != null) {
|
||||||
nextEntity.setCurrentVersionEntity(next);
|
// nextEntity.setCurrentVersionEntity(next);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
});
|
// });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -827,7 +827,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?active=true", new SystemRequestDetails());
|
DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?active=true", new SystemRequestDetails());
|
||||||
|
|
||||||
// Validate
|
// Validate
|
||||||
assertEquals(13, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
assertEquals(12, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
||||||
assertEquals(10, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
assertEquals(10, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
||||||
assertEquals(10, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
assertEquals(10, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
||||||
assertEquals(30, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
|
assertEquals(30, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
|
||||||
|
@ -1026,10 +1026,10 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
@ParameterizedTest
|
@ParameterizedTest
|
||||||
@CsvSource({
|
@CsvSource({
|
||||||
// OptimisticLock OptimizeMode ExpectedSelect ExpectedUpdate
|
// OptimisticLock OptimizeMode ExpectedSelect ExpectedUpdate
|
||||||
" false, CURRENT_VERSION, 2, 0",
|
" false, CURRENT_VERSION, 1, 0",
|
||||||
" true, CURRENT_VERSION, 12, 0",
|
" true, CURRENT_VERSION, 11, 0",
|
||||||
" false, ALL_VERSIONS, 12, 0",
|
" false, ALL_VERSIONS, 11, 0",
|
||||||
" true, ALL_VERSIONS, 22, 0",
|
" true, ALL_VERSIONS, 21, 0",
|
||||||
})
|
})
|
||||||
public void testReindexJob_OptimizeStorage(boolean theOptimisticLock, ReindexParameters.OptimizeStorageModeEnum theOptimizeStorageModeEnum, int theExpectedSelectCount, int theExpectedUpdateCount) {
|
public void testReindexJob_OptimizeStorage(boolean theOptimisticLock, ReindexParameters.OptimizeStorageModeEnum theOptimizeStorageModeEnum, int theExpectedSelectCount, int theExpectedUpdateCount) {
|
||||||
// Setup
|
// Setup
|
||||||
|
@ -1841,7 +1841,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(4, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -1857,7 +1857,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(4, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(2, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -1927,7 +1927,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(9, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(7, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(7, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -1943,7 +1943,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(7, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(6, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(5, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(5, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -2239,7 +2239,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(9, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -2256,7 +2256,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(8, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(7, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -2271,7 +2271,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
outcome = mySystemDao.transaction(mySrd, input.get());
|
outcome = mySystemDao.transaction(mySrd, input.get());
|
||||||
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome));
|
||||||
myCaptureQueriesListener.logSelectQueries();
|
myCaptureQueriesListener.logSelectQueries();
|
||||||
assertEquals(6, myCaptureQueriesListener.countSelectQueries());
|
assertEquals(5, myCaptureQueriesListener.countSelectQueries());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
assertEquals(4, myCaptureQueriesListener.countInsertQueries());
|
||||||
myCaptureQueriesListener.logUpdateQueries();
|
myCaptureQueriesListener.logUpdateQueries();
|
||||||
|
@ -3365,7 +3365,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
|
|
||||||
myCaptureQueriesListener.clear();
|
myCaptureQueriesListener.clear();
|
||||||
Bundle outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
|
Bundle outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
|
||||||
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
||||||
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
||||||
|
@ -3388,7 +3388,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
|
|
||||||
myCaptureQueriesListener.clear();
|
myCaptureQueriesListener.clear();
|
||||||
outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
|
outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get());
|
||||||
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
||||||
myCaptureQueriesListener.logInsertQueries();
|
myCaptureQueriesListener.logInsertQueries();
|
||||||
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
||||||
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
||||||
|
@ -3449,7 +3449,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
|
||||||
myCaptureQueriesListener.clear();
|
myCaptureQueriesListener.clear();
|
||||||
mySystemDao.transaction(new SystemRequestDetails(), loadResourceFromClasspath(Bundle.class, "r4/transaction-perf-bundle-smallchanges.json"));
|
mySystemDao.transaction(new SystemRequestDetails(), loadResourceFromClasspath(Bundle.class, "r4/transaction-perf-bundle-smallchanges.json"));
|
||||||
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
|
||||||
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
|
||||||
assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
|
||||||
assertEquals(5, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
assertEquals(5, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
|
||||||
assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
|
assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
|
||||||
|
|
Loading…
Reference in New Issue