diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5824-fix-history-prefetch.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5824-fix-history-prefetch.yaml new file mode 100644 index 00000000000..ab44d762996 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5824-fix-history-prefetch.yaml @@ -0,0 +1,4 @@ +--- +type: fix +issue: 5824 +title: "We now avoid a query during reindex and transaction processing that was very slow on Sql Server." diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java index 951045de041..b200a4de82c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java @@ -38,7 +38,6 @@ import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory; -import ca.uhn.fhir.jpa.search.builder.SearchBuilder; import ca.uhn.fhir.jpa.util.QueryChunker; import ca.uhn.fhir.jpa.util.ResourceCountCache; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -51,12 +50,9 @@ import jakarta.annotation.Nullable; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.PersistenceContextType; +import jakarta.persistence.Query; import jakarta.persistence.TypedQuery; -import jakarta.persistence.criteria.CriteriaBuilder; -import jakarta.persistence.criteria.CriteriaQuery; -import jakarta.persistence.criteria.JoinType; import jakarta.persistence.criteria.Predicate; -import jakarta.persistence.criteria.Root; import org.hl7.fhir.instance.model.api.IBaseBundle; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -206,8 +202,25 @@ public abstract class BaseHapiFhirSystemDao extends B new QueryChunker() .chunk( ids, - nextChunk -> - loadedResourceTableEntries.addAll(myResourceTableDao.findAllById(nextChunk))); + nextChunk -> { + //List allById = myResourceTableDao.findAllById(nextChunk); + Query query = myEntityManager.createQuery( + "select r, h FROM ResourceTable r " + + " LEFT JOIN fetch ResourceHistoryTable h on r.myVersion = h.myResourceVersion and r.id = h.myResourceId" + + " left join fetch h.myProvenance" + + " WHERE r.myId IN ( :IDS )"); + query.setParameter("IDS", ids); + + @SuppressWarnings("unchecked") + List allById = query.getResultList(); + + for (Object[] nextPair : allById) { + ResourceTable r = (ResourceTable) nextPair[0]; + ResourceHistoryTable h = (ResourceHistoryTable) nextPair[1]; + r.setCurrentVersionEntity(h); + loadedResourceTableEntries.add(r); + } + }); List entityIds; @@ -269,33 +282,33 @@ public abstract class BaseHapiFhirSystemDao extends B } } - new QueryChunker() - .chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> { - Map entities = - entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t)); - - CriteriaBuilder b = myEntityManager.getCriteriaBuilder(); - CriteriaQuery q = b.createQuery(ResourceHistoryTable.class); - Root from = q.from(ResourceHistoryTable.class); - - from.fetch("myProvenance", JoinType.LEFT); - - List orPredicates = new ArrayList<>(); - for (ResourceTable next : entries) { - Predicate resId = b.equal(from.get("myResourceId"), next.getId()); - Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion()); - orPredicates.add(b.and(resId, resVer)); - } - q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY))); - List resultList = - myEntityManager.createQuery(q).getResultList(); - for (ResourceHistoryTable next : resultList) { - ResourceTable nextEntity = entities.get(next.getResourceId()); - if (nextEntity != null) { - nextEntity.setCurrentVersionEntity(next); - } - } - }); +// new QueryChunker() +// .chunk(loadedResourceTableEntries, SearchBuilder.getMaximumPageSize() / 2, entries -> { +// Map entities = +// entries.stream().collect(Collectors.toMap(ResourceTable::getId, t -> t)); +// +// CriteriaBuilder b = myEntityManager.getCriteriaBuilder(); +// CriteriaQuery q = b.createQuery(ResourceHistoryTable.class); +// Root from = q.from(ResourceHistoryTable.class); +// +// from.fetch("myProvenance", JoinType.LEFT); +// +// List orPredicates = new ArrayList<>(); +// for (ResourceTable next : entries) { +// Predicate resId = b.equal(from.get("myResourceId"), next.getId()); +// Predicate resVer = b.equal(from.get("myResourceVersion"), next.getVersion()); +// orPredicates.add(b.and(resId, resVer)); +// } +// q.where(b.or(orPredicates.toArray(EMPTY_PREDICATE_ARRAY))); +// List resultList = +// myEntityManager.createQuery(q).getResultList(); +// for (ResourceHistoryTable next : resultList) { +// ResourceTable nextEntity = entities.get(next.getResourceId()); +// if (nextEntity != null) { +// nextEntity.setCurrentVersionEntity(next); +// } +// } +// }); } }); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java index 1f299d970ef..da58faa2c6a 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java @@ -827,7 +827,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?active=true", new SystemRequestDetails()); // Validate - assertEquals(13, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(12, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); assertEquals(10, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); assertEquals(10, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(30, myCaptureQueriesListener.countDeleteQueriesForCurrentThread()); @@ -1026,10 +1026,10 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test @ParameterizedTest @CsvSource({ // OptimisticLock OptimizeMode ExpectedSelect ExpectedUpdate - " false, CURRENT_VERSION, 2, 0", - " true, CURRENT_VERSION, 12, 0", - " false, ALL_VERSIONS, 12, 0", - " true, ALL_VERSIONS, 22, 0", + " false, CURRENT_VERSION, 1, 0", + " true, CURRENT_VERSION, 11, 0", + " false, ALL_VERSIONS, 11, 0", + " true, ALL_VERSIONS, 21, 0", }) public void testReindexJob_OptimizeStorage(boolean theOptimisticLock, ReindexParameters.OptimizeStorageModeEnum theOptimizeStorageModeEnum, int theExpectedSelectCount, int theExpectedUpdateCount) { // Setup @@ -1841,7 +1841,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(5, myCaptureQueriesListener.countSelectQueries()); + assertEquals(4, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(2, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -1857,7 +1857,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(5, myCaptureQueriesListener.countSelectQueries()); + assertEquals(4, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(2, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -1927,7 +1927,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(9, myCaptureQueriesListener.countSelectQueries()); + assertEquals(8, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(7, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -1943,7 +1943,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(7, myCaptureQueriesListener.countSelectQueries()); + assertEquals(6, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(5, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -2239,7 +2239,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(9, myCaptureQueriesListener.countSelectQueries()); + assertEquals(8, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(4, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -2256,7 +2256,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(8, myCaptureQueriesListener.countSelectQueries()); + assertEquals(7, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(4, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -2271,7 +2271,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test outcome = mySystemDao.transaction(mySrd, input.get()); ourLog.debug("Resp: {}", myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome)); myCaptureQueriesListener.logSelectQueries(); - assertEquals(6, myCaptureQueriesListener.countSelectQueries()); + assertEquals(5, myCaptureQueriesListener.countSelectQueries()); myCaptureQueriesListener.logInsertQueries(); assertEquals(4, myCaptureQueriesListener.countInsertQueries()); myCaptureQueriesListener.logUpdateQueries(); @@ -3365,7 +3365,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test myCaptureQueriesListener.clear(); Bundle outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get()); - assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); myCaptureQueriesListener.logInsertQueries(); assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); @@ -3388,7 +3388,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test myCaptureQueriesListener.clear(); outcome = mySystemDao.transaction(new SystemRequestDetails(), supplier.get()); - assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); myCaptureQueriesListener.logInsertQueries(); assertEquals(4, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(6, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); @@ -3449,7 +3449,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test myCaptureQueriesListener.clear(); mySystemDao.transaction(new SystemRequestDetails(), loadResourceFromClasspath(Bundle.class, "r4/transaction-perf-bundle-smallchanges.json")); myCaptureQueriesListener.logSelectQueriesForCurrentThread(); - assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); + assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread()); assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread()); assertEquals(5, myCaptureQueriesListener.countUpdateQueriesForCurrentThread()); assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());