From 4afa55ea260e136a83ea40568d7757619fdb68c0 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 22 Aug 2019 08:49:34 -0400 Subject: [PATCH 01/16] Initial commit - Not yet compiling --- .../ca/uhn/fhir/jpa/config/BaseConfig.java | 7 +++++++ .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 5 +++-- ...earchDao.java => IAAAAAAAAASearchDao.java} | 2 +- ...o.java => IAAAAAAAAASearchIncludeDao.java} | 2 +- ...Dao.java => IAAAAAAAASearchResultDao.java} | 2 +- .../java/ca/uhn/fhir/jpa/search/.editorconfig | 0 .../search/PersistedJpaBundleProvider.java | 16 +++++++++------- .../cache/BaseSearchResultCacheSvcImpl.java | 4 ++++ .../DatabaseSearchResultCacheSvcImpl.java | 19 +++++++++++++++++++ .../search/cache/ISearchResultCacheSvc.java | 14 ++++++++++++++ 10 files changed, 59 insertions(+), 12 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{ISearchDao.java => IAAAAAAAAASearchDao.java} (96%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{ISearchIncludeDao.java => IAAAAAAAAASearchIncludeDao.java} (92%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{ISearchResultDao.java => IAAAAAAAASearchResultDao.java} (95%) delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/.editorconfig create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 0f9b12a8d94..db1975c2132 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -14,6 +14,8 @@ import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.IStaleSearchDeletingSvc; import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl; import ca.uhn.fhir.jpa.subscription.dbmatcher.CompositeInMemoryDaoSubscriptionMatcher; @@ -143,6 +145,11 @@ public abstract class BaseConfig implements SchedulingConfigurer { return new BinaryStorageInterceptor(); } + @Bean + public ISearchResultCacheSvc searchResultCacheSvc() { + return new DatabaseSearchResultCacheSvcImpl(); + } + @Bean public TaskScheduler taskScheduler() { ConcurrentTaskScheduler retVal = new ConcurrentTaskScheduler(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index 602daaeb653..1c915711d36 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -18,6 +18,7 @@ import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.ResourceMetaParams; import ca.uhn.fhir.jpa.searchparam.extractor.LogicalReferenceHelper; import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; @@ -152,7 +153,7 @@ public abstract class BaseHapiFhirDao implements IDao, @Autowired private PlatformTransactionManager myPlatformTransactionManager; @Autowired - private ISearchDao mySearchDao; + private ISearchResultCacheSvc mySearchResultCacheSvc; @Autowired private ISearchParamPresenceSvc mySearchParamPresenceSvc; @Autowired @@ -466,7 +467,7 @@ public abstract class BaseHapiFhirDao implements IDao, } } - search = mySearchDao.save(search); + search = mySearchResultCacheSvc.saveNew(search); return new PersistedJpaBundleProvider(theRequest, search.getUuid(), this); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java similarity index 96% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java index e3716f54999..b3bc8aa9bd0 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java @@ -31,7 +31,7 @@ import java.util.Date; * #L% */ -public interface ISearchDao extends JpaRepository { +public interface IAAAAAAAAASearchDao extends JpaRepository { @Query("SELECT s FROM Search s WHERE s.myUuid = :uuid") Search findByUuid(@Param("uuid") String theUuid); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java similarity index 92% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java index 3307bb8c2be..16db58ed172 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java @@ -27,7 +27,7 @@ import org.springframework.data.repository.query.Param; import ca.uhn.fhir.jpa.entity.SearchInclude; -public interface ISearchIncludeDao extends JpaRepository { +public interface IAAAAAAAAASearchIncludeDao extends JpaRepository { @Modifying @Query(value="DELETE FROM SearchInclude r WHERE r.mySearchPid = :search") diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java similarity index 95% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java index 58c5c0eae06..ba9429234ee 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java @@ -34,7 +34,7 @@ import java.util.Set; * #L% */ -public interface ISearchResultDao extends JpaRepository { +public interface IAAAAAAAASearchResultDao extends JpaRepository { @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearch = :search ORDER BY r.myOrder ASC") Page findWithSearchUuid(@Param("search") Search theSearch, Pageable thePage); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/.editorconfig b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/.editorconfig deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index 5f61a46a253..cf110832aaf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -31,6 +31,7 @@ import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.BaseHasResource; import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; import ca.uhn.fhir.model.primitive.InstantDt; import ca.uhn.fhir.rest.api.server.*; @@ -64,7 +65,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { private EntityManager myEntityManager; private PlatformTransactionManager myPlatformTransactionManager; private ISearchCoordinatorSvc mySearchCoordinatorSvc; - private ISearchDao mySearchDao; + private ISearchResultCacheSvc mySearchResultCacheSvc; private Search mySearchEntity; private String myUuid; private boolean myCacheHit; @@ -196,8 +197,12 @@ public class PersistedJpaBundleProvider implements IBundleProvider { txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); txTemplate.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); return txTemplate.execute(s -> { - try { - setSearchEntity(mySearchDao.findByUuid(myUuid)); + Optional search = mySearchResultCacheSvc.fetchByUuid(myUuid); + if (!search.isPresent()) { + return false; + } + + setSearchEntity(search); if (mySearchEntity == null) { return false; @@ -209,10 +214,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { mySearchEntity.getIncludes().size(); return true; - } catch (NoResultException e) { - return false; - } - }); + }); } return true; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java new file mode 100644 index 00000000000..73ff5a2d546 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java @@ -0,0 +1,4 @@ +package ca.uhn.fhir.jpa.search.cache; + +public abstract class BaseSearchResultCacheSvcImpl implements ISearchResultCacheSvc { +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java new file mode 100644 index 00000000000..603d683f0f8 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -0,0 +1,19 @@ +package ca.uhn.fhir.jpa.search.cache; + +import ca.uhn.fhir.jpa.dao.data.IAAAAAAAAASearchDao; +import ca.uhn.fhir.jpa.entity.Search; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.transaction.Transactional; + +public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcImpl { + + @Autowired + private IAAAAAAAAASearchDao mySearchDao; + + @Transactional(Transactional.TxType.MANDATORY) + @Override + public Search saveNew(Search theSearch) { + return mySearchDao.save(theSearch); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java new file mode 100644 index 00000000000..fbe808b0a17 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java @@ -0,0 +1,14 @@ +package ca.uhn.fhir.jpa.search.cache; + +import ca.uhn.fhir.jpa.entity.Search; + +public interface ISearchResultCacheSvc { + + /** + * Places a new search of some sort in the cache. + * + * @param theSearch The search to store + * @return Returns a copy of the search as it was saved. Callers should use the returned Search object for any further processing. + */ + Search saveNew(Search theSearch); +} From 6fa27934a80092b8669cfb223ff5c8a63a1fe360 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 23 Aug 2019 10:14:13 -0400 Subject: [PATCH 02/16] Tests all seem to be working --- .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 4 +- ...AAAAAAAASearchDao.java => ISearchDao.java} | 10 +- ...IncludeDao.java => ISearchIncludeDao.java} | 2 +- ...chResultDao.java => ISearchResultDao.java} | 14 +- .../jpa/dao/expunge/ExpungeOperation.java | 5 - .../dao/expunge/IResourceExpungeService.java | 2 - .../dao/expunge/ResourceExpungeService.java | 8 - .../java/ca/uhn/fhir/jpa/entity/Search.java | 5 +- .../ca/uhn/fhir/jpa/entity/SearchResult.java | 14 +- .../search/PersistedJpaBundleProvider.java | 36 +- .../jpa/search/SearchCoordinatorSvcImpl.java | 131 ++---- .../search/StaleSearchDeletingSvcImpl.java | 145 +------ .../cache/BaseSearchResultCacheSvcImpl.java | 41 ++ .../DatabaseSearchResultCacheSvcImpl.java | 279 +++++++++++- .../search/cache/ISearchResultCacheSvc.java | 95 ++++- .../java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java | 3 + .../fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java | 2 - ...rResourceDaoDstu3SearchPageExpiryTest.java | 401 ------------------ .../ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java | 6 - .../r4/FhirResourceDaoR4SearchNoFtTest.java | 3 + .../FhirResourceDaoR4SearchNoHashesTest.java | 5 +- .../FhirResourceDaoR4SearchOptimizedTest.java | 34 +- ...FhirResourceDaoR4SearchPageExpiryTest.java | 80 ++-- .../jpa/dao/r4/FhirResourceDaoR4Test.java | 4 +- .../ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java | 6 - .../BaseResourceProviderDstu2Test.java | 17 +- .../dstu3/BaseResourceProviderDstu3Test.java | 6 +- .../dstu3/ResourceProviderDstu3Test.java | 36 +- .../StaleSearchDeletingSvcDstu3Test.java | 97 ----- .../r4/BaseResourceProviderR4Test.java | 1 - .../fhir/jpa/provider/r4/ExpungeR4Test.java | 7 + .../r4/ResourceProviderR4CacheTest.java | 20 +- .../provider/r4/ResourceProviderR4Test.java | 21 +- .../r4/StaleSearchDeletingSvcR4Test.java | 33 +- .../r5/BaseResourceProviderR5Test.java | 3 - .../search/SearchCoordinatorSvcImplTest.java | 120 +++--- .../migrate/taskdef/DropForeignKeyTask.java | 85 ++++ .../tasks/HapiFhirJpaMigrationTasks.java | 3 + .../migrate/tasks/api/BaseMigrationTasks.java | 7 + .../taskdef/DropForeignKeyTaskTest.java | 38 ++ src/changes/changes.xml | 4 + 41 files changed, 822 insertions(+), 1011 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{IAAAAAAAAASearchDao.java => ISearchDao.java} (85%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{IAAAAAAAAASearchIncludeDao.java => ISearchIncludeDao.java} (92%) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/{IAAAAAAAASearchResultDao.java => ISearchResultDao.java} (80%) delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchPageExpiryTest.java delete mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/StaleSearchDeletingSvcDstu3Test.java create mode 100644 hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java create mode 100644 hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index 1c915711d36..44f1698609d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -467,7 +467,7 @@ public abstract class BaseHapiFhirDao implements IDao, } } - search = mySearchResultCacheSvc.saveNew(search); + search = mySearchResultCacheSvc.save(search); return new PersistedJpaBundleProvider(theRequest, search.getUuid(), this); } @@ -493,7 +493,7 @@ public abstract class BaseHapiFhirDao implements IDao, theProvider.setContext(getContext()); theProvider.setEntityManager(myEntityManager); theProvider.setPlatformTransactionManager(myPlatformTransactionManager); - theProvider.setSearchDao(mySearchDao); + theProvider.setSearchResultCacheSvc(mySearchResultCacheSvc); theProvider.setSearchCoordinatorSvc(mySearchCoordinatorSvc); theProvider.setInterceptorBroadcaster(myInterceptorBroadcaster); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java similarity index 85% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java index b3bc8aa9bd0..3acc1a39d04 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java @@ -10,6 +10,7 @@ import org.springframework.data.repository.query.Param; import java.util.Collection; import java.util.Date; +import java.util.Optional; /* * #%L @@ -31,17 +32,14 @@ import java.util.Date; * #L% */ -public interface IAAAAAAAAASearchDao extends JpaRepository { +public interface ISearchDao extends JpaRepository { - @Query("SELECT s FROM Search s WHERE s.myUuid = :uuid") - Search findByUuid(@Param("uuid") String theUuid); + @Query("SELECT s FROM Search s LEFT OUTER JOIN FETCH s.myIncludes WHERE s.myUuid = :uuid") + Optional findByUuidAndFetchIncludes(@Param("uuid") String theUuid); @Query("SELECT s.myId FROM Search s WHERE s.mySearchLastReturned < :cutoff") Slice findWhereLastReturnedBefore(@Param("cutoff") Date theCutoff, Pageable thePage); -// @SqlQuery("SELECT s FROM Search s WHERE s.myCreated < :cutoff") -// public Collection findWhereCreatedBefore(@Param("cutoff") Date theCutoff); - @Query("SELECT s FROM Search s WHERE s.myResourceType = :type AND mySearchQueryStringHash = :hash AND s.myCreated > :cutoff AND s.myDeleted = false") Collection find(@Param("type") String theResourceType, @Param("hash") int theHashCode, @Param("cutoff") Date theCreatedCutoff); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java similarity index 92% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java index 16db58ed172..21a76447194 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAAASearchIncludeDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java @@ -27,7 +27,7 @@ import org.springframework.data.repository.query.Param; import ca.uhn.fhir.jpa.entity.SearchInclude; -public interface IAAAAAAAAASearchIncludeDao extends JpaRepository { +public interface ISearchIncludeDao extends JpaRepository { @Modifying @Query(value="DELETE FROM SearchInclude r WHERE r.mySearchPid = :search") diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java similarity index 80% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java index ba9429234ee..e0806cb01ec 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IAAAAAAAASearchResultDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java @@ -34,21 +34,17 @@ import java.util.Set; * #L% */ -public interface IAAAAAAAASearchResultDao extends JpaRepository { +public interface ISearchResultDao extends JpaRepository { - @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearch = :search ORDER BY r.myOrder ASC") - Page findWithSearchUuid(@Param("search") Search theSearch, Pageable thePage); + @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearchPid = :search ORDER BY r.myOrder ASC") + Slice findWithSearchUuid(@Param("search") Long theSearch, Pageable thePage); - @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearch = :search") - List findWithSearchUuidOrderIndependent(@Param("search") Search theSearch); + @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearchPid = :search") + List findWithSearchUuidOrderIndependent(@Param("search") Long theSearch); @Query(value="SELECT r.myId FROM SearchResult r WHERE r.mySearchPid = :search") Slice findForSearch(Pageable thePage, @Param("search") Long theSearchPid); - @Modifying - @Query("DELETE FROM SearchResult s WHERE s.myResourcePid IN :ids") - void deleteByResourceIds(@Param("ids") List theContent); - @Modifying @Query("DELETE FROM SearchResult s WHERE s.myId IN :ids") void deleteByIds(@Param("ids") List theContent); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeOperation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeOperation.java index f092b6d73e4..95aee1da8b7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeOperation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeOperation.java @@ -84,7 +84,6 @@ public class ExpungeOperation implements Callable { private void expungeDeletedResources() { Slice resourceIds = findHistoricalVersionsOfDeletedResources(); - deleteSearchResultCacheEntries(resourceIds); deleteHistoricalVersions(resourceIds); if (expungeLimitReached()) { return; @@ -123,10 +122,6 @@ public class ExpungeOperation implements Callable { myPartitionRunner.runInPartitionedThreads(theResourceIds, partition -> myExpungeDaoService.expungeHistoricalVersionsOfIds(myRequestDetails, partition, myRemainingCount)); } - private void deleteSearchResultCacheEntries(Slice theResourceIds) { - myPartitionRunner.runInPartitionedThreads(theResourceIds, partition -> myExpungeDaoService.deleteByResourceIdPartitions(partition)); - } - private ExpungeOutcome expungeOutcome() { return new ExpungeOutcome().setDeletedCount(myExpungeOptions.getLimit() - myRemainingCount.get()); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/IResourceExpungeService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/IResourceExpungeService.java index a8f1a52aafc..5beaf5fa548 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/IResourceExpungeService.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/IResourceExpungeService.java @@ -37,7 +37,5 @@ public interface IResourceExpungeService { void expungeHistoricalVersionsOfIds(RequestDetails theRequestDetails, List thePartition, AtomicInteger theRemainingCount); - void deleteByResourceIdPartitions(List thePartition); - void deleteAllSearchParams(Long theResourceId); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ResourceExpungeService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ResourceExpungeService.java index 5f5203a055f..ffc69bf92df 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ResourceExpungeService.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/ResourceExpungeService.java @@ -58,8 +58,6 @@ class ResourceExpungeService implements IResourceExpungeService { @Autowired private IResourceTableDao myResourceTableDao; @Autowired - private ISearchResultDao mySearchResultDao; - @Autowired private IResourceHistoryTableDao myResourceHistoryTableDao; @Autowired private IResourceIndexedSearchParamUriDao myResourceIndexedSearchParamUriDao; @@ -250,12 +248,6 @@ class ResourceExpungeService implements IResourceExpungeService { } } - @Override - @Transactional - public void deleteByResourceIdPartitions(List theResourceIds) { - mySearchResultDao.deleteByResourceIds(theResourceIds); - } - private Slice toSlice(ResourceHistoryTable myVersion) { Validate.notNull(myVersion); return new SliceImpl<>(Collections.singletonList(myVersion.getId())); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index ee52ab794e1..928e4b0d48c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -80,8 +80,9 @@ public class Search implements ICachedSearchDetails, Serializable { private Long myResourceId; @Column(name = "RESOURCE_TYPE", length = 200, nullable = true) private String myResourceType; - @OneToMany(mappedBy = "mySearch", fetch = FetchType.LAZY) - private Collection myResults; + // FIXME JA: delete this if we can - I don't want to imply that the results are a part of the search, they link to it but they don't need to be loaded just because we're loading the search + // @OneToMany(mappedBy = "mySearch", fetch = FetchType.LAZY) + // private Collection myResults; @NotNull @Temporal(TemporalType.TIMESTAMP) @Column(name = "SEARCH_LAST_RETURNED", nullable = false, updatable = false) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java index 62cb95309bb..ab8a53eed53 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java @@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.entity; */ import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.ToStringBuilder; import javax.persistence.*; @@ -39,17 +40,11 @@ public class SearchResult implements Serializable { @Id @Column(name = "PID") private Long myId; - @Column(name = "SEARCH_ORDER", nullable = false) + @Column(name = "SEARCH_ORDER", nullable = false, insertable = true, updatable = false) private int myOrder; - @ManyToOne - @JoinColumn(name = "RESOURCE_PID", referencedColumnName = "RES_ID", foreignKey = @ForeignKey(name = "FK_SEARCHRES_RES"), insertable = false, updatable = false, nullable = false) - private ResourceTable myResource; @Column(name = "RESOURCE_PID", insertable = true, updatable = false, nullable = false) private Long myResourcePid; - @ManyToOne - @JoinColumn(name = "SEARCH_PID", referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_SEARCHRES_SEARCH")) - private Search mySearch; - @Column(name = "SEARCH_PID", insertable = false, updatable = false, nullable = false) + @Column(name = "SEARCH_PID", insertable = true, updatable = false, nullable = false) private Long mySearchPid; /** @@ -63,7 +58,8 @@ public class SearchResult implements Serializable { * Constructor */ public SearchResult(Search theSearch) { - mySearch = theSearch; + Validate.notNull(theSearch.getId()); + mySearchPid = theSearch.getId(); } @Override diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index cf110832aaf..db3b3d9fb11 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -26,7 +26,6 @@ import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.dao.IDao; import ca.uhn.fhir.jpa.dao.ISearchBuilder; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.BaseHasResource; @@ -47,7 +46,6 @@ import org.springframework.transaction.support.TransactionTemplate; import javax.annotation.Nonnull; import javax.persistence.EntityManager; -import javax.persistence.NoResultException; import javax.persistence.TypedQuery; import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaQuery; @@ -193,28 +191,16 @@ public class PersistedJpaBundleProvider implements IBundleProvider { if (mySearchEntity == null) { ensureDependenciesInjected(); - TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - txTemplate.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); - return txTemplate.execute(s -> { - Optional search = mySearchResultCacheSvc.fetchByUuid(myUuid); - if (!search.isPresent()) { - return false; - } + Optional search = mySearchResultCacheSvc.fetchByUuid(myUuid); + if (!search.isPresent()) { + return false; + } - setSearchEntity(search); + setSearchEntity(search.get()); - if (mySearchEntity == null) { - return false; - } + ourLog.trace("Retrieved search with version {} and total {}", mySearchEntity.getVersion(), mySearchEntity.getTotalCount()); - ourLog.trace("Retrieved search with version {} and total {}", mySearchEntity.getVersion(), mySearchEntity.getTotalCount()); - - // Load the includes now so that they are available outside of this transaction - mySearchEntity.getIncludes().size(); - - return true; - }); + return true; } return true; } @@ -294,10 +280,6 @@ public class PersistedJpaBundleProvider implements IBundleProvider { mySearchCoordinatorSvc = theSearchCoordinatorSvc; } - public void setSearchDao(ISearchDao theSearchDao) { - mySearchDao = theSearchDao; - } - // Note: Leave as protected, HSPC depends on this @SuppressWarnings("WeakerAccess") protected void setSearchEntity(Search theSearchEntity) { @@ -355,4 +337,8 @@ public class PersistedJpaBundleProvider implements IBundleProvider { public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) { myInterceptorBroadcaster = theInterceptorBroadcaster; } + + public void setSearchResultCacheSvc(ISearchResultCacheSvc theSearchResultCacheSvc) { + mySearchResultCacheSvc = theSearchResultCacheSvc; + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index a6402100d4d..0f5c2c34eaa 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -25,16 +25,13 @@ import ca.uhn.fhir.interceptor.api.HookParams; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.dao.*; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; -import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; -import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchInclude; -import ca.uhn.fhir.jpa.entity.SearchResult; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; import ca.uhn.fhir.model.api.Include; @@ -56,7 +53,6 @@ import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import ca.uhn.fhir.util.StopWatch; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.time.DateUtils; @@ -64,7 +60,6 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.AbstractPageRequest; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.orm.jpa.JpaDialect; @@ -106,16 +101,12 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { private long myMaxMillisToWaitForRemoteResults = DateUtils.MILLIS_PER_MINUTE; private boolean myNeverUseLocalSearchForUnitTests; @Autowired - private ISearchDao mySearchDao; - @Autowired - private ISearchIncludeDao mySearchIncludeDao; - @Autowired - private ISearchResultDao mySearchResultDao; - @Autowired private IInterceptorBroadcaster myInterceptorBroadcaster; @Autowired private PlatformTransactionManager myManagedTxManager; @Autowired + private ISearchResultCacheSvc mySearchResultCacheSvc; + @Autowired private DaoRegistry myDaoRegistry; @Autowired private IPagingProvider myPagingProvider; @@ -134,6 +125,11 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { myExecutor = Executors.newCachedThreadPool(threadFactory); } + @VisibleForTesting + public void setSearchResultCacheSvcForUnitTest(ISearchResultCacheSvc theSearchResultCacheSvc) { + mySearchResultCacheSvc = theSearchResultCacheSvc; + } + @PostConstruct public void start() { if (myManagedTxManager instanceof JpaTransactionManager) { @@ -184,13 +180,13 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } } - search = txTemplate.execute(t -> mySearchDao.findByUuid(theUuid)); - - if (search == null) { - ourLog.debug("Client requested unknown paging ID[{}]", theUuid); - String msg = myContext.getLocalizer().getMessage(PageMethodBinding.class, "unknownSearchId", theUuid); - throw new ResourceGoneException(msg); - } + search = mySearchResultCacheSvc + .fetchByUuid(theUuid) + .orElseThrow(() -> { + ourLog.debug("Client requested unknown paging ID[{}]", theUuid); + String msg = myContext.getLocalizer().getMessage(PageMethodBinding.class, "unknownSearchId", theUuid); + return new ResourceGoneException(msg); + }); verifySearchHasntFailedOrThrowInternalErrorException(search); if (search.getStatus() == SearchStatusEnum.FINISHED) { @@ -210,7 +206,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { // If the search was saved in "pass complete mode" it's probably time to // start a new pass if (search.getStatus() == SearchStatusEnum.PASSCMPLET) { - Optional newSearch = tryToMarkSearchAsInProgress(search); + Optional newSearch = mySearchResultCacheSvc.tryToMarkSearchAsInProgress(search); if (newSearch.isPresent()) { search = newSearch.get(); String resourceType = search.getResourceType(); @@ -229,53 +225,16 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } } - final Pageable page = toPage(theFrom, theTo); - if (page == null) { - return Collections.emptyList(); - } - final Search foundSearch = search; - - ourLog.trace("Loading stored search"); - List retVal = txTemplate.execute(theStatus -> { - final List resultPids = new ArrayList<>(); - Page searchResultPids = mySearchResultDao.findWithSearchUuid(foundSearch, page); - for (Long next : searchResultPids) { - resultPids.add(next); - } - return resultPids; - }); - return retVal; + return mySearchResultCacheSvc.fetchResultPids(search, theFrom, theTo); } - private Optional tryToMarkSearchAsInProgress(Search theSearch) { - ourLog.trace("Going to try to change search status from {} to {}", theSearch.getStatus(), SearchStatusEnum.LOADING); - try { - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - txTemplate.afterPropertiesSet(); - return txTemplate.execute(t -> { - Search search = mySearchDao.findById(theSearch.getId()).orElse(theSearch); - - if (search.getStatus() != SearchStatusEnum.PASSCMPLET) { - throw new IllegalStateException("Can't change to LOADING because state is " + theSearch.getStatus()); - } - search.setStatus(SearchStatusEnum.LOADING); - Search newSearch = mySearchDao.save(search); - return Optional.of(newSearch); - }); - } catch (Exception e) { - ourLog.warn("Failed to activate search: {}", e.toString()); - ourLog.trace("Failed to activate search", e); - return Optional.empty(); - } - } private void populateBundleProvider(PersistedJpaBundleProvider theRetVal) { theRetVal.setContext(myContext); theRetVal.setEntityManager(myEntityManager); theRetVal.setPlatformTransactionManager(myManagedTxManager); - theRetVal.setSearchDao(mySearchDao); + theRetVal.setSearchResultCacheSvc(mySearchResultCacheSvc); theRetVal.setSearchCoordinatorSvc(this); theRetVal.setInterceptorBroadcaster(myInterceptorBroadcaster); } @@ -398,10 +357,9 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } // Check for a search matching the given hash - int hashCode = queryString.hashCode(); - Collection candidates = mySearchDao.find(resourceType, hashCode, createdCutoff); + Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(resourceType, queryString, createdCutoff); for (Search nextCandidateSearch : candidates) { - if (queryString.equals(nextCandidateSearch.getSearchQueryString())) { + if (queryString.equals(nextCandidateSearch.getSearchQueryString()) && nextCandidateSearch.getCreated().after(createdCutoff)) { searchToUse = nextCandidateSearch; break; } @@ -418,8 +376,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params); - searchToUse.setSearchLastReturned(new Date()); - mySearchDao.updateSearchLastReturned(searchToUse.getId(), new Date()); + mySearchResultCacheSvc.updateSearchLastReturned(searchToUse, new Date()); retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao); retVal.setCacheHit(true); @@ -500,21 +457,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { myNeverUseLocalSearchForUnitTests = theNeverUseLocalSearchForUnitTests; } - @VisibleForTesting - void setSearchDaoForUnitTest(ISearchDao theSearchDao) { - mySearchDao = theSearchDao; - } - - @VisibleForTesting - void setSearchDaoIncludeForUnitTest(ISearchIncludeDao theSearchIncludeDao) { - mySearchIncludeDao = theSearchIncludeDao; - } - - @VisibleForTesting - void setSearchDaoResultForUnitTest(ISearchResultDao theSearchResultDao) { - mySearchResultDao = theSearchResultDao; - } - @VisibleForTesting public void setSyncSizeForUnitTests(int theSyncSize) { mySyncSize = theSyncSize; @@ -696,20 +638,10 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } } - List resultsToSave = Lists.newArrayList(); - for (Long nextPid : unsyncedPids) { - SearchResult nextResult = new SearchResult(mySearch); - nextResult.setResourcePid(nextPid); - nextResult.setOrder(myCountSavedTotal); - resultsToSave.add(nextResult); - int order = nextResult.getOrder(); - ourLog.trace("Saving ORDER[{}] Resource {}", order, nextResult.getResourcePid()); - - myCountSavedTotal++; - myCountSavedThisPass++; - } - - mySearchResultDao.saveAll(resultsToSave); + // Actually store the results in the query cache storage + myCountSavedTotal += unsyncedPids.size(); + myCountSavedThisPass += unsyncedPids.size(); + mySearchResultCacheSvc.storeResults(mySearch, mySyncedPids, unsyncedPids); synchronized (mySyncedPids) { int numSyncedThisPass = unsyncedPids.size(); @@ -877,15 +809,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { private void doSaveSearch() { - Search newSearch; - if (mySearch.getId() == null) { - newSearch = mySearchDao.save(mySearch); - for (SearchInclude next : mySearch.getIncludes()) { - mySearchIncludeDao.save(next); - } - } else { - newSearch = mySearchDao.save(mySearch); - } + Search newSearch = mySearchResultCacheSvc.save(mySearch); // mySearchDao.save is not supposed to return null, but in unit tests // it can if the mock search dao isn't set up to handle that @@ -928,7 +852,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { mySearch.setStatus(SearchStatusEnum.FINISHED); } doSaveSearch(); - mySearchDao.flush(); } }); if (wantOnlyCount) { @@ -1058,7 +981,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); txTemplate.afterPropertiesSet(); txTemplate.execute(t -> { - List previouslyAddedResourcePids = mySearchResultDao.findWithSearchUuidOrderIndependent(getSearch()); + List previouslyAddedResourcePids = mySearchResultCacheSvc.fetchAllResultPids(getSearch()); ourLog.debug("Have {} previously added IDs in search: {}", previouslyAddedResourcePids.size(), getSearch().getUuid()); setPreviouslyAddedResourcePids(previouslyAddedResourcePids); return null; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java index b470e574e9d..c1c432c71bc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java @@ -21,24 +21,13 @@ package ca.uhn.fhir.jpa.search; */ import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; -import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; -import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.dstu3.model.InstantType; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Slice; import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionTemplate; -import java.util.Date; -import java.util.List; +import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK; /** * Deletes old searches @@ -49,111 +38,16 @@ import java.util.List; // in Smile. // public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc { - public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND; private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(StaleSearchDeletingSvcImpl.class); - /* - * Be careful increasing this number! We use the number of params here in a - * DELETE FROM foo WHERE params IN (aaaa) - * type query and this can fail if we have 1000s of params - */ - public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500; - public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000; - private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT; - private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS; - private static Long ourNowForUnitTests; - /* - * We give a bit of extra leeway just to avoid race conditions where a query result - * is being reused (because a new client request came in with the same params) right before - * the result is to be deleted - */ - private long myCutoffSlack = DEFAULT_CUTOFF_SLACK; @Autowired private DaoConfig myDaoConfig; @Autowired - private ISearchDao mySearchDao; - @Autowired - private ISearchIncludeDao mySearchIncludeDao; - @Autowired - private ISearchResultDao mySearchResultDao; - @Autowired - private PlatformTransactionManager myTransactionManager; - - private void deleteSearch(final Long theSearchPid) { - mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> { - mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); - - /* - * Note, we're only deleting up to 500 results in an individual search here. This - * is to prevent really long running transactions in cases where there are - * huge searches with tons of results in them. By the time we've gotten here - * we have marked the parent Search entity as deleted, so it's not such a - * huge deal to be only partially deleting search results. They'll get deleted - * eventually - */ - int max = ourMaximumResultsToDeleteInOnePass; - Slice resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId()); - if (resultPids.hasContent()) { - List> partitions = Lists.partition(resultPids.getContent(), ourMaximumResultsToDeleteInOneStatement); - for (List nextPartition : partitions) { - mySearchResultDao.deleteByIds(nextPartition); - } - - } - - // Only delete if we don't have results left in this search - if (resultPids.getNumberOfElements() < max) { - ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned())); - mySearchDao.deleteByPid(searchToDelete.getId()); - } else { - ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid()); - } - }); - } + private ISearchResultCacheSvc mySearchResultCacheSvc; @Override @Transactional(propagation = Propagation.NEVER) public void pollForStaleSearchesAndDeleteThem() { - if (!myDaoConfig.isExpireSearchResults()) { - return; - } - - long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis(); - if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) { - cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis()); - } - final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack); - - if (ourNowForUnitTests != null) { - ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now()))); - } - - ourLog.debug("Searching for searches which are before {}", cutoff); - - TransactionTemplate tt = new TransactionTemplate(myTransactionManager); - final Slice toDelete = tt.execute(theStatus -> - mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 2000)) - ); - for (final Long nextSearchToDelete : toDelete) { - ourLog.debug("Deleting search with PID {}", nextSearchToDelete); - tt.execute(t -> { - mySearchDao.updateDeleted(nextSearchToDelete, true); - return null; - }); - - tt.execute(t -> { - deleteSearch(nextSearchToDelete); - return null; - }); - } - - int count = toDelete.getContent().size(); - if (count > 0) { - if (ourLog.isDebugEnabled()) { - long total = tt.execute(t -> mySearchDao.count()); - ourLog.debug("Deleted {} searches, {} remaining", count, total); - } - } - + mySearchResultCacheSvc.pollForStaleSearchesAndDeleteThem(); } @Scheduled(fixedDelay = DEFAULT_CUTOFF_SLACK) @@ -164,35 +58,4 @@ public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc { pollForStaleSearchesAndDeleteThem(); } } - - @VisibleForTesting - public void setCutoffSlackForUnitTest(long theCutoffSlack) { - myCutoffSlack = theCutoffSlack; - } - - @VisibleForTesting - public static void setMaximumResultsToDeleteInOnePassForUnitTest(int theMaximumResultsToDeleteInOnePass) { - ourMaximumResultsToDeleteInOnePass = theMaximumResultsToDeleteInOnePass; - } - - @VisibleForTesting - public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) { - ourMaximumResultsToDeleteInOneStatement = theMaximumResultsToDelete; - } - - private static long now() { - if (ourNowForUnitTests != null) { - return ourNowForUnitTests; - } - return System.currentTimeMillis(); - } - - /** - * This is for unit tests only, do not call otherwise - */ - @VisibleForTesting - public static void setNowForUnitTests(Long theNowForUnitTests) { - ourNowForUnitTests = theNowForUnitTests; - } - } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java index 73ff5a2d546..4d1853b04e5 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java @@ -1,4 +1,45 @@ package ca.uhn.fhir.jpa.search.cache; +import ca.uhn.fhir.jpa.entity.Search; +import org.apache.commons.lang3.time.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + public abstract class BaseSearchResultCacheSvcImpl implements ISearchResultCacheSvc { + + @Autowired + private PlatformTransactionManager myTxManager; + + private ConcurrentHashMap myUnsyncedLastUpdated = new ConcurrentHashMap<>(); + + @Override + public void updateSearchLastReturned(Search theSearch, Date theDate) { + myUnsyncedLastUpdated.put(theSearch.getId(), theDate); + } + + + @Override + @Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND) + public void flushLastUpdated() { + TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); + txTemplate.execute(t->{ + for (Iterator> iter = myUnsyncedLastUpdated.entrySet().iterator(); iter.hasNext(); ) { + Map.Entry next = iter.next(); + flushLastUpdated(next.getKey(), next.getValue()); + iter.remove(); + } + return null; + }); + } + + protected abstract void flushLastUpdated(Long theSearchId, Date theLastUpdated); + + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index 603d683f0f8..c72a8f9ea44 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -1,19 +1,288 @@ package ca.uhn.fhir.jpa.search.cache; -import ca.uhn.fhir.jpa.dao.data.IAAAAAAAAASearchDao; +import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.entity.SearchInclude; +import ca.uhn.fhir.jpa.entity.SearchResult; +import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.time.DateUtils; +import org.hl7.fhir.dstu3.model.InstantType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import javax.transaction.Transactional; +import java.util.*; + +import static ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.toPage; public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcImpl { + /* + * Be careful increasing this number! We use the number of params here in a + * DELETE FROM foo WHERE params IN (aaaa) + * type query and this can fail if we have 1000s of params + */ + public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500; + public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000; + public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND; + private static final Logger ourLog = LoggerFactory.getLogger(DatabaseSearchResultCacheSvcImpl.class); + private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT; + private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS; + private static Long ourNowForUnitTests; + /* + * We give a bit of extra leeway just to avoid race conditions where a query result + * is being reused (because a new client request came in with the same params) right before + * the result is to be deleted + */ + private long myCutoffSlack = DEFAULT_CUTOFF_SLACK; @Autowired - private IAAAAAAAAASearchDao mySearchDao; + private ISearchDao mySearchDao; + @Autowired + private ISearchIncludeDao mySearchIncludeDao; + @Autowired + private ISearchResultDao mySearchResultDao; + @Autowired + private PlatformTransactionManager myTxManager; + @Autowired + private DaoConfig myDaoConfig; - @Transactional(Transactional.TxType.MANDATORY) + @VisibleForTesting + public void setCutoffSlackForUnitTest(long theCutoffSlack) { + myCutoffSlack = theCutoffSlack; + } + + @Transactional(Transactional.TxType.REQUIRED) @Override - public Search saveNew(Search theSearch) { - return mySearchDao.save(theSearch); + public Search save(Search theSearch) { + Search newSearch; + if (theSearch.getId() == null) { + newSearch = mySearchDao.save(theSearch); + for (SearchInclude next : theSearch.getIncludes()) { + mySearchIncludeDao.save(next); + } + } else { + newSearch = mySearchDao.save(theSearch); + } + return newSearch; + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public Optional fetchByUuid(String theUuid) { + Validate.notBlank(theUuid); + return mySearchDao.findByUuidAndFetchIncludes(theUuid); + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public List fetchResultPids(Search theSearch, int theFrom, int theTo) { + final Pageable page = toPage(theFrom, theTo); + if (page == null) { + return Collections.emptyList(); + } + + List retVal = mySearchResultDao + .findWithSearchUuid(theSearch.getId(), page) + .getContent(); + + ourLog.trace("fetchResultPids for range {}-{} returned {} pids", theFrom, theTo, retVal.size()); + + return new ArrayList<>(retVal); + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public List fetchAllResultPids(Search theSearch) { + List retVal = mySearchResultDao.findWithSearchUuidOrderIndependent(theSearch.getId()); + ourLog.trace("fetchAllResultPids returned {} pids", retVal.size()); + return retVal; + } + + @Override + @Transactional(Transactional.TxType.NEVER) + public Optional tryToMarkSearchAsInProgress(Search theSearch) { + ourLog.trace("Going to try to change search status from {} to {}", theSearch.getStatus(), SearchStatusEnum.LOADING); + try { + TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + txTemplate.afterPropertiesSet(); + return txTemplate.execute(t -> { + Search search = mySearchDao.findById(theSearch.getId()).orElse(theSearch); + + if (search.getStatus() != SearchStatusEnum.PASSCMPLET) { + throw new IllegalStateException("Can't change to LOADING because state is " + theSearch.getStatus()); + } + search.setStatus(SearchStatusEnum.LOADING); + Search newSearch = mySearchDao.save(search); + return Optional.of(newSearch); + }); + } catch (Exception e) { + ourLog.warn("Failed to activate search: {}", e.toString()); + ourLog.trace("Failed to activate search", e); + return Optional.empty(); + } + } + + @Override + public Collection findCandidatesForReuse(String theResourceType, String theQueryString, Date theCreatedAfter) { + int hashCode = theQueryString.hashCode(); + return mySearchDao.find(theResourceType, hashCode, theCreatedAfter); + + } + + @Transactional(Transactional.TxType.NEVER) + @Override + public void pollForStaleSearchesAndDeleteThem() { + if (!myDaoConfig.isExpireSearchResults()) { + return; + } + + long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis(); + if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) { + cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis()); + } + final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack); + + if (ourNowForUnitTests != null) { + ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now()))); + } + + ourLog.debug("Searching for searches which are before {}", cutoff); + + TransactionTemplate tt = new TransactionTemplate(myTxManager); + final Slice toDelete = tt.execute(theStatus -> + mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 2000)) + ); + for (final Long nextSearchToDelete : toDelete) { + ourLog.debug("Deleting search with PID {}", nextSearchToDelete); + tt.execute(t -> { + mySearchDao.updateDeleted(nextSearchToDelete, true); + return null; + }); + + tt.execute(t -> { + deleteSearch(nextSearchToDelete); + return null; + }); + } + + int count = toDelete.getContent().size(); + if (count > 0) { + if (ourLog.isDebugEnabled()) { + long total = tt.execute(t -> mySearchDao.count()); + ourLog.debug("Deleted {} searches, {} remaining", count, total); + } + } + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public void storeResults(Search theSearch, List thePreviouslyStoredResourcePids, List theNewResourcePids) { + List resultsToSave = Lists.newArrayList(); + + ourLog.trace("Storing {} results with {} previous for search", theNewResourcePids.size(), thePreviouslyStoredResourcePids.size()); + + int order = thePreviouslyStoredResourcePids.size(); + for (Long nextPid : theNewResourcePids) { + SearchResult nextResult = new SearchResult(theSearch); + nextResult.setResourcePid(nextPid); + nextResult.setOrder(order); + resultsToSave.add(nextResult); + ourLog.trace("Saving ORDER[{}] Resource {}", order, nextResult.getResourcePid()); + + order++; + } + + mySearchResultDao.saveAll(resultsToSave); + } + + @Override + protected void flushLastUpdated(Long theSearchId, Date theLastUpdated) { + mySearchDao.updateSearchLastReturned(theSearchId, theLastUpdated); + } + + @VisibleForTesting + void setSearchDaoForUnitTest(ISearchDao theSearchDao) { + mySearchDao = theSearchDao; + } + + @VisibleForTesting + void setSearchDaoIncludeForUnitTest(ISearchIncludeDao theSearchIncludeDao) { + mySearchIncludeDao = theSearchIncludeDao; + } + + @VisibleForTesting + void setSearchDaoResultForUnitTest(ISearchResultDao theSearchResultDao) { + mySearchResultDao = theSearchResultDao; + } + + private void deleteSearch(final Long theSearchPid) { + mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> { + mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); + + /* + * Note, we're only deleting up to 500 results in an individual search here. This + * is to prevent really long running transactions in cases where there are + * huge searches with tons of results in them. By the time we've gotten here + * we have marked the parent Search entity as deleted, so it's not such a + * huge deal to be only partially deleting search results. They'll get deleted + * eventually + */ + int max = ourMaximumResultsToDeleteInOnePass; + Slice resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId()); + if (resultPids.hasContent()) { + List> partitions = Lists.partition(resultPids.getContent(), ourMaximumResultsToDeleteInOneStatement); + for (List nextPartition : partitions) { + mySearchResultDao.deleteByIds(nextPartition); + } + + } + + // Only delete if we don't have results left in this search + if (resultPids.getNumberOfElements() < max) { + ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned())); + mySearchDao.deleteByPid(searchToDelete.getId()); + } else { + ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid()); + } + }); + } + + @VisibleForTesting + public static void setMaximumResultsToDeleteInOnePassForUnitTest(int theMaximumResultsToDeleteInOnePass) { + ourMaximumResultsToDeleteInOnePass = theMaximumResultsToDeleteInOnePass; + } + + @VisibleForTesting + public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) { + ourMaximumResultsToDeleteInOneStatement = theMaximumResultsToDelete; + } + + /** + * This is for unit tests only, do not call otherwise + */ + @VisibleForTesting + public static void setNowForUnitTests(Long theNowForUnitTests) { + ourNowForUnitTests = theNowForUnitTests; + } + + private static long now() { + if (ourNowForUnitTests != null) { + return ourNowForUnitTests; + } + return System.currentTimeMillis(); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java index fbe808b0a17..0cfc23bb36e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java @@ -2,13 +2,104 @@ package ca.uhn.fhir.jpa.search.cache; import ca.uhn.fhir.jpa.entity.Search; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Optional; + public interface ISearchResultCacheSvc { /** - * Places a new search of some sort in the cache. + * Places a new search of some sort in the cache, or updates an existing search. The search passed in is guaranteed to have + * a {@link Search#getUuid() UUID} so that is a good candidate for consistent identification. * * @param theSearch The search to store * @return Returns a copy of the search as it was saved. Callers should use the returned Search object for any further processing. */ - Search saveNew(Search theSearch); + Search save(Search theSearch); + + /** + * Fetch a search using its UUID. The search should be fully loaded when it is returned (i.e. includes are fetched, so that access to its + * fields will not cause database errors if the current tranaction scope ends. + * + * @param theUuid The search UUID + * @return The search if it exists + */ + Optional fetchByUuid(String theUuid); + + /** + * Fetch a sunset of the search result IDs from the cache + * + * @param theSearch The search to fetch IDs for + * @param theFrom The starting index (inclusive) + * @param theTo The ending index (exclusive) + * @return A list of resource PIDs + */ + List fetchResultPids(Search theSearch, int theFrom, int theTo); + + /** + * Fetch all result PIDs for a given search with no particular order required + * @param theSearch + * @return + */ + List fetchAllResultPids(Search theSearch); + + /** + * TODO: this is perhaps an inappropriate responsibility for this service + * + *

+ * This method marks a search as in progress, but should only allow exactly one call to do so across the cluster. This + * is done so that if two client threads request the next page at the exact same time (which is unlikely but not + * impossible) only one will actually proceed to load the next results and the other will just wait for them + * to arrive. + * + * @param theSearch The search to mark + * @return This method should return an empty optional if the search was not marked (meaning that another thread + * succeeded in marking it). If the search doesn't exist or some other error occurred, an exception will be thrown + * instead of {@link Optional#empty()} + */ + Optional tryToMarkSearchAsInProgress(Search theSearch); + + /** + * Look for any existing searches matching the given resource type and query string. + *

+ * This method is allowed to perofrm a "best effort" return, so it can return searches that don't match the query string exactly, or + * which have a created timestamp before theCreatedAfter date. The caller is responsible for removing + * any inappropriate Searches and picking the most relevant one. + *

+ * + * @param theResourceType The resource type of the search. Results MUST match this type + * @param theQueryString The query string. Results SHOULD match this type + * @param theCreatedAfter Results SHOULD not include any searches created before this cutoff timestamp + * @return A collection of candidate searches + */ + Collection findCandidatesForReuse(String theResourceType, String theQueryString, Date theCreatedAfter); + + /** + * Mark a search as having been "last used" at the given time. This method may (and probably should) be implemented + * to work asynchronously in order to avoid hammering the database if the search gets reused many times. + * + * @param theSearch The search + * @param theDate The "last returned" timestamp + */ + void updateSearchLastReturned(Search theSearch, Date theDate); + + /** + * This is mostly public for unit tests + */ + void flushLastUpdated(); + + /** + * This method will be called periodically to delete stale searches. Implementations are not required to do anything + * if they have some other mechanism for expiring stale results other than manually looking for them + * and deleting them. + */ + void pollForStaleSearchesAndDeleteThem(); + + /** + * @param theSearch The search - This method is not required to persist any chances to the Search object, it is only provided here for identification + * @param thePreviouslyStoredResourcePids A list of resource PIDs that have previously been saved to this search + * @param theNewResourcePids A list of new resoure PIDs to add to this search (these ones have not been previously saved) + */ + void storeResults(Search theSearch, List thePreviouslyStoredResourcePids, List theNewResourcePids); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java index 597a012ab4c..8461872f6e0 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java @@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.provider.SystemProviderDstu2Test; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; import ca.uhn.fhir.jpa.term.VersionIndependentConcept; @@ -91,6 +92,8 @@ public abstract class BaseJpaTest { protected IInterceptorService myInterceptorRegistry; @Autowired protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; + @Autowired + protected ISearchResultCacheSvc mySearchResultCacheSvc; @After public void afterPerformCleanup() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java index edd63ceb41a..a21515375f6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/BaseJpaDstu3Test.java @@ -206,8 +206,6 @@ public abstract class BaseJpaDstu3Test extends BaseJpaTest { @Autowired(required = false) protected IFulltextSearchSvc mySearchDao; @Autowired - protected ISearchDao mySearchEntityDao; - @Autowired @Qualifier("mySearchParameterDaoDstu3") protected IFhirResourceDao mySearchParameterDao; @Autowired diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchPageExpiryTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchPageExpiryTest.java deleted file mode 100644 index 779e8175506..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchPageExpiryTest.java +++ /dev/null @@ -1,401 +0,0 @@ -package ca.uhn.fhir.jpa.dao.dstu3; - -import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; -import ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoR4SearchPageExpiryTest; -import ca.uhn.fhir.jpa.entity.Search; -import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; -import ca.uhn.fhir.jpa.util.TestUtil; -import ca.uhn.fhir.util.StopWatch; -import ca.uhn.fhir.rest.api.server.IBundleProvider; -import ca.uhn.fhir.rest.param.StringParam; -import org.apache.commons.lang3.Validate; -import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.dstu3.model.InstantType; -import org.hl7.fhir.dstu3.model.Patient; -import org.hl7.fhir.instance.model.api.IIdType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.lang.Nullable; -import org.springframework.test.util.AopTestUtils; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; -import org.springframework.transaction.support.TransactionTemplate; - -import java.util.Date; -import java.util.concurrent.atomic.AtomicLong; - -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.*; - -public class FhirResourceDaoDstu3SearchPageExpiryTest extends BaseJpaDstu3Test { - private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoDstu3SearchPageExpiryTest.class); - - @After() - public void after() { - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_CUTOFF_SLACK); - StaleSearchDeletingSvcImpl.setNowForUnitTests(null); - - myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis()); - myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults()); - } - - @Before - public void before() { - myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis()); - myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults()); - myDaoConfig.setCountSearchResultsUpTo(null); - - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); - } - - @Before - public void beforeDisableResultReuse() { - myDaoConfig.setReuseCachedSearchResultsForMillis(null); - } - - @Test - public void testExpirePagesAfterReuse() throws Exception { - IIdType pid1; - IIdType pid2; - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - - myDaoConfig.setExpireSearchResultsAfterMillis(1000L); - myDaoConfig.setReuseCachedSearchResultsForMillis(500L); - long start = System.currentTimeMillis(); - - final String searchUuid1; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid1 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid1); - } - - waitForSearchToSave(searchUuid1); - - final String searchUuid2; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid2 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid2); - } - assertEquals(searchUuid1, searchUuid2); - - TestUtil.sleepAtLeast(501); - // We're now past 500ms so we shouldn't reuse the search - - final String searchUuid3; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid3 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid3); - } - assertNotEquals(searchUuid1, searchUuid3); - - // Search just got used so it shouldn't be deleted - - StaleSearchDeletingSvcImpl.setNowForUnitTests(start + 500); - final AtomicLong search3timestamp = new AtomicLong(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search3 = mySearchEntityDao.findByUuid(searchUuid3); - assertNotNull(search3); - Search search2 = mySearchEntityDao.findByUuid(searchUuid2); - assertNotNull(search2); - search3timestamp.set(search2.getSearchLastReturned().getTime()); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid3)); - } - }); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid1)); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNotNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100); - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); - } - }); - - } - - @Test - public void testExpirePagesAfterSingleUse() throws Exception { - IIdType pid1; - IIdType pid2; - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - - final StopWatch sw = new StopWatch(); - - SearchParameterMap params; - params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - - waitForSearchToSave(bundleProvider.getUuid()); - final AtomicLong start = new AtomicLong(); - - TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); - assertNotNull("Failed after " + sw.toString(), search); - start.set(search.getCreated().getTime()); - ourLog.info("Search was created: {}", new InstantType(new Date(start.get()))); - } - }); - - myDaoConfig.setExpireSearchResultsAfterMillis(500); - myDaoConfig.setReuseCachedSearchResultsForMillis(500L); - StaleSearchDeletingSvcImpl.setNowForUnitTests(start.get() + 499); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(bundleProvider.getUuid())); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(start.get() + 600); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - txTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull(mySearchEntityDao.findByUuid(bundleProvider.getUuid())); - } - }); - } - - @Test - public void testExpirePagesAfterSingleUse2() throws Exception { - IIdType pid1; - IIdType pid2; - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - Thread.sleep(10); - - myDaoConfig.setExpireSearchResultsAfterMillis(1000L); - myDaoConfig.setReuseCachedSearchResultsForMillis(500L); - long start = System.currentTimeMillis(); - - final String searchUuid1; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid1 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid1); - } - - String searchUuid2; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid2 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid2); - } - assertEquals(searchUuid1, searchUuid2); - - TestUtil.sleepAtLeast(501); - - // We're now past 500ms so we shouldn't reuse the search - - final String searchUuid3; - { - SearchParameterMap params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - final IBundleProvider bundleProvider = myPatientDao.search(params); - assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); - searchUuid3 = bundleProvider.getUuid(); - Validate.notBlank(searchUuid3); - } - assertNotEquals(searchUuid1, searchUuid3); - - // Search just got used so it shouldn't be deleted - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - final AtomicLong search3timestamp = new AtomicLong(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search3 = mySearchEntityDao.findByUuid(searchUuid3); - assertNotNull(search3); - search3timestamp.set(search3.getCreated().getTime()); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid3)); - } - }); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull(mySearchEntityDao.findByUuid(searchUuid1)); - } - }); - - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); - } - }); - - } - - @Test - public void testSearchPagesExpiryDisabled() throws Exception { - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - { - Patient patient = new Patient(); - patient.addName().setFamily("EXPIRE"); - myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); - } - - SearchParameterMap params; - params = new SearchParameterMap(); - params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); - params.setCount(1); - final IBundleProvider bundleProvider = myPatientDao.search(params); - - Search search = null; - for (int i = 0; i < 100 && search == null; i++) { - search = newTxTemplate().execute(new TransactionCallback() { - @Nullable - @Override - public Search doInTransaction(TransactionStatus status) { - return mySearchEntityDao.findByUuid(bundleProvider.getUuid()); - } - }); - if (search == null) { - TestUtil.sleepAtLeast(100); - } - } - assertNotNull("Search " + bundleProvider.getUuid() + " not found on disk after 10 seconds", search); - - - myDaoConfig.setExpireSearchResults(false); - StaleSearchDeletingSvcImpl.setNowForUnitTests(System.currentTimeMillis() + DateUtils.MILLIS_PER_DAY); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); - assertNotNull(search); - } - }); - - myDaoConfig.setExpireSearchResults(true); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - - newTxTemplate().execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); - assertNull(search); - } - }); - - } - - private void waitForSearchToSave(final String theUuid) { - final ISearchDao searchEntityDao = mySearchEntityDao; - TransactionTemplate txTemplate = newTxTemplate(); - FhirResourceDaoR4SearchPageExpiryTest.waitForSearchToSave(theUuid, searchEntityDao, txTemplate); - } -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java index d7d26c21eb3..fd771372875 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java @@ -251,12 +251,6 @@ public abstract class BaseJpaR4Test extends BaseJpaTest { @Autowired(required = false) protected IFulltextSearchSvc mySearchDao; @Autowired - protected ISearchDao mySearchEntityDao; - @Autowired - protected ISearchResultDao mySearchResultDao; - @Autowired - protected ISearchIncludeDao mySearchIncludeDao; - @Autowired protected IResourceReindexJobDao myResourceReindexJobDao; @Autowired @Qualifier("mySearchParameterDaoR4") diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index 6c9c0cd571a..7c18e672068 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; @@ -55,6 +56,8 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { @Autowired MatchUrlService myMatchUrlService; + @Autowired + private ISearchDao mySearchEntityDao; @After public void afterResetSearchSize() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java index f7257a82f63..826a387cd6f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.dao.r4; -import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; @@ -30,6 +30,7 @@ import org.hl7.fhir.r4.model.Observation.ObservationStatus; import org.hl7.fhir.r4.model.Subscription.SubscriptionChannelType; import org.hl7.fhir.r4.model.Subscription.SubscriptionStatus; import org.junit.*; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionCallbackWithoutResult; @@ -49,6 +50,8 @@ import static org.mockito.Mockito.mock; @SuppressWarnings({"unchecked", "Duplicates"}) public class FhirResourceDaoR4SearchNoHashesTest extends BaseJpaR4Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchNoHashesTest.class); + @Autowired + private ISearchDao mySearchEntityDao; @After public void afterResetSearchSize() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index de55c65a11b..6d0931d484d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -1,6 +1,8 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; @@ -14,6 +16,7 @@ import ca.uhn.fhir.rest.param.ReferenceOrListParam; import ca.uhn.fhir.rest.param.ReferenceParam; import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.util.TestUtil; import org.apache.commons.lang3.StringUtils; import org.hl7.fhir.instance.model.api.IAnyResource; @@ -24,6 +27,7 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; import org.springframework.aop.framework.AopProxyUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.concurrent.ThreadPoolExecutorFactoryBean; import org.springframework.test.context.TestPropertySource; @@ -48,6 +52,10 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchOptimizedTest.class); private SearchCoordinatorSvcImpl mySearchCoordinatorSvcImpl; + @Autowired + private ISearchDao mySearchEntityDao; + @Autowired + private ISearchResultDao mySearchResultDao; @Before public void before() { @@ -288,7 +296,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(200, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertNull(search.getTotalCount()); @@ -307,7 +315,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search gets incremented twice as a part of loading the next batch */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(SearchStatusEnum.FINISHED, search.getStatus()); assertEquals(200, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); @@ -343,7 +351,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(20, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertNull(search.getTotalCount()); @@ -367,7 +375,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search should be untouched */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(1, search.getVersion().intValue()); }); @@ -383,7 +391,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search gets incremented twice as a part of loading the next batch */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(SearchStatusEnum.PASSCMPLET, search.getStatus()); assertEquals(50, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); @@ -407,7 +415,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search should be untouched */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(3, search.getVersion().intValue()); }); @@ -423,7 +431,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search gets incremented twice as a part of loading the next batch */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(190, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertEquals(190, search.getTotalCount().intValue()); @@ -470,7 +478,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(50, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertEquals(null, search.getTotalCount()); @@ -505,7 +513,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(20, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertNull(search.getTotalCount()); @@ -529,7 +537,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * Search should be untouched */ runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(200, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertEquals(200, search.getTotalCount().intValue()); @@ -562,9 +570,9 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { * 20 should be prefetched since that's the initial page size */ - waitForSize(20, () -> runInTransaction(() -> mySearchEntityDao.findByUuid(uuid).getNumFound())); + waitForSize(20, () -> runInTransaction(() -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")).getNumFound())); runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(20, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); assertNull(search.getTotalCount()); @@ -625,7 +633,7 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { assertEquals(1, ids.size()); runInTransaction(() -> { - Search search = mySearchEntityDao.findByUuid(uuid); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException("")); assertEquals(SearchStatusEnum.FINISHED, search.getStatus()); assertEquals(1, search.getNumFound()); assertEquals(search.getNumFound(), mySearchResultDao.count()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java index 17b3bf93c4a..af443190438 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java @@ -1,15 +1,16 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; -import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.TestUtil; -import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.StringParam; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.util.StopWatch; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.instance.model.api.IIdType; @@ -20,6 +21,7 @@ import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; @@ -30,6 +32,7 @@ import javax.annotation.Nullable; import java.util.Date; import java.util.concurrent.atomic.AtomicLong; +import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.*; @@ -37,16 +40,19 @@ import static org.junit.Assert.*; public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { private static final Logger ourLog = LoggerFactory.getLogger(FhirResourceDaoR4SearchPageExpiryTest.class); + @Autowired + private ISearchDao mySearchEntityDao; + @After() public void after() { - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_CUTOFF_SLACK); - StaleSearchDeletingSvcImpl.setNowForUnitTests(null); + DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); + staleSearchDeletingSvc.setCutoffSlackForUnitTest(DEFAULT_CUTOFF_SLACK); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(null); } @Before public void before() { - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); + DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); myDaoConfig.setCountSearchResultsUpTo(new DaoConfig().getCountSearchResultsUpTo()); } @@ -77,7 +83,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResultsAfterMillis(1000L); myDaoConfig.setReuseCachedSearchResultsForMillis(500L); long start = System.currentTimeMillis(); - StaleSearchDeletingSvcImpl.setNowForUnitTests(start); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start); final String searchUuid1; { @@ -117,53 +123,53 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { // Search just got used so it shouldn't be deleted - StaleSearchDeletingSvcImpl.setNowForUnitTests(start + 500); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start + 500); final AtomicLong search3timestamp = new AtomicLong(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search3 = mySearchEntityDao.findByUuid(searchUuid3); + Search search3 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).orElseThrow(()->new InternalErrorException("Search doesn't exist")); assertNotNull(search3); - Search search2 = mySearchEntityDao.findByUuid(searchUuid2); + Search search2 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid2).orElseThrow(()->new InternalErrorException("Search doesn't exist")); assertNotNull(search2); search3timestamp.set(search2.getSearchLastReturned().getTime()); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid3)); + assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3)); } }); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid1)); + assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1)); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNotNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); + assertFalse("Search 1 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent()); + assertTrue("Search 3 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent()); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); + assertFalse("Search 1 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent()); + assertFalse("Search 3 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent()); } }); @@ -202,7 +208,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { txTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid()).orElseThrow(()->new InternalErrorException("Search doesn't exist")); assertNotNull("Failed after " + sw.toString(), search); start.set(search.getCreated().getTime()); ourLog.info("Search was created: {}", new InstantType(new Date(start.get()))); @@ -211,21 +217,21 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResultsAfterMillis(500); myDaoConfig.setReuseCachedSearchResultsForMillis(500L); - StaleSearchDeletingSvcImpl.setNowForUnitTests(start.get() + 499); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start.get() + 499); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); txTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(bundleProvider.getUuid())); + assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid())); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(start.get() + 600); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start.get() + 600); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); txTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull(mySearchEntityDao.findByUuid(bundleProvider.getUuid())); + assertFalse(mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid()).isPresent()); } }); } @@ -296,36 +302,36 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search3 = mySearchEntityDao.findByUuid(searchUuid3); + Search search3 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).orElseThrow(()->new InternalErrorException("Search doesn't exist")); assertNotNull(search3); search3timestamp.set(search3.getCreated().getTime()); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNotNull(mySearchEntityDao.findByUuid(searchUuid3)); + assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3)); } }); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull(mySearchEntityDao.findByUuid(searchUuid1)); + assertFalse(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent()); } }); - StaleSearchDeletingSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - assertNull("Search 1 still exists", mySearchEntityDao.findByUuid(searchUuid1)); - assertNull("Search 3 still exists", mySearchEntityDao.findByUuid(searchUuid3)); + assertFalse("Search 1 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent()); + assertFalse("Search 3 still exists", mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent()); } }); @@ -356,7 +362,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { @Nullable @Override public Search doInTransaction(TransactionStatus status) { - return mySearchEntityDao.findByUuid(bundleProvider.getUuid()); + return mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid()).orElse(null); } }); if (search == null) { @@ -367,13 +373,13 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResults(false); - StaleSearchDeletingSvcImpl.setNowForUnitTests(System.currentTimeMillis() + DateUtils.MILLIS_PER_DAY); + DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(System.currentTimeMillis() + DateUtils.MILLIS_PER_DAY); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid()).orElseThrow(()->new InternalErrorException("Search doesn't exist")); assertNotNull(search); } }); @@ -386,7 +392,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search search = mySearchEntityDao.findByUuid(bundleProvider.getUuid()); + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(bundleProvider.getUuid()).orElse(null); assertNull(search); } }); @@ -405,7 +411,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { protected void doInTransactionWithoutResult(TransactionStatus theArg0) { Search search = null; for (int i = 0; i < 20 && search == null; i++) { - search = theSearchEntityDao.findByUuid(theUuid); + search = theSearchEntityDao.findByUuidAndFetchIncludes(theUuid).orElse(null); if (search == null || search.getStatus() == SearchStatusEnum.LOADING) { TestUtil.sleepAtLeast(100); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java index f709df0beb8..0a74bbc1ce6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParamConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.TestUtil; @@ -44,6 +45,7 @@ import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; import org.hl7.fhir.r4.model.OperationOutcome.IssueType; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; import org.junit.*; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; @@ -3860,7 +3862,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test { search.setStatus(SearchStatusEnum.FAILED); search.setFailureCode(500); search.setFailureMessage("FOO"); - mySearchEntityDao.save(search); + mySearchResultCacheSvc.save(search); }); IBundleProvider results = myEncounterDao.search(map); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java index b4c44fa3eb7..15c56df0035 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java @@ -249,12 +249,6 @@ public abstract class BaseJpaR5Test extends BaseJpaTest { @Autowired(required = false) protected IFulltextSearchSvc mySearchDao; @Autowired - protected ISearchDao mySearchEntityDao; - @Autowired - protected ISearchResultDao mySearchResultDao; - @Autowired - protected ISearchIncludeDao mySearchIncludeDao; - @Autowired protected IResourceReindexJobDao myResourceReindexJobDao; @Autowired @Qualifier("mySearchParameterDaoR5") diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/BaseResourceProviderDstu2Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/BaseResourceProviderDstu2Test.java index e386a885eac..93ce55c988d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/BaseResourceProviderDstu2Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/BaseResourceProviderDstu2Test.java @@ -13,6 +13,7 @@ import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.test.utilities.JettyUtil; import ca.uhn.fhir.util.TestUtil; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -36,8 +37,6 @@ import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import ca.uhn.fhir.test.utilities.JettyUtil; - public abstract class BaseResourceProviderDstu2Test extends BaseJpaDstu2Test { protected static IGenericClient ourClient; @@ -61,12 +60,12 @@ public abstract class BaseResourceProviderDstu2Test extends BaseJpaDstu2Test { myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.ONCE); } - @SuppressWarnings({ "unchecked", "rawtypes" }) + @SuppressWarnings({"unchecked", "rawtypes"}) @Before public void before() throws Exception { myFhirCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); myFhirCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000); - + if (ourServer == null) { ourRestServer = new RestfulServer(myFhirCtx); ourRestServer.registerProviders(myResourceProviders.createProviders()); @@ -82,7 +81,7 @@ public abstract class BaseResourceProviderDstu2Test extends BaseJpaDstu2Test { ourConnectionPoolSize = myAppCtx.getBean("maxDatabaseThreadsForTest", Integer.class); ourRestServer.setPagingProvider(ourPagingProvider); - Server server = new Server(ourPort); + Server server = new Server(0); ServletContextHandler proxyHandler = new ServletContextHandler(); proxyHandler.setContextPath("/"); @@ -103,16 +102,14 @@ public abstract class BaseResourceProviderDstu2Test extends BaseJpaDstu2Test { dispatcherServlet.setContextClass(AnnotationConfigWebApplicationContext.class); ServletHolder subsServletHolder = new ServletHolder(); subsServletHolder.setServlet(dispatcherServlet); - subsServletHolder.setInitParameter( - ContextLoader.CONFIG_LOCATION_PARAM, - WebsocketDispatcherConfig.class.getName()); + subsServletHolder.setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, WebsocketDispatcherConfig.class.getName()); proxyHandler.addServlet(subsServletHolder, "/*"); server.setHandler(proxyHandler); JettyUtil.startServer(server); - ourPort = JettyUtil.getPortForStartedServer(server); - ourServerBase = "http://localhost:" + ourPort + "/fhir/context"; + ourPort = JettyUtil.getPortForStartedServer(server); + ourServerBase = "http://localhost:" + ourPort + "/fhir/context"; ourClient = myFhirCtx.newRestfulGenericClient(ourServerBase); ourClient.registerInterceptor(new LoggingInterceptor()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java index a0a0cdfa278..891be247054 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java @@ -1,7 +1,6 @@ package ca.uhn.fhir.jpa.provider.dstu3; import ca.uhn.fhir.jpa.config.WebsocketDispatcherConfig; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.dstu3.BaseJpaDstu3Test; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.provider.SubscriptionTriggeringProvider; @@ -18,6 +17,7 @@ import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.rest.server.interceptor.CorsInterceptor; +import ca.uhn.fhir.test.utilities.JettyUtil; import ca.uhn.fhir.util.TestUtil; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -48,8 +48,6 @@ import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import ca.uhn.fhir.test.utilities.JettyUtil; - public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test { @@ -62,7 +60,6 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test { protected static GenericWebApplicationContext ourWebApplicationContext; protected static SearchParamRegistryDstu3 ourSearchParamRegistry; protected static DatabaseBackedPagingProvider ourPagingProvider; - protected static ISearchDao mySearchEntityDao; protected static ISearchCoordinatorSvc ourSearchCoordinatorSvc; private static Server ourServer; protected static SubscriptionTriggeringProvider ourSubscriptionTriggeringProvider; @@ -156,7 +153,6 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test { WebApplicationContext wac = WebApplicationContextUtils.getWebApplicationContext(subsServletHolder.getServlet().getServletConfig().getServletContext()); myValidationSupport = wac.getBean(JpaValidationSupportChainDstu3.class); ourSearchCoordinatorSvc = wac.getBean(ISearchCoordinatorSvc.class); - mySearchEntityDao = wac.getBean(ISearchDao.class); ourSearchParamRegistry = wac.getBean(SearchParamRegistryDstu3.class); ourSubscriptionTriggeringProvider = wac.getBean(SubscriptionTriggeringProvider.class); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index 0b6ea5d58b7..c004a6834f8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.jpa.provider.dstu3; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.provider.r4.ResourceProviderR4Test; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; @@ -18,10 +19,7 @@ import ca.uhn.fhir.rest.client.api.IHttpRequest; import ca.uhn.fhir.rest.client.api.IHttpResponse; import ca.uhn.fhir.rest.gclient.StringClientParam; import ca.uhn.fhir.rest.param.*; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; -import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; -import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.rest.server.exceptions.*; import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor; import ca.uhn.fhir.util.TestUtil; import ca.uhn.fhir.util.UrlUtil; @@ -51,7 +49,11 @@ import org.hl7.fhir.dstu3.model.Subscription.SubscriptionStatus; import org.hl7.fhir.instance.model.api.IBaseOperationOutcome; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Ignore; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; @@ -74,6 +76,8 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderDstu3Test.class); private SearchCoordinatorSvcImpl mySearchCoordinatorSvcRaw; + @Autowired + private ISearchDao mySearchEntityDao; @Override @After @@ -2969,12 +2973,13 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .count(5) .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(new TransactionCallback() { @Override public Search doInTransaction(TransactionStatus theStatus) { - return mySearchEntityDao.findByUuid(uuid1); + return mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException("")); } }); Date lastReturned1 = search1.getSearchLastReturned(); @@ -2986,12 +2991,13 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .count(5) .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); Search search2 = newTxTemplate().execute(new TransactionCallback() { @Override public Search doInTransaction(TransactionStatus theStatus) { - return mySearchEntityDao.findByUuid(uuid2); + return mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException("")); } }); Date lastReturned2 = search2.getSearchLastReturned(); @@ -3031,14 +3037,10 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); - Search search1 = newTxTemplate().execute(new TransactionCallback() { - @Override - public Search doInTransaction(TransactionStatus theStatus) { - return mySearchEntityDao.findByUuid(uuid1); - } - }); + Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException(""))); Date lastReturned1 = search1.getSearchLastReturned(); Bundle result2 = ourClient @@ -3046,14 +3048,10 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); - Search search2 = newTxTemplate().execute(new TransactionCallback() { - @Override - public Search doInTransaction(TransactionStatus theStatus) { - return mySearchEntityDao.findByUuid(uuid2); - } - }); + Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException(""))); Date lastReturned2 = search2.getSearchLastReturned(); assertTrue(lastReturned2.getTime() > lastReturned1.getTime()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/StaleSearchDeletingSvcDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/StaleSearchDeletingSvcDstu3Test.java deleted file mode 100644 index 2561e0841af..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/StaleSearchDeletingSvcDstu3Test.java +++ /dev/null @@ -1,97 +0,0 @@ -package ca.uhn.fhir.jpa.provider.dstu3; - -import static org.hamcrest.Matchers.blankOrNullString; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -import org.hl7.fhir.dstu3.model.Bundle; -import org.hl7.fhir.dstu3.model.Bundle.BundleLinkComponent; -import org.hl7.fhir.dstu3.model.Patient; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.springframework.test.util.AopTestUtils; - -import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; -import ca.uhn.fhir.rest.gclient.IClientExecutable; -import ca.uhn.fhir.rest.gclient.IQuery; -import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; -import ca.uhn.fhir.util.TestUtil; - -public class StaleSearchDeletingSvcDstu3Test extends BaseResourceProviderDstu3Test { - - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(StaleSearchDeletingSvcDstu3Test.class); - - @AfterClass - public static void afterClassClearContext() { - TestUtil.clearAllStaticFieldsForUnitTest(); - } - - - @After() - public void after() throws Exception { - super.after(); - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_CUTOFF_SLACK); - } - - @Before - public void before() throws Exception { - super.before(); - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); - } - - @Test - public void testEverythingInstanceWithContentFilter() throws Exception { - - for (int i = 0; i < 20; i++) { - Patient pt1 = new Patient(); - pt1.addName().setFamily("Everything").addGiven("Arthur"); - myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless(); - } - - //@formatter:off - IClientExecutable, Bundle> search = ourClient - .search() - .forResource(Patient.class) - .where(Patient.NAME.matches().value("Everything")) - .returnBundle(Bundle.class); - //@formatter:on - - Bundle resp1 = search.execute(); - - for (int i = 0; i < 20; i++) { - search.execute(); - } - - BundleLinkComponent nextLink = resp1.getLink("next"); - assertNotNull(nextLink); - String nextLinkUrl = nextLink.getUrl(); - assertThat(nextLinkUrl, not(blankOrNullString())); - - Bundle resp2 = ourClient.search().byUrl(nextLinkUrl).returnBundle(Bundle.class).execute(); - ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(resp2)); - - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - - ourClient.search().byUrl(nextLinkUrl).returnBundle(Bundle.class).execute(); - - Thread.sleep(20); - myDaoConfig.setExpireSearchResultsAfterMillis(10); - myDaoConfig.setReuseCachedSearchResultsForMillis(null); - myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); - - try { - ourClient.search().byUrl(nextLinkUrl).returnBundle(Bundle.class).execute(); - fail(); - } catch (ResourceGoneException e) { - assertThat(e.getMessage(), containsString("does not exist and may have expired")); - } - } - -} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java index abdd8dffb15..7cdf48c75c6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java @@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.config.WebsocketDispatcherConfig; import ca.uhn.fhir.jpa.dao.DaoRegistry; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ExpungeR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ExpungeR4Test.java index 9bc035957ab..182b39366d5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ExpungeR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ExpungeR4Test.java @@ -2,6 +2,8 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.IFhirResourceDao; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.search.PersistedJpaSearchFirstPageBundleProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.ExpungeOptions; @@ -21,6 +23,7 @@ import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import java.util.List; @@ -38,6 +41,10 @@ public class ExpungeR4Test extends BaseResourceProviderR4Test { private IIdType myOneVersionObservationId; private IIdType myTwoVersionObservationId; private IIdType myDeletedObservationId; + @Autowired + private ISearchDao mySearchEntityDao; + @Autowired + private ISearchResultDao mySearchResultDao; @After public void afterDisableExpunge() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CacheTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CacheTest.java index dd4eb173b2f..ca08514295b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CacheTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4CacheTest.java @@ -1,7 +1,7 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.util.TestUtil; import ca.uhn.fhir.parser.StrictErrorHandler; import ca.uhn.fhir.rest.api.CacheControlDirective; @@ -13,9 +13,8 @@ import org.hl7.fhir.r4.model.Patient; import org.junit.After; import org.junit.AfterClass; import org.junit.Test; -import org.springframework.test.util.AopTestUtils; +import org.springframework.beans.factory.annotation.Autowired; -import java.io.IOException; import java.util.Date; import static org.hamcrest.Matchers.*; @@ -24,9 +23,9 @@ import static org.junit.Assert.*; public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { - private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderR4CacheTest.class); - private SearchCoordinatorSvcImpl mySearchCoordinatorSvcRaw; private CapturingInterceptor myCapturingInterceptor; + @Autowired + private ISearchDao mySearchEntityDao; @Override @After @@ -42,14 +41,13 @@ public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { public void before() throws Exception { super.before(); myFhirCtx.setParserErrorHandler(new StrictErrorHandler()); - mySearchCoordinatorSvcRaw = AopTestUtils.getTargetObject(mySearchCoordinatorSvc); myCapturingInterceptor = new CapturingInterceptor(); ourClient.registerInterceptor(myCapturingInterceptor); } @Test - public void testCacheNoStore() throws IOException { + public void testCacheNoStore() { Patient pt1 = new Patient(); pt1.addName().setFamily("FAM"); @@ -84,7 +82,7 @@ public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { } @Test - public void testCacheNoStoreMaxResults() throws IOException { + public void testCacheNoStoreMaxResults() { for (int i = 0; i < 10; i++) { Patient pt1 = new Patient(); @@ -106,7 +104,7 @@ public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { } @Test - public void testCacheNoStoreMaxResultsWithIllegalValue() throws IOException { + public void testCacheNoStoreMaxResultsWithIllegalValue() { myDaoConfig.setCacheControlNoStoreMaxResultsUpperLimit(123); try { ourClient @@ -123,7 +121,7 @@ public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { } @Test - public void testCacheSuppressed() throws IOException { + public void testCacheSuppressed() { Patient pt1 = new Patient(); pt1.addName().setFamily("FAM"); @@ -152,7 +150,7 @@ public class ResourceProviderR4CacheTest extends BaseResourceProviderR4Test { } @Test - public void testCacheUsedNormally() throws IOException { + public void testCacheUsedNormally() { Patient pt1 = new Patient(); pt1.addName().setFamily("FAM"); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index 06b8eca19b9..7831d4b633f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.jpa.config.TestR4Config; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; @@ -21,10 +22,7 @@ import ca.uhn.fhir.rest.client.api.IHttpResponse; import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor; import ca.uhn.fhir.rest.gclient.StringClientParam; import ca.uhn.fhir.rest.param.*; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; -import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; -import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.rest.server.exceptions.*; import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor; import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.UrlUtil; @@ -54,6 +52,7 @@ import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemType; import org.hl7.fhir.r4.model.Subscription.SubscriptionChannelType; import org.hl7.fhir.r4.model.Subscription.SubscriptionStatus; import org.junit.*; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; @@ -82,6 +81,8 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { public static final int LARGE_NUMBER = 77; private SearchCoordinatorSvcImpl mySearchCoordinatorSvcRaw; private CapturingInterceptor myCapturingInterceptor = new CapturingInterceptor(); + @Autowired + private ISearchDao mySearchEntityDao; @Override @After @@ -3939,7 +3940,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .execute(); final String uuid1 = toSearchUuidFromLinkNext(result1); - Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuid(uuid1)); + Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(()->new InternalErrorException(""))); Date lastReturned1 = search1.getSearchLastReturned(); Bundle result2 = ourClient @@ -3949,9 +3950,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); - Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuid(uuid2)); + Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(()->new InternalErrorException(""))); Date lastReturned2 = search2.getSearchLastReturned(); assertTrue(lastReturned2.getTime() > lastReturned1.getTime()); @@ -3965,6 +3967,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); String uuid3 = toSearchUuidFromLinkNext(result3); @@ -3989,9 +3992,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); - Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuid(uuid1)); + Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(()->new InternalErrorException(""))); Date lastReturned1 = search1.getSearchLastReturned(); sleepOneClick(); @@ -4001,9 +4005,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); - Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuid(uuid2)); + Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(()->new InternalErrorException(""))); Date lastReturned2 = search2.getSearchLastReturned(); assertTrue(lastReturned2.getTime() > lastReturned1.getTime()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java index 3a2441bb556..2048d6d7a67 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java @@ -1,9 +1,15 @@ package ca.uhn.fhir.jpa.provider.r4; -import ca.uhn.fhir.jpa.entity.*; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; +import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.entity.SearchInclude; +import ca.uhn.fhir.jpa.entity.SearchResult; +import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; -import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.rest.gclient.IClientExecutable; import ca.uhn.fhir.rest.gclient.IQuery; import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; @@ -16,6 +22,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; import java.util.Date; @@ -27,22 +34,28 @@ import static org.junit.Assert.*; public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(StaleSearchDeletingSvcR4Test.class); + @Autowired + private ISearchDao mySearchEntityDao; + @Autowired + private ISearchResultDao mySearchResultDao; + @Autowired + private ISearchIncludeDao mySearchIncludeDao; @Override @After() public void after() throws Exception { super.after(); - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_CUTOFF_SLACK); - StaleSearchDeletingSvcImpl.setMaximumResultsToDeleteForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT); - StaleSearchDeletingSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(StaleSearchDeletingSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS); + DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); + staleSearchDeletingSvc.setCutoffSlackForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK); + DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT); + DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS); } @Override @Before public void before() throws Exception { super.before(); - StaleSearchDeletingSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(myStaleSearchDeletingSvc); + DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); } @@ -96,8 +109,8 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { @Test public void testDeleteVeryLargeSearch() { - StaleSearchDeletingSvcImpl.setMaximumResultsToDeleteForUnitTest(10); - StaleSearchDeletingSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(10); + DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); + DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(10); runInTransaction(() -> { Search search = new Search(); @@ -139,7 +152,7 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { @Test public void testDeleteVerySmallSearch() { - StaleSearchDeletingSvcImpl.setMaximumResultsToDeleteForUnitTest(10); + DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); runInTransaction(() -> { Search search = new Search(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/BaseResourceProviderR5Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/BaseResourceProviderR5Test.java index 5ddb411c440..534899c45cb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/BaseResourceProviderR5Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/BaseResourceProviderR5Test.java @@ -2,7 +2,6 @@ package ca.uhn.fhir.jpa.provider.r5; import ca.uhn.fhir.jpa.config.WebsocketDispatcherConfig; import ca.uhn.fhir.jpa.dao.DaoRegistry; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.r5.BaseJpaR5Test; import ca.uhn.fhir.jpa.provider.GraphQLProvider; import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; @@ -64,7 +63,6 @@ public abstract class BaseResourceProviderR5Test extends BaseJpaR5Test { protected static String ourServerBase; protected static SearchParamRegistryR5 ourSearchParamRegistry; private static DatabaseBackedPagingProvider ourPagingProvider; - protected static ISearchDao mySearchEntityDao; protected static ISearchCoordinatorSvc mySearchCoordinatorSvc; private static GenericWebApplicationContext ourWebApplicationContext; private static SubscriptionMatcherInterceptor ourSubscriptionMatcherInterceptor; @@ -166,7 +164,6 @@ public abstract class BaseResourceProviderR5Test extends BaseJpaR5Test { WebApplicationContext wac = WebApplicationContextUtils.getWebApplicationContext(subsServletHolder.getServlet().getServletConfig().getServletContext()); myValidationSupport = wac.getBean(JpaValidationSupportChainR5.class); mySearchCoordinatorSvc = wac.getBean(ISearchCoordinatorSvc.class); - mySearchEntityDao = wac.getBean(ISearchDao.class); ourSearchParamRegistry = wac.getBean(SearchParamRegistryR5.class); ourSubscriptionMatcherInterceptor = wac.getBean(SubscriptionMatcherInterceptor.class); ourSubscriptionMatcherInterceptor.start(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java index 4125ce03821..ffa6ddaa0c3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java @@ -3,13 +3,10 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.jpa.dao.*; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; -import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; -import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; -import ca.uhn.fhir.jpa.entity.SearchResult; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.BaseIterator; import ca.uhn.fhir.model.dstu2.resource.Patient; @@ -34,7 +31,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; @@ -53,20 +49,16 @@ public class SearchCoordinatorSvcImplTest { private static final Logger ourLog = LoggerFactory.getLogger(SearchCoordinatorSvcImplTest.class); private static FhirContext ourCtx = FhirContext.forDstu3(); @Captor - ArgumentCaptor> mySearchResultIterCaptor; + ArgumentCaptor> mySearchResultIterCaptor; @Mock private IFhirResourceDao myCallingDao; @Mock private EntityManager myEntityManager; private int myExpectedNumberOfSearchBuildersCreated = 2; @Mock - private ISearchBuilder mySearchBuider; + private ISearchBuilder mySearchBuilder; @Mock - private ISearchDao mySearchDao; - @Mock - private ISearchIncludeDao mySearchIncludeDao; - @Mock - private ISearchResultDao mySearchResultDao; + private ISearchResultCacheSvc mySearchResultCacheSvc; private SearchCoordinatorSvcImpl mySvc; @Mock private PlatformTransactionManager myTxManager; @@ -90,16 +82,14 @@ public class SearchCoordinatorSvcImplTest { mySvc.setEntityManagerForUnitTest(myEntityManager); mySvc.setTransactionManagerForUnitTest(myTxManager); mySvc.setContextForUnitTest(ourCtx); - mySvc.setSearchDaoForUnitTest(mySearchDao); - mySvc.setSearchDaoIncludeForUnitTest(mySearchIncludeDao); - mySvc.setSearchDaoResultForUnitTest(mySearchResultDao); + mySvc.setSearchResultCacheSvcForUnitTest(mySearchResultCacheSvc); mySvc.setDaoRegistryForUnitTest(myDaoRegistry); mySvc.setInterceptorBroadcasterForUnitTest(myInterceptorBroadcaster); myDaoConfig = new DaoConfig(); mySvc.setDaoConfigForUnitTest(myDaoConfig); - when(myCallingDao.newSearchBuilder()).thenReturn(mySearchBuider); + when(myCallingDao.newSearchBuilder()).thenReturn(mySearchBuilder); when(myTxManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class)); @@ -107,7 +97,7 @@ public class SearchCoordinatorSvcImplTest { PersistedJpaBundleProvider provider = (PersistedJpaBundleProvider) theInvocation.getArguments()[0]; provider.setSearchCoordinatorSvc(mySvc); provider.setPlatformTransactionManager(myTxManager); - provider.setSearchDao(mySearchDao); + provider.setSearchResultCacheSvc(mySearchResultCacheSvc); provider.setEntityManager(myEntityManager); provider.setContext(ourCtx); provider.setInterceptorBroadcaster(myInterceptorBroadcaster); @@ -143,7 +133,7 @@ public class SearchCoordinatorSvcImplTest { List pids = createPidSequence(10, 800); IResultIterator iter = new FailAfterNIterator(new SlowIterator(pids.iterator(), 2), 300); - when(mySearchBuider.createQuery(Mockito.same(params), any(), any())).thenReturn(iter); + when(mySearchBuilder.createQuery(Mockito.same(params), any(), any())).thenReturn(iter); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNotNull(result.getUuid()); @@ -159,23 +149,42 @@ public class SearchCoordinatorSvcImplTest { @Test public void testAsyncSearchLargeResultSetBigCountSameCoordinator() { + List allResults = new ArrayList<>(); + doAnswer(t->{ + List oldResults = t.getArgument(1, List.class); + List newResults = t.getArgument(2, List.class); + ourLog.info("Saving {} new results - have {} old results", newResults.size(), oldResults.size()); + assertEquals(allResults.size(), oldResults.size()); + allResults.addAll(newResults); + return null; + }).when(mySearchResultCacheSvc).storeResults(any(),anyList(),anyList()); + + SearchParameterMap params = new SearchParameterMap(); params.add("name", new StringParam("ANAME")); List pids = createPidSequence(10, 800); SlowIterator iter = new SlowIterator(pids.iterator(), 1); - when(mySearchBuider.createQuery(any(), any(), any())).thenReturn(iter); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + when(mySearchBuilder.createQuery(any(), any(), any())).thenReturn(iter); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); - when(mySearchResultDao.findWithSearchUuid(any(), any())).thenAnswer(t -> { + when(mySearchResultCacheSvc.fetchResultPids(any(), anyInt(), anyInt())).thenAnswer(t -> { List returnedValues = iter.getReturnedValues(); - Pageable page = (Pageable) t.getArguments()[1]; - int offset = (int) page.getOffset(); - int end = (int) (page.getOffset() + page.getPageSize()); + int offset = t.getArgument(1, Integer.class); + int end = t.getArgument(2, Integer.class); end = Math.min(end, returnedValues.size()); offset = Math.min(offset, returnedValues.size()); ourLog.info("findWithSearchUuid {} - {} out of {} values", offset, end, returnedValues.size()); - return new PageImpl<>(returnedValues.subList(offset, end)); + return returnedValues.subList(offset, end); + }); + + when(mySearchResultCacheSvc.fetchAllResultPids(any())).thenReturn(allResults); + + when(mySearchResultCacheSvc.tryToMarkSearchAsInProgress(any())).thenAnswer(t->{ + Search search = t.getArgument(0, Search.class); + assertEquals(SearchStatusEnum.PASSCMPLET, search.getStatus()); + search.setStatus(SearchStatusEnum.LOADING); + return Optional.of(search); }); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); @@ -184,12 +193,14 @@ public class SearchCoordinatorSvcImplTest { List resources; - when(mySearchDao.save(any())).thenAnswer(t -> { + when(mySearchResultCacheSvc.save(any())).thenAnswer(t -> { Search search = (Search) t.getArguments()[0]; myCurrentSearch = search; return search; }); - when(mySearchDao.findByUuid(any())).thenAnswer(t -> myCurrentSearch); + when(mySearchResultCacheSvc.fetchByUuid(any())).thenAnswer(t -> { + return Optional.ofNullable(myCurrentSearch); + }); IFhirResourceDao dao = myCallingDao; when(myDaoRegistry.getResourceDao(any(String.class))).thenReturn(dao); @@ -199,17 +210,11 @@ public class SearchCoordinatorSvcImplTest { assertEquals("799", resources.get(789).getIdElement().getValueAsString()); ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(mySearchDao, atLeastOnce()).save(searchCaptor.capture()); - - verify(mySearchResultDao, atLeastOnce()).saveAll(mySearchResultIterCaptor.capture()); - List allResults = new ArrayList<>(); - for (Iterable next : mySearchResultIterCaptor.getAllValues()) { - allResults.addAll(Lists.newArrayList(next)); - } + verify(mySearchResultCacheSvc, atLeastOnce()).save(searchCaptor.capture()); assertEquals(790, allResults.size()); - assertEquals(10, allResults.get(0).getResourcePid().longValue()); - assertEquals(799, allResults.get(789).getResourcePid().longValue()); + assertEquals(10, allResults.get(0).longValue()); + assertEquals(799, allResults.get(789).longValue()); myExpectedNumberOfSearchBuildersCreated = 4; } @@ -221,9 +226,9 @@ public class SearchCoordinatorSvcImplTest { List pids = createPidSequence(10, 800); SlowIterator iter = new SlowIterator(pids.iterator(), 2); - when(mySearchBuider.createQuery(same(params), any(), any())).thenReturn(iter); + when(mySearchBuilder.createQuery(same(params), any(), any())).thenReturn(iter); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNotNull(result.getUuid()); @@ -249,16 +254,16 @@ public class SearchCoordinatorSvcImplTest { List pids = createPidSequence(10, 800); IResultIterator iter = new SlowIterator(pids.iterator(), 2); - when(mySearchBuider.createQuery(same(params), any(), any())).thenReturn(iter); - when(mySearchDao.save(any())).thenAnswer(t -> t.getArguments()[0]); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + when(mySearchBuilder.createQuery(same(params), any(), any())).thenReturn(iter); + when(mySearchResultCacheSvc.save(any())).thenAnswer(t -> t.getArguments()[0]); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNotNull(result.getUuid()); assertEquals(null, result.size()); ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(mySearchDao, atLeast(1)).save(searchCaptor.capture()); + verify(mySearchResultCacheSvc, atLeast(1)).save(searchCaptor.capture()); Search search = searchCaptor.getValue(); assertEquals(SearchTypeEnum.SEARCH, search.getSearchType()); @@ -271,7 +276,7 @@ public class SearchCoordinatorSvcImplTest { assertEquals("10", resources.get(0).getIdElement().getValueAsString()); assertEquals("19", resources.get(9).getIdElement().getValueAsString()); - when(mySearchDao.findByUuid(eq(result.getUuid()))).thenReturn(search); + when(mySearchResultCacheSvc.fetchByUuid(eq(result.getUuid()))).thenReturn(Optional.of(search)); /* * Now call from a new bundle provider. This simulates a separate HTTP @@ -293,9 +298,9 @@ public class SearchCoordinatorSvcImplTest { List pids = createPidSequence(10, 100); SlowIterator iter = new SlowIterator(pids.iterator(), 2); - when(mySearchBuider.createQuery(same(params), any(), any())).thenReturn(iter); + when(mySearchBuilder.createQuery(same(params), any(), any())).thenReturn(iter); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNotNull(result.getUuid()); @@ -323,8 +328,8 @@ public class SearchCoordinatorSvcImplTest { search.setSearchType(SearchTypeEnum.SEARCH); search.setResourceType("Patient"); - when(mySearchDao.findByUuid(eq(uuid))).thenReturn(search); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + when(mySearchResultCacheSvc.fetchByUuid(eq(uuid))).thenReturn(Optional.of(search)); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); PersistedJpaBundleProvider provider; List resources; @@ -336,16 +341,13 @@ public class SearchCoordinatorSvcImplTest { // ignore } - when(mySearchResultDao.findWithSearchUuid(any(Search.class), any(Pageable.class))).thenAnswer(theInvocation -> { - Pageable page = (Pageable) theInvocation.getArguments()[1]; - + when(mySearchResultCacheSvc.fetchResultPids(any(Search.class), anyInt(), anyInt())).thenAnswer(theInvocation -> { ArrayList results = new ArrayList<>(); - int max = (page.getPageNumber() * page.getPageSize()) + page.getPageSize(); - for (long i = page.getOffset(); i < max; i++) { - results.add(i + 10L); - } + for (long i = theInvocation.getArgument(1, Integer.class); i < theInvocation.getArgument(2, Integer.class); i++) { + results.add(i + 10L); + } - return new PageImpl<>(results); + return results; }); search.setStatus(SearchStatusEnum.FINISHED); }).start(); @@ -376,9 +378,9 @@ public class SearchCoordinatorSvcImplTest { params.add("name", new StringParam("ANAME")); List pids = createPidSequence(10, 800); - when(mySearchBuider.createQuery(same(params), any(), any())).thenReturn(new ResultIterator(pids.iterator())); + when(mySearchBuilder.createQuery(same(params), any(), any())).thenReturn(new ResultIterator(pids.iterator())); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNull(result.getUuid()); @@ -397,10 +399,10 @@ public class SearchCoordinatorSvcImplTest { params.add("name", new StringParam("ANAME")); List pids = createPidSequence(10, 800); - when(mySearchBuider.createQuery(Mockito.same(params), any(), nullable(RequestDetails.class))).thenReturn(new ResultIterator(pids.iterator())); + when(mySearchBuilder.createQuery(Mockito.same(params), any(), nullable(RequestDetails.class))).thenReturn(new ResultIterator(pids.iterator())); pids = createPidSequence(10, 110); - doAnswer(loadPids()).when(mySearchBuider).loadResourcesByPid(eq(pids), any(Collection.class), any(List.class), anyBoolean(), nullable(RequestDetails.class)); + doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(eq(pids), any(Collection.class), any(List.class), anyBoolean(), nullable(RequestDetails.class)); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); assertNull(result.getUuid()); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java new file mode 100644 index 00000000000..3d777691df6 --- /dev/null +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java @@ -0,0 +1,85 @@ +package ca.uhn.fhir.jpa.migrate.taskdef; + +/*- + * #%L + * HAPI FHIR JPA Server - Migration + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.migrate.JdbcUtils; +import org.apache.commons.lang3.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.Set; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +public class DropForeignKeyTask extends BaseTableColumnTask { + + private static final Logger ourLog = LoggerFactory.getLogger(DropForeignKeyTask.class); + private String myConstraintName; + + public void setConstraintName(String theConstraintName) { + myConstraintName = theConstraintName; + } + + @Override + public void validate() { + super.validate(); + + Validate.isTrue(isNotBlank(myConstraintName)); + } + + @Override + public void execute() throws SQLException { + + Set existing = JdbcUtils.getForeignKeys(getConnectionProperties(), null, null); + if (!existing.contains(myConstraintName)) { + ourLog.info("Don't have constraint named {} - No action performed", myConstraintName); + return; + } + + String sql = null; + String sql2 = null; + switch (getDriverType()) { + case MYSQL_5_7: + // Lousy MYQL.... + sql = "alter table " + getTableName() + " drop constraint " + myConstraintName; + sql2 = "alter table " + getTableName() + " drop index " + myConstraintName; + break; + case MARIADB_10_1: + case POSTGRES_9_4: + case DERBY_EMBEDDED: + case H2_EMBEDDED: + case ORACLE_12C: + case MSSQL_2012: + sql = "alter table " + getTableName() + " drop constraint " + myConstraintName; + break; + default: + throw new IllegalStateException(); + } + + executeSql(getTableName(), sql); + if (isNotBlank(sql2)) { + executeSql(getTableName(), sql2); + } + + } + +} diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index 8dae8204771..cec3133a85a 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -79,6 +79,9 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { resVerProv.addIndex("IDX_RESVERPROV_SOURCEURI").unique(false).withColumns("SOURCE_URI"); resVerProv.addIndex("IDX_RESVERPROV_REQUESTID").unique(false).withColumns("REQUEST_ID"); + // Drop HFJ_SEARCH_RESULT foreign keys + version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_RES"); + version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_SEARCH"); } protected void init400() { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java index cf9fc105ed7..d536578fd8b 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java @@ -241,6 +241,13 @@ public class BaseMigrationTasks { return this; } + public void dropForeignKey(String theFkName) { + DropForeignKeyTask task = new DropForeignKeyTask(); + task.setConstraintName(theFkName); + task.setTableName(getTableName()); + addTask(task); + } + public class BuilderAddIndexWithName { private final String myIndexName; diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java new file mode 100644 index 00000000000..4150d24c30e --- /dev/null +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java @@ -0,0 +1,38 @@ +package ca.uhn.fhir.jpa.migrate.taskdef; + +import ca.uhn.fhir.jpa.migrate.JdbcUtils; +import org.hamcrest.Matchers; +import org.junit.Test; + +import java.sql.SQLException; + +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertThat; + +public class DropForeignKeyTaskTest extends BaseTest { + + @Test + public void testDropForeignKey() throws SQLException { + executeSql("create table HOME (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table FOREIGNTBL (PID bigint not null, HOMEREF bigint)"); + executeSql("alter table HOME add foreign key FK_FOO (PID) references FOREIGNTABLE(HOMEREF)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "HOME", "FOREIGNTBL"), empty()); + + DropForeignKeyTask task = new DropForeignKeyTask(); + task.setTableName("FOREIGNTBL"); + task.setColumnName("HOMEREF"); + task.setConstraintName("FK_FOO"); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "HOME", "FOREIGNTBL"), empty()); + + // Make sure additional calls don't crash + getMigrator().migrate(); + getMigrator().migrate(); + } + + +} diff --git a/src/changes/changes.xml b/src/changes/changes.xml index fc505a69c21..8329d021eb1 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -73,6 +73,10 @@ The informational message returned in an OperationOutcome when a delete failed due to cascades not being enabled contained an incorrect example. This has been corrected. + + Two foreign keys have been dropped from the HFJ_SEARCH_RESULT table used by the FHIR search query cache. These + constraints did not add value and caused unneccessary contention when used under high load. + From abc894ce90c46425ce0fcafe8cc01ab1d71fda19 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 23 Aug 2019 10:57:58 -0400 Subject: [PATCH 03/16] A few refactors based on ken's suggestions --- .../jpa/search/SearchCoordinatorSvcImpl.java | 2 +- .../DatabaseSearchResultCacheSvcImpl.java | 2 +- .../search/cache/ISearchResultCacheSvc.java | 39 ++++++++++--------- .../jpa/term/BaseHapiTerminologySvcImpl.java | 1 + 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 0f5c2c34eaa..a22d854bd89 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -357,7 +357,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } // Check for a search matching the given hash - Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(resourceType, queryString, createdCutoff); + Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(resourceType, queryString, queryString.hashCode(), createdCutoff); for (Search nextCandidateSearch : candidates) { if (queryString.equals(nextCandidateSearch.getSearchQueryString()) && nextCandidateSearch.getCreated().after(createdCutoff)) { searchToUse = nextCandidateSearch; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index c72a8f9ea44..ec2c672c1fb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -137,7 +137,7 @@ public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcIm } @Override - public Collection findCandidatesForReuse(String theResourceType, String theQueryString, Date theCreatedAfter) { + public Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter) { int hashCode = theQueryString.hashCode(); return mySearchDao.find(theResourceType, hashCode, theCreatedAfter); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java index 0cfc23bb36e..77b9805960d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java @@ -27,26 +27,9 @@ public interface ISearchResultCacheSvc { */ Optional fetchByUuid(String theUuid); - /** - * Fetch a sunset of the search result IDs from the cache - * - * @param theSearch The search to fetch IDs for - * @param theFrom The starting index (inclusive) - * @param theTo The ending index (exclusive) - * @return A list of resource PIDs - */ - List fetchResultPids(Search theSearch, int theFrom, int theTo); - - /** - * Fetch all result PIDs for a given search with no particular order required - * @param theSearch - * @return - */ - List fetchAllResultPids(Search theSearch); - /** * TODO: this is perhaps an inappropriate responsibility for this service - * + * *

* This method marks a search as in progress, but should only allow exactly one call to do so across the cluster. This * is done so that if two client threads request the next page at the exact same time (which is unlikely but not @@ -70,10 +53,11 @@ public interface ISearchResultCacheSvc { * * @param theResourceType The resource type of the search. Results MUST match this type * @param theQueryString The query string. Results SHOULD match this type + * @param theQueryStringHash The query string hash. Results SHOULD match this type * @param theCreatedAfter Results SHOULD not include any searches created before this cutoff timestamp * @return A collection of candidate searches */ - Collection findCandidatesForReuse(String theResourceType, String theQueryString, Date theCreatedAfter); + Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter); /** * Mark a search as having been "last used" at the given time. This method may (and probably should) be implemented @@ -102,4 +86,21 @@ public interface ISearchResultCacheSvc { * @param theNewResourcePids A list of new resoure PIDs to add to this search (these ones have not been previously saved) */ void storeResults(Search theSearch, List thePreviouslyStoredResourcePids, List theNewResourcePids); + + /** + * Fetch a sunset of the search result IDs from the cache + * + * @param theSearch The search to fetch IDs for + * @param theFrom The starting index (inclusive) + * @param theTo The ending index (exclusive) + * @return A list of resource PIDs + */ + List fetchResultPids(Search theSearch, int theFrom, int theTo); + + /** + * Fetch all result PIDs for a given search with no particular order required + * @param theSearch + * @return + */ + List fetchAllResultPids(Search theSearch); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index c574f84a122..53298913b0b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -648,6 +648,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class); int maxResult = 50000; jpaQuery.setMaxResults(maxResult); + jpaQuery.setFirstResult() StopWatch sw = new StopWatch(); AtomicInteger count = new AtomicInteger(0); From c346ba92c88a73846e8986a08b2b70b3b200e41d Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 23 Aug 2019 11:17:02 -0400 Subject: [PATCH 04/16] Revert accidental commit --- .../java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index 53298913b0b..c574f84a122 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -648,7 +648,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class); int maxResult = 50000; jpaQuery.setMaxResults(maxResult); - jpaQuery.setFirstResult() StopWatch sw = new StopWatch(); AtomicInteger count = new AtomicInteger(0); From c564052b15c40b992cd708b17973d1adb7422d70 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 23 Aug 2019 11:46:56 -0400 Subject: [PATCH 05/16] One more test fix --- .../java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index 7831d4b633f..4d3ed3d9fa5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -3938,6 +3938,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); + mySearchResultCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(()->new InternalErrorException(""))); From 2dc70c92f1f19b6376ee9a2774cc2ba4ed94eb43 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Sat, 24 Aug 2019 14:10:17 -0400 Subject: [PATCH 06/16] add accessor to search hash --- .../src/main/java/ca/uhn/fhir/jpa/entity/Search.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index 928e4b0d48c..b417f5d7ef6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -319,4 +319,8 @@ public class Search implements ICachedSearchDetails, Serializable { public void setCannotBeReused() { mySearchQueryStringHash = null; } + + public Integer getSearchQueryStringHash() { + return mySearchQueryStringHash; + } } From 17f58a61c314c3d3d48155a036183336d1ff448d Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Sat, 24 Aug 2019 14:13:06 -0400 Subject: [PATCH 07/16] Revert "add accessor to search hash" This reverts commit 2dc70c92 --- .../src/main/java/ca/uhn/fhir/jpa/entity/Search.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index b417f5d7ef6..928e4b0d48c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -319,8 +319,4 @@ public class Search implements ICachedSearchDetails, Serializable { public void setCannotBeReused() { mySearchQueryStringHash = null; } - - public Integer getSearchQueryStringHash() { - return mySearchQueryStringHash; - } } From eab589bcac094a9bcbe444e81550f57d796eae39 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Wed, 28 Aug 2019 13:11:44 -0400 Subject: [PATCH 08/16] fixed intermittently failing test --- .../jpa/search/SearchCoordinatorSvcImpl.java | 334 ++++++++++-------- .../DatabaseSearchResultCacheSvcImpl.java | 1 + 2 files changed, 184 insertions(+), 151 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index a22d854bd89..5ea9c657a18 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -79,6 +79,9 @@ import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.persistence.EntityManager; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; import java.util.*; import java.util.concurrent.*; @@ -241,7 +244,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { @Override public IBundleProvider registerSearch(final IDao theCallingDao, final SearchParameterMap theParams, String theResourceType, CacheControlDirective theCacheControlDirective, RequestDetails theRequestDetails) { - StopWatch w = new StopWatch(); final String searchUuid = UUID.randomUUID().toString(); ourLog.debug("Registering new search {}", searchUuid); @@ -251,6 +253,185 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { sb.setType(resourceTypeClass, theResourceType); sb.setFetchSize(mySyncSize); + final Integer loadSynchronousUpTo = getLoadSynchronousUpToOrNull(theCacheControlDirective); + + if (theParams.isLoadSynchronous() || loadSynchronousUpTo != null) { + ourLog.debug("Search {} is loading in synchronous mode", searchUuid); + return executeQuery(theParams, theRequestDetails, searchUuid, sb, loadSynchronousUpTo); + } + + /* + * See if there are any cached searches whose results we can return + * instead + */ + boolean useCache = true; + if (theCacheControlDirective != null && theCacheControlDirective.isNoCache() == true) { + useCache = false; + } + + final String queryString = theParams.toNormalizedQueryString(myContext); + if (theParams.getEverythingMode() == null) { + if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null && useCache) { + IBundleProvider foundSearchProvider = findCachedQuery(theCallingDao, theParams, theResourceType, theRequestDetails, queryString); + if (foundSearchProvider != null) { + return foundSearchProvider; + } + } + } + + return submitSearch(theCallingDao, theParams, theResourceType, theRequestDetails, searchUuid, sb, queryString); + + } + + @NotNull + private IBundleProvider submitSearch(IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequestDetails, String theSearchUuid, ISearchBuilder theSb, String theQueryString) { + StopWatch w = new StopWatch(); + Search search = new Search(); + populateSearchEntity(theParams, theResourceType, theSearchUuid, theQueryString, search); + + // Interceptor call: STORAGE_PRESEARCH_REGISTERED + HookParams params = new HookParams() + .add(ICachedSearchDetails.class, search) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRESEARCH_REGISTERED, params); + + SearchTask task = new SearchTask(search, theCallingDao, theParams, theResourceType, theRequestDetails); + myIdToSearchTask.put(search.getUuid(), task); + myExecutor.submit(task); + + PersistedJpaSearchFirstPageBundleProvider retVal = new PersistedJpaSearchFirstPageBundleProvider(search, theCallingDao, task, theSb, myManagedTxManager, theRequestDetails); + populateBundleProvider(retVal); + + ourLog.debug("Search initial phase completed in {}ms", w.getMillis()); + return retVal; + } + + @org.jetbrains.annotations.Nullable + private IBundleProvider findCachedQuery(IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequestDetails, String theQueryString) { + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + PersistedJpaBundleProvider foundSearchProvider = txTemplate.execute(t -> { + + // Interceptor call: STORAGE_PRECHECK_FOR_CACHED_SEARCH + HookParams params = new HookParams() + .add(SearchParameterMap.class, theParams) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); + Object outcome = JpaInterceptorBroadcaster.doCallHooksAndReturnObject(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRECHECK_FOR_CACHED_SEARCH, params); + if (Boolean.FALSE.equals(outcome)) { + return null; + } + + // Check for a search matching the given hash + Search searchToUse = findSearchToUseOrNull(theQueryString, theResourceType); + if (searchToUse == null) { + return null; + } + + ourLog.debug("Reusing search {} from cache", searchToUse.getUuid()); + // Interceptor call: JPA_PERFTRACE_SEARCH_REUSING_CACHED + params = new HookParams() + .add(SearchParameterMap.class, theParams) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params); + + mySearchResultCacheSvc.updateSearchLastReturned(searchToUse, new Date()); + + PersistedJpaBundleProvider retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao); + retVal.setCacheHit(true); + populateBundleProvider(retVal); + + return retVal; + }); + + if (foundSearchProvider != null) { + return foundSearchProvider; + } + return null; + } + + @Nullable + private Search findSearchToUseOrNull(String theQueryString, String theResourceType) { + Search searchToUse = null; + + // createdCutoff is in recent past + final Instant createdCutoff = Instant.now().minus(myDaoConfig.getReuseCachedSearchResultsForMillis(), ChronoUnit.MILLIS); + Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(theResourceType, theQueryString, theQueryString.hashCode(), Date.from(createdCutoff)); + + for (Search nextCandidateSearch : candidates) { + // We should only reuse our search if it was created within the permitted window + // Date.after() is unreliable. Instant.isAfter() always works. + if (theQueryString.equals(nextCandidateSearch.getSearchQueryString()) && nextCandidateSearch.getCreated().toInstant().isAfter(createdCutoff)) { + searchToUse = nextCandidateSearch; + break; + } + } + return searchToUse; + } + + private IBundleProvider executeQuery(SearchParameterMap theParams, RequestDetails theRequestDetails, String theSearchUuid, ISearchBuilder theSb, Integer theLoadSynchronousUpTo) { + SearchRuntimeDetails searchRuntimeDetails = new SearchRuntimeDetails(theRequestDetails, theSearchUuid); + searchRuntimeDetails.setLoadSynchronous(true); + + // Execute the query and make sure we return distinct results + TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); + return txTemplate.execute(t -> { + + // Load the results synchronously + final List pids = new ArrayList<>(); + + try (IResultIterator resultIter = theSb.createQuery(theParams, searchRuntimeDetails, theRequestDetails)) { + while (resultIter.hasNext()) { + pids.add(resultIter.next()); + if (theLoadSynchronousUpTo != null && pids.size() >= theLoadSynchronousUpTo) { + break; + } + if (theParams.getLoadSynchronousUpTo() != null && pids.size() >= theParams.getLoadSynchronousUpTo()) { + break; + } + } + } catch (IOException e) { + ourLog.error("IO failure during database access", e); + throw new InternalErrorException(e); + } + + JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(pids, () -> theSb); + HookParams params = new HookParams() + .add(IPreResourceAccessDetails.class, accessDetails) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); + JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PREACCESS_RESOURCES, params); + + for (int i = pids.size() - 1; i >= 0; i--) { + if (accessDetails.isDontReturnResourceAtIndex(i)) { + pids.remove(i); + } + } + + /* + * For synchronous queries, we load all the includes right away + * since we're returning a static bundle with all the results + * pre-loaded. This is ok because syncronous requests are not + * expected to be paged + * + * On the other hand for async queries we load includes/revincludes + * individually for pages as we return them to clients + */ + final Set includedPids = new HashSet<>(); + includedPids.addAll(theSb.loadIncludes(myContext, myEntityManager, pids, theParams.getRevIncludes(), true, theParams.getLastUpdated(), "(synchronous)", theRequestDetails)); + includedPids.addAll(theSb.loadIncludes(myContext, myEntityManager, pids, theParams.getIncludes(), false, theParams.getLastUpdated(), "(synchronous)", theRequestDetails)); + List includedPidsList = new ArrayList<>(includedPids); + + List resources = new ArrayList<>(); + theSb.loadResourcesByPid(pids, includedPidsList, resources, false, theRequestDetails); + return new SimpleBundleProvider(resources); + }); + } + + @org.jetbrains.annotations.Nullable + private Integer getLoadSynchronousUpToOrNull(CacheControlDirective theCacheControlDirective) { final Integer loadSynchronousUpTo; if (theCacheControlDirective != null && theCacheControlDirective.isNoStore()) { if (theCacheControlDirective.getMaxResults() != null) { @@ -264,156 +445,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } else { loadSynchronousUpTo = null; } - - if (theParams.isLoadSynchronous() || loadSynchronousUpTo != null) { - - ourLog.debug("Search {} is loading in synchronous mode", searchUuid); - SearchRuntimeDetails searchRuntimeDetails = new SearchRuntimeDetails(theRequestDetails, searchUuid); - searchRuntimeDetails.setLoadSynchronous(true); - - // Execute the query and make sure we return distinct results - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - return txTemplate.execute(t -> { - - // Load the results synchronously - final List pids = new ArrayList<>(); - - try (IResultIterator resultIter = sb.createQuery(theParams, searchRuntimeDetails, theRequestDetails)) { - while (resultIter.hasNext()) { - pids.add(resultIter.next()); - if (loadSynchronousUpTo != null && pids.size() >= loadSynchronousUpTo) { - break; - } - if (theParams.getLoadSynchronousUpTo() != null && pids.size() >= theParams.getLoadSynchronousUpTo()) { - break; - } - } - } catch (IOException e) { - ourLog.error("IO failure during database access", e); - throw new InternalErrorException(e); - } - - JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(pids, () -> sb); - HookParams params = new HookParams() - .add(IPreResourceAccessDetails.class, accessDetails) - .add(RequestDetails.class, theRequestDetails) - .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); - JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PREACCESS_RESOURCES, params); - - for (int i = pids.size() - 1; i >= 0; i--) { - if (accessDetails.isDontReturnResourceAtIndex(i)) { - pids.remove(i); - } - } - - /* - * For synchronous queries, we load all the includes right away - * since we're returning a static bundle with all the results - * pre-loaded. This is ok because syncronous requests are not - * expected to be paged - * - * On the other hand for async queries we load includes/revincludes - * individually for pages as we return them to clients - */ - final Set includedPids = new HashSet<>(); - includedPids.addAll(sb.loadIncludes(myContext, myEntityManager, pids, theParams.getRevIncludes(), true, theParams.getLastUpdated(), "(synchronous)", theRequestDetails)); - includedPids.addAll(sb.loadIncludes(myContext, myEntityManager, pids, theParams.getIncludes(), false, theParams.getLastUpdated(), "(synchronous)", theRequestDetails)); - List includedPidsList = new ArrayList<>(includedPids); - - List resources = new ArrayList<>(); - sb.loadResourcesByPid(pids, includedPidsList, resources, false, theRequestDetails); - return new SimpleBundleProvider(resources); - }); - } - - /* - * See if there are any cached searches whose results we can return - * instead - */ - boolean useCache = true; - if (theCacheControlDirective != null && theCacheControlDirective.isNoCache() == true) { - useCache = false; - } - final String queryString = theParams.toNormalizedQueryString(myContext); - if (theParams.getEverythingMode() == null) { - if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null && useCache) { - - final Date createdCutoff = new Date(System.currentTimeMillis() - myDaoConfig.getReuseCachedSearchResultsForMillis()); - final String resourceType = theResourceType; - - TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager); - PersistedJpaBundleProvider foundSearchProvider = txTemplate.execute(t -> { - Search searchToUse = null; - - // Interceptor call: STORAGE_PRECHECK_FOR_CACHED_SEARCH - HookParams params = new HookParams() - .add(SearchParameterMap.class, theParams) - .add(RequestDetails.class, theRequestDetails) - .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); - Object outcome = JpaInterceptorBroadcaster.doCallHooksAndReturnObject(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRECHECK_FOR_CACHED_SEARCH, params); - if (Boolean.FALSE.equals(outcome)) { - return null; - } - - // Check for a search matching the given hash - Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(resourceType, queryString, queryString.hashCode(), createdCutoff); - for (Search nextCandidateSearch : candidates) { - if (queryString.equals(nextCandidateSearch.getSearchQueryString()) && nextCandidateSearch.getCreated().after(createdCutoff)) { - searchToUse = nextCandidateSearch; - break; - } - } - - PersistedJpaBundleProvider retVal = null; - if (searchToUse != null) { - ourLog.debug("Reusing search {} from cache", searchToUse.getUuid()); - - // Interceptor call: JPA_PERFTRACE_SEARCH_REUSING_CACHED - params = new HookParams() - .add(SearchParameterMap.class, theParams) - .add(RequestDetails.class, theRequestDetails) - .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); - JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params); - - mySearchResultCacheSvc.updateSearchLastReturned(searchToUse, new Date()); - - retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao); - retVal.setCacheHit(true); - - populateBundleProvider(retVal); - } - - return retVal; - }); - - if (foundSearchProvider != null) { - return foundSearchProvider; - } - - } - } - - Search search = new Search(); - populateSearchEntity(theParams, theResourceType, searchUuid, queryString, search); - - // Interceptor call: STORAGE_PRESEARCH_REGISTERED - HookParams params = new HookParams() - .add(ICachedSearchDetails.class, search) - .add(RequestDetails.class, theRequestDetails) - .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); - JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRESEARCH_REGISTERED, params); - - SearchTask task = new SearchTask(search, theCallingDao, theParams, theResourceType, theRequestDetails); - myIdToSearchTask.put(search.getUuid(), task); - myExecutor.submit(task); - - PersistedJpaSearchFirstPageBundleProvider retVal = new PersistedJpaSearchFirstPageBundleProvider(search, theCallingDao, task, sb, myManagedTxManager, theRequestDetails); - populateBundleProvider(retVal); - - ourLog.debug("Search initial phase completed in {}ms", w.getMillis()); - return retVal; - + return loadSynchronousUpTo; } private void callInterceptorStoragePreAccessResources(IInterceptorBroadcaster theInterceptorBroadcaster, RequestDetails theRequestDetails, ISearchBuilder theSb, List thePids) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index ec2c672c1fb..e67f1bf1aac 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -31,6 +31,7 @@ import static ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.toPage; public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcImpl { /* * Be careful increasing this number! We use the number of params here in a + * // FIXME KHS * DELETE FROM foo WHERE params IN (aaaa) * type query and this can fail if we have 1000s of params */ From 3e58962ac16df03617ad763b971c0a4cb085f470 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Wed, 28 Aug 2019 16:24:56 -0400 Subject: [PATCH 09/16] Split search cache api from search result cache api. --- .../ca/uhn/fhir/jpa/config/BaseConfig.java | 7 + .../ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java | 7 +- .../fhir/jpa/dao/data/ISearchResultDao.java | 4 +- .../search/PersistedJpaBundleProvider.java | 9 +- .../jpa/search/SearchCoordinatorSvcImpl.java | 19 +- .../search/StaleSearchDeletingSvcImpl.java | 7 +- ...cImpl.java => BaseSearchCacheSvcImpl.java} | 3 +- .../cache/DatabaseSearchCacheSvcImpl.java | 242 ++++++++++++++++++ .../DatabaseSearchResultCacheSvcImpl.java | 219 +--------------- .../jpa/search/cache/ISearchCacheSvc.java | 81 ++++++ .../search/cache/ISearchResultCacheSvc.java | 72 ------ .../java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java | 3 + ...FhirResourceDaoR4SearchPageExpiryTest.java | 29 ++- .../jpa/dao/r4/FhirResourceDaoR4Test.java | 2 +- .../dstu3/ResourceProviderDstu3Test.java | 8 +- .../provider/r4/ResourceProviderR4Test.java | 10 +- .../r4/StaleSearchDeletingSvcR4Test.java | 20 +- .../search/SearchCoordinatorSvcImplTest.java | 24 +- 18 files changed, 412 insertions(+), 354 deletions(-) rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/{BaseSearchResultCacheSvcImpl.java => BaseSearchCacheSvcImpl.java} (93%) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchCacheSvc.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index db1975c2132..b996327a717 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -14,7 +14,9 @@ import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.IStaleSearchDeletingSvc; import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl; @@ -145,6 +147,11 @@ public abstract class BaseConfig implements SchedulingConfigurer { return new BinaryStorageInterceptor(); } + @Bean + public ISearchCacheSvc searchCacheSvc() { + return new DatabaseSearchCacheSvcImpl(); + } + @Bean public ISearchResultCacheSvc searchResultCacheSvc() { return new DatabaseSearchResultCacheSvcImpl(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java index 44f1698609d..29d4a04080a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java @@ -18,6 +18,7 @@ import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.ResourceMetaParams; import ca.uhn.fhir.jpa.searchparam.extractor.LogicalReferenceHelper; @@ -153,6 +154,8 @@ public abstract class BaseHapiFhirDao implements IDao, @Autowired private PlatformTransactionManager myPlatformTransactionManager; @Autowired + private ISearchCacheSvc mySearchCacheSvc; + @Autowired private ISearchResultCacheSvc mySearchResultCacheSvc; @Autowired private ISearchParamPresenceSvc mySearchParamPresenceSvc; @@ -467,7 +470,7 @@ public abstract class BaseHapiFhirDao implements IDao, } } - search = mySearchResultCacheSvc.save(search); + search = mySearchCacheSvc.save(search); return new PersistedJpaBundleProvider(theRequest, search.getUuid(), this); } @@ -493,7 +496,7 @@ public abstract class BaseHapiFhirDao implements IDao, theProvider.setContext(getContext()); theProvider.setEntityManager(myEntityManager); theProvider.setPlatformTransactionManager(myPlatformTransactionManager); - theProvider.setSearchResultCacheSvc(mySearchResultCacheSvc); + theProvider.setSearchCacheSvc(mySearchCacheSvc); theProvider.setSearchCoordinatorSvc(mySearchCoordinatorSvc); theProvider.setInterceptorBroadcaster(myInterceptorBroadcaster); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java index e0806cb01ec..db7b620c7cc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java @@ -37,10 +37,10 @@ import java.util.Set; public interface ISearchResultDao extends JpaRepository { @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearchPid = :search ORDER BY r.myOrder ASC") - Slice findWithSearchUuid(@Param("search") Long theSearch, Pageable thePage); + Slice findWithSearchPid(@Param("search") Long theSearchPid, Pageable thePage); @Query(value="SELECT r.myResourcePid FROM SearchResult r WHERE r.mySearchPid = :search") - List findWithSearchUuidOrderIndependent(@Param("search") Long theSearch); + List findWithSearchPidOrderIndependent(@Param("search") Long theSearchPid); @Query(value="SELECT r.myId FROM SearchResult r WHERE r.mySearchPid = :search") Slice findForSearch(Pageable thePage, @Param("search") Long theSearchPid); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index db3b3d9fb11..b380816ae9b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -30,6 +30,7 @@ import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.BaseHasResource; import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; import ca.uhn.fhir.model.primitive.InstantDt; @@ -63,7 +64,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { private EntityManager myEntityManager; private PlatformTransactionManager myPlatformTransactionManager; private ISearchCoordinatorSvc mySearchCoordinatorSvc; - private ISearchResultCacheSvc mySearchResultCacheSvc; + private ISearchCacheSvc mySearchCacheSvc; private Search mySearchEntity; private String myUuid; private boolean myCacheHit; @@ -191,7 +192,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { if (mySearchEntity == null) { ensureDependenciesInjected(); - Optional search = mySearchResultCacheSvc.fetchByUuid(myUuid); + Optional search = mySearchCacheSvc.fetchByUuid(myUuid); if (!search.isPresent()) { return false; } @@ -338,7 +339,7 @@ public class PersistedJpaBundleProvider implements IBundleProvider { myInterceptorBroadcaster = theInterceptorBroadcaster; } - public void setSearchResultCacheSvc(ISearchResultCacheSvc theSearchResultCacheSvc) { - mySearchResultCacheSvc = theSearchResultCacheSvc; + public void setSearchCacheSvc(ISearchCacheSvc theSearchCacheSvc) { + mySearchCacheSvc = theSearchCacheSvc; } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 5ea9c657a18..2ea494daf1d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -31,6 +31,7 @@ import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; @@ -81,7 +82,6 @@ import javax.persistence.EntityManager; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; import java.util.*; import java.util.concurrent.*; @@ -108,6 +108,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { @Autowired private PlatformTransactionManager myManagedTxManager; @Autowired + private ISearchCacheSvc mySearchCacheSvc; + @Autowired private ISearchResultCacheSvc mySearchResultCacheSvc; @Autowired private DaoRegistry myDaoRegistry; @@ -129,7 +131,8 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } @VisibleForTesting - public void setSearchResultCacheSvcForUnitTest(ISearchResultCacheSvc theSearchResultCacheSvc) { + public void setSearchCacheServicesForUnitTest(ISearchCacheSvc theSearchCacheSvc, ISearchResultCacheSvc theSearchResultCacheSvc) { + mySearchCacheSvc = theSearchCacheSvc; mySearchResultCacheSvc = theSearchResultCacheSvc; } @@ -183,7 +186,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } } - search = mySearchResultCacheSvc + search = mySearchCacheSvc .fetchByUuid(theUuid) .orElseThrow(() -> { ourLog.debug("Client requested unknown paging ID[{}]", theUuid); @@ -209,7 +212,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { // If the search was saved in "pass complete mode" it's probably time to // start a new pass if (search.getStatus() == SearchStatusEnum.PASSCMPLET) { - Optional newSearch = mySearchResultCacheSvc.tryToMarkSearchAsInProgress(search); + Optional newSearch = mySearchCacheSvc.tryToMarkSearchAsInProgress(search); if (newSearch.isPresent()) { search = newSearch.get(); String resourceType = search.getResourceType(); @@ -237,7 +240,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { theRetVal.setContext(myContext); theRetVal.setEntityManager(myEntityManager); theRetVal.setPlatformTransactionManager(myManagedTxManager); - theRetVal.setSearchResultCacheSvc(mySearchResultCacheSvc); + theRetVal.setSearchCacheSvc(mySearchCacheSvc); theRetVal.setSearchCoordinatorSvc(this); theRetVal.setInterceptorBroadcaster(myInterceptorBroadcaster); } @@ -336,7 +339,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params); - mySearchResultCacheSvc.updateSearchLastReturned(searchToUse, new Date()); + mySearchCacheSvc.updateSearchLastReturned(searchToUse, new Date()); PersistedJpaBundleProvider retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao); retVal.setCacheHit(true); @@ -357,7 +360,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { // createdCutoff is in recent past final Instant createdCutoff = Instant.now().minus(myDaoConfig.getReuseCachedSearchResultsForMillis(), ChronoUnit.MILLIS); - Collection candidates = mySearchResultCacheSvc.findCandidatesForReuse(theResourceType, theQueryString, theQueryString.hashCode(), Date.from(createdCutoff)); + Collection candidates = mySearchCacheSvc.findCandidatesForReuse(theResourceType, theQueryString, theQueryString.hashCode(), Date.from(createdCutoff)); for (Search nextCandidateSearch : candidates) { // We should only reuse our search if it was created within the permitted window @@ -841,7 +844,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { private void doSaveSearch() { - Search newSearch = mySearchResultCacheSvc.save(mySearch); + Search newSearch = mySearchCacheSvc.save(mySearch); // mySearchDao.save is not supposed to return null, but in unit tests // it can if the mock search dao isn't set up to handle that diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java index c1c432c71bc..ccc6bc0fac4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java @@ -21,13 +21,14 @@ package ca.uhn.fhir.jpa.search; */ import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK; +import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK; /** * Deletes old searches @@ -42,12 +43,12 @@ public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc { @Autowired private DaoConfig myDaoConfig; @Autowired - private ISearchResultCacheSvc mySearchResultCacheSvc; + private ISearchCacheSvc mySearchCacheSvc; @Override @Transactional(propagation = Propagation.NEVER) public void pollForStaleSearchesAndDeleteThem() { - mySearchResultCacheSvc.pollForStaleSearchesAndDeleteThem(); + mySearchCacheSvc.pollForStaleSearchesAndDeleteThem(); } @Scheduled(fixedDelay = DEFAULT_CUTOFF_SLACK) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchCacheSvcImpl.java similarity index 93% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchCacheSvcImpl.java index 4d1853b04e5..86899d2b5ba 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchCacheSvcImpl.java @@ -12,7 +12,7 @@ import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public abstract class BaseSearchResultCacheSvcImpl implements ISearchResultCacheSvc { +public abstract class BaseSearchCacheSvcImpl implements ISearchCacheSvc { @Autowired private PlatformTransactionManager myTxManager; @@ -24,7 +24,6 @@ public abstract class BaseSearchResultCacheSvcImpl implements ISearchResultCache myUnsyncedLastUpdated.put(theSearch.getId(), theDate); } - @Override @Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND) public void flushLastUpdated() { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java new file mode 100644 index 00000000000..0e49395434d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java @@ -0,0 +1,242 @@ +package ca.uhn.fhir.jpa.search.cache; + +import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; +import ca.uhn.fhir.jpa.entity.Search; +import ca.uhn.fhir.jpa.entity.SearchInclude; +import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.time.DateUtils; +import org.hl7.fhir.dstu3.model.InstantType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Slice; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.transaction.Transactional; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Optional; + +public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl { + private static final Logger ourLog = LoggerFactory.getLogger(DatabaseSearchCacheSvcImpl.class); + + /* + * Be careful increasing this number! We use the number of params here in a + * // FIXME KHS + * DELETE FROM foo WHERE params IN (aaaa) + * type query and this can fail if we have 1000s of params + */ + public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500; + public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000; + public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND; + + private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT; + private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS; + private static Long ourNowForUnitTests; + /* + * We give a bit of extra leeway just to avoid race conditions where a query result + * is being reused (because a new client request came in with the same params) right before + * the result is to be deleted + */ + private long myCutoffSlack = DEFAULT_CUTOFF_SLACK; + + @Autowired + private ISearchDao mySearchDao; + @Autowired + private ISearchResultDao mySearchResultDao; + @Autowired + private ISearchIncludeDao mySearchIncludeDao; + @Autowired + private PlatformTransactionManager myTxManager; + @Autowired + private DaoConfig myDaoConfig; + + @VisibleForTesting + public void setCutoffSlackForUnitTest(long theCutoffSlack) { + myCutoffSlack = theCutoffSlack; + } + + @Transactional(Transactional.TxType.REQUIRED) + @Override + public Search save(Search theSearch) { + Search newSearch; + if (theSearch.getId() == null) { + newSearch = mySearchDao.save(theSearch); + for (SearchInclude next : theSearch.getIncludes()) { + mySearchIncludeDao.save(next); + } + } else { + newSearch = mySearchDao.save(theSearch); + } + return newSearch; + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public Optional fetchByUuid(String theUuid) { + Validate.notBlank(theUuid); + return mySearchDao.findByUuidAndFetchIncludes(theUuid); + } + + + @Override + @Transactional(Transactional.TxType.NEVER) + public Optional tryToMarkSearchAsInProgress(Search theSearch) { + ourLog.trace("Going to try to change search status from {} to {}", theSearch.getStatus(), SearchStatusEnum.LOADING); + try { + TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + txTemplate.afterPropertiesSet(); + return txTemplate.execute(t -> { + Search search = mySearchDao.findById(theSearch.getId()).orElse(theSearch); + + if (search.getStatus() != SearchStatusEnum.PASSCMPLET) { + throw new IllegalStateException("Can't change to LOADING because state is " + theSearch.getStatus()); + } + search.setStatus(SearchStatusEnum.LOADING); + Search newSearch = mySearchDao.save(search); + return Optional.of(newSearch); + }); + } catch (Exception e) { + ourLog.warn("Failed to activate search: {}", e.toString()); + ourLog.trace("Failed to activate search", e); + return Optional.empty(); + } + } + + @Override + public Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter) { + int hashCode = theQueryString.hashCode(); + return mySearchDao.find(theResourceType, hashCode, theCreatedAfter); + + } + + @Override + protected void flushLastUpdated(Long theSearchId, Date theLastUpdated) { + mySearchDao.updateSearchLastReturned(theSearchId, theLastUpdated); + } + + @Transactional(Transactional.TxType.NEVER) + @Override + public void pollForStaleSearchesAndDeleteThem() { + if (!myDaoConfig.isExpireSearchResults()) { + return; + } + + long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis(); + if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) { + cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis()); + } + final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack); + + if (ourNowForUnitTests != null) { + ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now()))); + } + + ourLog.debug("Searching for searches which are before {}", cutoff); + + TransactionTemplate tt = new TransactionTemplate(myTxManager); + final Slice toDelete = tt.execute(theStatus -> + mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 2000)) + ); + for (final Long nextSearchToDelete : toDelete) { + ourLog.debug("Deleting search with PID {}", nextSearchToDelete); + tt.execute(t -> { + mySearchDao.updateDeleted(nextSearchToDelete, true); + return null; + }); + + tt.execute(t -> { + deleteSearch(nextSearchToDelete); + return null; + }); + } + + int count = toDelete.getContent().size(); + if (count > 0) { + if (ourLog.isDebugEnabled()) { + long total = tt.execute(t -> mySearchDao.count()); + ourLog.debug("Deleted {} searches, {} remaining", count, total); + } + } + } + + + @VisibleForTesting + void setSearchDaoForUnitTest(ISearchDao theSearchDao) { + mySearchDao = theSearchDao; + } + + @VisibleForTesting + void setSearchDaoIncludeForUnitTest(ISearchIncludeDao theSearchIncludeDao) { + mySearchIncludeDao = theSearchIncludeDao; + } + + private void deleteSearch(final Long theSearchPid) { + mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> { + mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); + + /* + * Note, we're only deleting up to 500 results in an individual search here. This + * is to prevent really long running transactions in cases where there are + * huge searches with tons of results in them. By the time we've gotten here + * we have marked the parent Search entity as deleted, so it's not such a + * huge deal to be only partially deleting search results. They'll get deleted + * eventually + */ + int max = ourMaximumResultsToDeleteInOnePass; + Slice resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId()); + if (resultPids.hasContent()) { + List> partitions = Lists.partition(resultPids.getContent(), ourMaximumResultsToDeleteInOneStatement); + for (List nextPartition : partitions) { + mySearchResultDao.deleteByIds(nextPartition); + } + + } + + // Only delete if we don't have results left in this search + if (resultPids.getNumberOfElements() < max) { + ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned())); + mySearchDao.deleteByPid(searchToDelete.getId()); + } else { + ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid()); + } + }); + } + + @VisibleForTesting + public static void setMaximumResultsToDeleteInOnePassForUnitTest(int theMaximumResultsToDeleteInOnePass) { + ourMaximumResultsToDeleteInOnePass = theMaximumResultsToDeleteInOnePass; + } + + @VisibleForTesting + public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) { + ourMaximumResultsToDeleteInOneStatement = theMaximumResultsToDelete; + } + + /** + * This is for unit tests only, do not call otherwise + */ + @VisibleForTesting + public static void setNowForUnitTests(Long theNowForUnitTests) { + ourNowForUnitTests = theNowForUnitTests; + } + + private static long now() { + if (ourNowForUnitTests != null) { + return ourNowForUnitTests; + } + return System.currentTimeMillis(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index e67f1bf1aac..7684c268ee0 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -1,91 +1,25 @@ package ca.uhn.fhir.jpa.search.cache; -import ca.uhn.fhir.jpa.dao.DaoConfig; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; -import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao; import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; -import ca.uhn.fhir.jpa.entity.SearchInclude; import ca.uhn.fhir.jpa.entity.SearchResult; -import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import org.apache.commons.lang3.Validate; -import org.apache.commons.lang3.time.DateUtils; -import org.hl7.fhir.dstu3.model.InstantType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Slice; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.support.TransactionTemplate; import javax.transaction.Transactional; import java.util.*; import static ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.toPage; -public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcImpl { - /* - * Be careful increasing this number! We use the number of params here in a - * // FIXME KHS - * DELETE FROM foo WHERE params IN (aaaa) - * type query and this can fail if we have 1000s of params - */ - public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500; - public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000; - public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND; +public class DatabaseSearchResultCacheSvcImpl implements ISearchResultCacheSvc { private static final Logger ourLog = LoggerFactory.getLogger(DatabaseSearchResultCacheSvcImpl.class); - private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT; - private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS; - private static Long ourNowForUnitTests; - /* - * We give a bit of extra leeway just to avoid race conditions where a query result - * is being reused (because a new client request came in with the same params) right before - * the result is to be deleted - */ - private long myCutoffSlack = DEFAULT_CUTOFF_SLACK; - @Autowired - private ISearchDao mySearchDao; - @Autowired - private ISearchIncludeDao mySearchIncludeDao; @Autowired private ISearchResultDao mySearchResultDao; - @Autowired - private PlatformTransactionManager myTxManager; - @Autowired - private DaoConfig myDaoConfig; - - @VisibleForTesting - public void setCutoffSlackForUnitTest(long theCutoffSlack) { - myCutoffSlack = theCutoffSlack; - } - - @Transactional(Transactional.TxType.REQUIRED) - @Override - public Search save(Search theSearch) { - Search newSearch; - if (theSearch.getId() == null) { - newSearch = mySearchDao.save(theSearch); - for (SearchInclude next : theSearch.getIncludes()) { - mySearchIncludeDao.save(next); - } - } else { - newSearch = mySearchDao.save(theSearch); - } - return newSearch; - } - - @Override - @Transactional(Transactional.TxType.REQUIRED) - public Optional fetchByUuid(String theUuid) { - Validate.notBlank(theUuid); - return mySearchDao.findByUuidAndFetchIncludes(theUuid); - } @Override @Transactional(Transactional.TxType.REQUIRED) @@ -96,7 +30,7 @@ public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcIm } List retVal = mySearchResultDao - .findWithSearchUuid(theSearch.getId(), page) + .findWithSearchPid(theSearch.getId(), page) .getContent(); ourLog.trace("fetchResultPids for range {}-{} returned {} pids", theFrom, theTo, retVal.size()); @@ -107,88 +41,11 @@ public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcIm @Override @Transactional(Transactional.TxType.REQUIRED) public List fetchAllResultPids(Search theSearch) { - List retVal = mySearchResultDao.findWithSearchUuidOrderIndependent(theSearch.getId()); + List retVal = mySearchResultDao.findWithSearchPidOrderIndependent(theSearch.getId()); ourLog.trace("fetchAllResultPids returned {} pids", retVal.size()); return retVal; } - @Override - @Transactional(Transactional.TxType.NEVER) - public Optional tryToMarkSearchAsInProgress(Search theSearch) { - ourLog.trace("Going to try to change search status from {} to {}", theSearch.getStatus(), SearchStatusEnum.LOADING); - try { - TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - txTemplate.afterPropertiesSet(); - return txTemplate.execute(t -> { - Search search = mySearchDao.findById(theSearch.getId()).orElse(theSearch); - - if (search.getStatus() != SearchStatusEnum.PASSCMPLET) { - throw new IllegalStateException("Can't change to LOADING because state is " + theSearch.getStatus()); - } - search.setStatus(SearchStatusEnum.LOADING); - Search newSearch = mySearchDao.save(search); - return Optional.of(newSearch); - }); - } catch (Exception e) { - ourLog.warn("Failed to activate search: {}", e.toString()); - ourLog.trace("Failed to activate search", e); - return Optional.empty(); - } - } - - @Override - public Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter) { - int hashCode = theQueryString.hashCode(); - return mySearchDao.find(theResourceType, hashCode, theCreatedAfter); - - } - - @Transactional(Transactional.TxType.NEVER) - @Override - public void pollForStaleSearchesAndDeleteThem() { - if (!myDaoConfig.isExpireSearchResults()) { - return; - } - - long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis(); - if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) { - cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis()); - } - final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack); - - if (ourNowForUnitTests != null) { - ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now()))); - } - - ourLog.debug("Searching for searches which are before {}", cutoff); - - TransactionTemplate tt = new TransactionTemplate(myTxManager); - final Slice toDelete = tt.execute(theStatus -> - mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 2000)) - ); - for (final Long nextSearchToDelete : toDelete) { - ourLog.debug("Deleting search with PID {}", nextSearchToDelete); - tt.execute(t -> { - mySearchDao.updateDeleted(nextSearchToDelete, true); - return null; - }); - - tt.execute(t -> { - deleteSearch(nextSearchToDelete); - return null; - }); - } - - int count = toDelete.getContent().size(); - if (count > 0) { - if (ourLog.isDebugEnabled()) { - long total = tt.execute(t -> mySearchDao.count()); - ourLog.debug("Deleted {} searches, {} remaining", count, total); - } - } - } - @Override @Transactional(Transactional.TxType.REQUIRED) public void storeResults(Search theSearch, List thePreviouslyStoredResourcePids, List theNewResourcePids) { @@ -210,80 +67,10 @@ public class DatabaseSearchResultCacheSvcImpl extends BaseSearchResultCacheSvcIm mySearchResultDao.saveAll(resultsToSave); } - @Override - protected void flushLastUpdated(Long theSearchId, Date theLastUpdated) { - mySearchDao.updateSearchLastReturned(theSearchId, theLastUpdated); - } - - @VisibleForTesting - void setSearchDaoForUnitTest(ISearchDao theSearchDao) { - mySearchDao = theSearchDao; - } - - @VisibleForTesting - void setSearchDaoIncludeForUnitTest(ISearchIncludeDao theSearchIncludeDao) { - mySearchIncludeDao = theSearchIncludeDao; - } - @VisibleForTesting void setSearchDaoResultForUnitTest(ISearchResultDao theSearchResultDao) { mySearchResultDao = theSearchResultDao; } - private void deleteSearch(final Long theSearchPid) { - mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> { - mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); - /* - * Note, we're only deleting up to 500 results in an individual search here. This - * is to prevent really long running transactions in cases where there are - * huge searches with tons of results in them. By the time we've gotten here - * we have marked the parent Search entity as deleted, so it's not such a - * huge deal to be only partially deleting search results. They'll get deleted - * eventually - */ - int max = ourMaximumResultsToDeleteInOnePass; - Slice resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId()); - if (resultPids.hasContent()) { - List> partitions = Lists.partition(resultPids.getContent(), ourMaximumResultsToDeleteInOneStatement); - for (List nextPartition : partitions) { - mySearchResultDao.deleteByIds(nextPartition); - } - - } - - // Only delete if we don't have results left in this search - if (resultPids.getNumberOfElements() < max) { - ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned())); - mySearchDao.deleteByPid(searchToDelete.getId()); - } else { - ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid()); - } - }); - } - - @VisibleForTesting - public static void setMaximumResultsToDeleteInOnePassForUnitTest(int theMaximumResultsToDeleteInOnePass) { - ourMaximumResultsToDeleteInOnePass = theMaximumResultsToDeleteInOnePass; - } - - @VisibleForTesting - public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) { - ourMaximumResultsToDeleteInOneStatement = theMaximumResultsToDelete; - } - - /** - * This is for unit tests only, do not call otherwise - */ - @VisibleForTesting - public static void setNowForUnitTests(Long theNowForUnitTests) { - ourNowForUnitTests = theNowForUnitTests; - } - - private static long now() { - if (ourNowForUnitTests != null) { - return ourNowForUnitTests; - } - return System.currentTimeMillis(); - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchCacheSvc.java new file mode 100644 index 00000000000..74b84fc58a3 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchCacheSvc.java @@ -0,0 +1,81 @@ +package ca.uhn.fhir.jpa.search.cache; + +import ca.uhn.fhir.jpa.entity.Search; + +import java.util.Collection; +import java.util.Date; +import java.util.Optional; + +public interface ISearchCacheSvc { + + /** + * Places a new search of some sort in the cache, or updates an existing search. The search passed in is guaranteed to have + * a {@link Search#getUuid() UUID} so that is a good candidate for consistent identification. + * + * @param theSearch The search to store + * @return Returns a copy of the search as it was saved. Callers should use the returned Search object for any further processing. + */ + Search save(Search theSearch); + + /** + * Fetch a search using its UUID. The search should be fully loaded when it is returned (i.e. includes are fetched, so that access to its + * fields will not cause database errors if the current tranaction scope ends. + * + * @param theUuid The search UUID + * @return The search if it exists + */ + Optional fetchByUuid(String theUuid); + + /** + * TODO: this is perhaps an inappropriate responsibility for this service + * + *

+ * This method marks a search as in progress, but should only allow exactly one call to do so across the cluster. This + * is done so that if two client threads request the next page at the exact same time (which is unlikely but not + * impossible) only one will actually proceed to load the next results and the other will just wait for them + * to arrive. + * + * @param theSearch The search to mark + * @return This method should return an empty optional if the search was not marked (meaning that another thread + * succeeded in marking it). If the search doesn't exist or some other error occurred, an exception will be thrown + * instead of {@link Optional#empty()} + */ + Optional tryToMarkSearchAsInProgress(Search theSearch); + + /** + * Look for any existing searches matching the given resource type and query string. + *

+ * This method is allowed to perofrm a "best effort" return, so it can return searches that don't match the query string exactly, or + * which have a created timestamp before theCreatedAfter date. The caller is responsible for removing + * any inappropriate Searches and picking the most relevant one. + *

+ * + * @param theResourceType The resource type of the search. Results MUST match this type + * @param theQueryString The query string. Results SHOULD match this type + * @param theQueryStringHash The query string hash. Results SHOULD match this type + * @param theCreatedAfter Results SHOULD not include any searches created before this cutoff timestamp + * @return A collection of candidate searches + */ + Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter); + + /** + * Mark a search as having been "last used" at the given time. This method may (and probably should) be implemented + * to work asynchronously in order to avoid hammering the database if the search gets reused many times. + * + * @param theSearch The search + * @param theDate The "last returned" timestamp + */ + void updateSearchLastReturned(Search theSearch, Date theDate); + + /** + * This is mostly public for unit tests + */ + void flushLastUpdated(); + + /** + * This method will be called periodically to delete stale searches. Implementations are not required to do anything + * if they have some other mechanism for expiring stale results other than manually looking for them + * and deleting them. + */ + void pollForStaleSearchesAndDeleteThem(); +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java index 77b9805960d..352c1a7e407 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java @@ -8,78 +8,6 @@ import java.util.List; import java.util.Optional; public interface ISearchResultCacheSvc { - - /** - * Places a new search of some sort in the cache, or updates an existing search. The search passed in is guaranteed to have - * a {@link Search#getUuid() UUID} so that is a good candidate for consistent identification. - * - * @param theSearch The search to store - * @return Returns a copy of the search as it was saved. Callers should use the returned Search object for any further processing. - */ - Search save(Search theSearch); - - /** - * Fetch a search using its UUID. The search should be fully loaded when it is returned (i.e. includes are fetched, so that access to its - * fields will not cause database errors if the current tranaction scope ends. - * - * @param theUuid The search UUID - * @return The search if it exists - */ - Optional fetchByUuid(String theUuid); - - /** - * TODO: this is perhaps an inappropriate responsibility for this service - * - *

- * This method marks a search as in progress, but should only allow exactly one call to do so across the cluster. This - * is done so that if two client threads request the next page at the exact same time (which is unlikely but not - * impossible) only one will actually proceed to load the next results and the other will just wait for them - * to arrive. - * - * @param theSearch The search to mark - * @return This method should return an empty optional if the search was not marked (meaning that another thread - * succeeded in marking it). If the search doesn't exist or some other error occurred, an exception will be thrown - * instead of {@link Optional#empty()} - */ - Optional tryToMarkSearchAsInProgress(Search theSearch); - - /** - * Look for any existing searches matching the given resource type and query string. - *

- * This method is allowed to perofrm a "best effort" return, so it can return searches that don't match the query string exactly, or - * which have a created timestamp before theCreatedAfter date. The caller is responsible for removing - * any inappropriate Searches and picking the most relevant one. - *

- * - * @param theResourceType The resource type of the search. Results MUST match this type - * @param theQueryString The query string. Results SHOULD match this type - * @param theQueryStringHash The query string hash. Results SHOULD match this type - * @param theCreatedAfter Results SHOULD not include any searches created before this cutoff timestamp - * @return A collection of candidate searches - */ - Collection findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter); - - /** - * Mark a search as having been "last used" at the given time. This method may (and probably should) be implemented - * to work asynchronously in order to avoid hammering the database if the search gets reused many times. - * - * @param theSearch The search - * @param theDate The "last returned" timestamp - */ - void updateSearchLastReturned(Search theSearch, Date theDate); - - /** - * This is mostly public for unit tests - */ - void flushLastUpdated(); - - /** - * This method will be called periodically to delete stale searches. Implementations are not required to do anything - * if they have some other mechanism for expiring stale results other than manually looking for them - * and deleting them. - */ - void pollForStaleSearchesAndDeleteThem(); - /** * @param theSearch The search - This method is not required to persist any chances to the Search object, it is only provided here for identification * @param thePreviouslyStoredResourcePids A list of resource PIDs that have previously been saved to this search diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java index 8461872f6e0..8b7583f180d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java @@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.provider.SystemProviderDstu2Test; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; @@ -94,6 +95,8 @@ public abstract class BaseJpaTest { protected CircularQueueCaptureQueriesListener myCaptureQueriesListener; @Autowired protected ISearchResultCacheSvc mySearchResultCacheSvc; + @Autowired + protected ISearchCacheSvc mySearchCacheSvc; @After public void afterPerformCleanup() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java index af443190438..75f7d4c6f1f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java @@ -4,6 +4,7 @@ import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.TestUtil; @@ -32,7 +33,7 @@ import javax.annotation.Nullable; import java.util.Date; import java.util.concurrent.atomic.AtomicLong; -import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK; +import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.*; @@ -45,14 +46,14 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { @After() public void after() { - DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); + DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc); staleSearchDeletingSvc.setCutoffSlackForUnitTest(DEFAULT_CUTOFF_SLACK); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(null); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(null); } @Before public void before() { - DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); + DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc); staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); myDaoConfig.setCountSearchResultsUpTo(new DaoConfig().getCountSearchResultsUpTo()); } @@ -83,7 +84,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResultsAfterMillis(1000L); myDaoConfig.setReuseCachedSearchResultsForMillis(500L); long start = System.currentTimeMillis(); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(start); final String searchUuid1; { @@ -123,7 +124,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { // Search just got used so it shouldn't be deleted - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start + 500); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(start + 500); final AtomicLong search3timestamp = new AtomicLong(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override @@ -136,7 +137,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @Override @@ -151,7 +152,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @@ -162,7 +163,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @@ -217,7 +218,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResultsAfterMillis(500); myDaoConfig.setReuseCachedSearchResultsForMillis(500L); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start.get() + 499); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + 499); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); txTemplate.execute(new TransactionCallbackWithoutResult() { @Override @@ -226,7 +227,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(start.get() + 600); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + 600); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); txTemplate.execute(new TransactionCallbackWithoutResult() { @Override @@ -308,7 +309,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @@ -324,7 +325,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { } }); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { @@ -373,7 +374,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test { myDaoConfig.setExpireSearchResults(false); - DatabaseSearchResultCacheSvcImpl.setNowForUnitTests(System.currentTimeMillis() + DateUtils.MILLIS_PER_DAY); + DatabaseSearchCacheSvcImpl.setNowForUnitTests(System.currentTimeMillis() + DateUtils.MILLIS_PER_DAY); myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem(); newTxTemplate().execute(new TransactionCallbackWithoutResult() { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java index 0a74bbc1ce6..3d3f422b2df 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java @@ -3862,7 +3862,7 @@ public class FhirResourceDaoR4Test extends BaseJpaR4Test { search.setStatus(SearchStatusEnum.FAILED); search.setFailureCode(500); search.setFailureMessage("FOO"); - mySearchResultCacheSvc.save(search); + mySearchCacheSvc.save(search); }); IBundleProvider results = myEncounterDao.search(map); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index c004a6834f8..91d6fa55695 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -2973,7 +2973,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .count(5) .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(new TransactionCallback() { @@ -2991,7 +2991,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .count(5) .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); Search search2 = newTxTemplate().execute(new TransactionCallback() { @@ -3037,7 +3037,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException(""))); @@ -3048,7 +3048,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException(""))); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index 4d3ed3d9fa5..ef8e70e40fe 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -3938,7 +3938,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(()->new InternalErrorException(""))); @@ -3951,7 +3951,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(()->new InternalErrorException(""))); @@ -3968,7 +3968,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .count(5) .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); String uuid3 = toSearchUuidFromLinkNext(result3); @@ -3993,7 +3993,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid1 = toSearchUuidFromLinkNext(result1); Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(()->new InternalErrorException(""))); @@ -4006,7 +4006,7 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test { .forResource("Organization") .returnBundle(Bundle.class) .execute(); - mySearchResultCacheSvc.flushLastUpdated(); + mySearchCacheSvc.flushLastUpdated(); final String uuid2 = toSearchUuidFromLinkNext(result2); Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(()->new InternalErrorException(""))); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java index 2048d6d7a67..33282d2a386 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java @@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.entity.SearchResult; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.rest.gclient.IClientExecutable; import ca.uhn.fhir.rest.gclient.IQuery; @@ -45,17 +46,17 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { @After() public void after() throws Exception { super.after(); - DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); - staleSearchDeletingSvc.setCutoffSlackForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_CUTOFF_SLACK); - DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT); - DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(DatabaseSearchResultCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS); + DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc); + staleSearchDeletingSvc.setCutoffSlackForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK); + DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT); + DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS); } @Override @Before public void before() throws Exception { super.before(); - DatabaseSearchResultCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchResultCacheSvc); + DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc); staleSearchDeletingSvc.setCutoffSlackForUnitTest(0); } @@ -109,8 +110,8 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { @Test public void testDeleteVeryLargeSearch() { - DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); - DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(10); + DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); + DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(10); runInTransaction(() -> { Search search = new Search(); @@ -152,7 +153,7 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { @Test public void testDeleteVerySmallSearch() { - DatabaseSearchResultCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); + DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(10); runInTransaction(() -> { Search search = new Search(); @@ -162,8 +163,7 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test { search.setSearchType(SearchTypeEnum.SEARCH); search.setResourceType("Patient"); search.setSearchLastReturned(DateUtils.addDays(new Date(), -10000)); - search = mySearchEntityDao.save(search); - + mySearchEntityDao.save(search); }); // It should take one pass to delete the search fully diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java index ffa6ddaa0c3..049ffae39a0 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImplTest.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.dao.*; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.BaseIterator; @@ -16,7 +17,6 @@ import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.util.TestUtil; -import com.google.common.collect.Lists; import org.hl7.fhir.instance.model.api.IBaseResource; import org.junit.After; import org.junit.AfterClass; @@ -58,6 +58,8 @@ public class SearchCoordinatorSvcImplTest { @Mock private ISearchBuilder mySearchBuilder; @Mock + private ISearchCacheSvc mySearchCacheSvc; + @Mock private ISearchResultCacheSvc mySearchResultCacheSvc; private SearchCoordinatorSvcImpl mySvc; @Mock @@ -82,7 +84,7 @@ public class SearchCoordinatorSvcImplTest { mySvc.setEntityManagerForUnitTest(myEntityManager); mySvc.setTransactionManagerForUnitTest(myTxManager); mySvc.setContextForUnitTest(ourCtx); - mySvc.setSearchResultCacheSvcForUnitTest(mySearchResultCacheSvc); + mySvc.setSearchCacheServicesForUnitTest(mySearchCacheSvc, mySearchResultCacheSvc); mySvc.setDaoRegistryForUnitTest(myDaoRegistry); mySvc.setInterceptorBroadcasterForUnitTest(myInterceptorBroadcaster); @@ -97,7 +99,7 @@ public class SearchCoordinatorSvcImplTest { PersistedJpaBundleProvider provider = (PersistedJpaBundleProvider) theInvocation.getArguments()[0]; provider.setSearchCoordinatorSvc(mySvc); provider.setPlatformTransactionManager(myTxManager); - provider.setSearchResultCacheSvc(mySearchResultCacheSvc); + provider.setSearchCacheSvc(mySearchCacheSvc); provider.setEntityManager(myEntityManager); provider.setContext(ourCtx); provider.setInterceptorBroadcaster(myInterceptorBroadcaster); @@ -180,7 +182,7 @@ public class SearchCoordinatorSvcImplTest { when(mySearchResultCacheSvc.fetchAllResultPids(any())).thenReturn(allResults); - when(mySearchResultCacheSvc.tryToMarkSearchAsInProgress(any())).thenAnswer(t->{ + when(mySearchCacheSvc.tryToMarkSearchAsInProgress(any())).thenAnswer(t->{ Search search = t.getArgument(0, Search.class); assertEquals(SearchStatusEnum.PASSCMPLET, search.getStatus()); search.setStatus(SearchStatusEnum.LOADING); @@ -193,12 +195,12 @@ public class SearchCoordinatorSvcImplTest { List resources; - when(mySearchResultCacheSvc.save(any())).thenAnswer(t -> { + when(mySearchCacheSvc.save(any())).thenAnswer(t -> { Search search = (Search) t.getArguments()[0]; myCurrentSearch = search; return search; }); - when(mySearchResultCacheSvc.fetchByUuid(any())).thenAnswer(t -> { + when(mySearchCacheSvc.fetchByUuid(any())).thenAnswer(t -> { return Optional.ofNullable(myCurrentSearch); }); IFhirResourceDao dao = myCallingDao; @@ -210,7 +212,7 @@ public class SearchCoordinatorSvcImplTest { assertEquals("799", resources.get(789).getIdElement().getValueAsString()); ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(mySearchResultCacheSvc, atLeastOnce()).save(searchCaptor.capture()); + verify(mySearchCacheSvc, atLeastOnce()).save(searchCaptor.capture()); assertEquals(790, allResults.size()); assertEquals(10, allResults.get(0).longValue()); @@ -255,7 +257,7 @@ public class SearchCoordinatorSvcImplTest { List pids = createPidSequence(10, 800); IResultIterator iter = new SlowIterator(pids.iterator(), 2); when(mySearchBuilder.createQuery(same(params), any(), any())).thenReturn(iter); - when(mySearchResultCacheSvc.save(any())).thenAnswer(t -> t.getArguments()[0]); + when(mySearchCacheSvc.save(any())).thenAnswer(t -> t.getArguments()[0]); doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); IBundleProvider result = mySvc.registerSearch(myCallingDao, params, "Patient", new CacheControlDirective(), null); @@ -263,7 +265,7 @@ public class SearchCoordinatorSvcImplTest { assertEquals(null, result.size()); ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(mySearchResultCacheSvc, atLeast(1)).save(searchCaptor.capture()); + verify(mySearchCacheSvc, atLeast(1)).save(searchCaptor.capture()); Search search = searchCaptor.getValue(); assertEquals(SearchTypeEnum.SEARCH, search.getSearchType()); @@ -276,7 +278,7 @@ public class SearchCoordinatorSvcImplTest { assertEquals("10", resources.get(0).getIdElement().getValueAsString()); assertEquals("19", resources.get(9).getIdElement().getValueAsString()); - when(mySearchResultCacheSvc.fetchByUuid(eq(result.getUuid()))).thenReturn(Optional.of(search)); + when(mySearchCacheSvc.fetchByUuid(eq(result.getUuid()))).thenReturn(Optional.of(search)); /* * Now call from a new bundle provider. This simulates a separate HTTP @@ -328,7 +330,7 @@ public class SearchCoordinatorSvcImplTest { search.setSearchType(SearchTypeEnum.SEARCH); search.setResourceType("Patient"); - when(mySearchResultCacheSvc.fetchByUuid(eq(uuid))).thenReturn(Optional.of(search)); + when(mySearchCacheSvc.fetchByUuid(eq(uuid))).thenReturn(Optional.of(search)); doAnswer(loadPids()).when(mySearchBuilder).loadResourcesByPid(any(Collection.class), any(Collection.class), any(List.class), anyBoolean(), any()); PersistedJpaBundleProvider provider; From d2ae3c24a13fcc59420ee08047268c14437e8338 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Wed, 28 Aug 2019 16:26:56 -0400 Subject: [PATCH 10/16] FIXME --- .../ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java index 0e49395434d..30e1616c4ea 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java @@ -32,7 +32,6 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl { /* * Be careful increasing this number! We use the number of params here in a - * // FIXME KHS * DELETE FROM foo WHERE params IN (aaaa) * type query and this can fail if we have 1000s of params */ From 5c08e8fdf81fe0e048d1763e896c5d014f677c91 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Tue, 3 Sep 2019 15:05:44 -0400 Subject: [PATCH 11/16] anylist r5 support also added another accessor --- hapi-fhir-jpaserver-model/pom.xml | 5 ++ .../fhir/jpa/model/any/AnyListResource.java | 85 +++++++++++++++++++ .../jpa/model/any/AnyListResourceTest.java | 24 ++++++ .../fhir/r4/hapi/ctx/HapiWorkerContext.java | 5 ++ .../fhir/r5/hapi/ctx/HapiWorkerContext.java | 5 ++ 5 files changed, 124 insertions(+) create mode 100644 hapi-fhir-jpaserver-model/src/test/java/ca/uhn/fhir/jpa/model/any/AnyListResourceTest.java diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index 45ab231bec8..8ae1cc7e7f8 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -46,6 +46,11 @@ hapi-fhir-structures-r4 ${project.version} + + ca.uhn.hapi.fhir + hapi-fhir-structures-r5 + ${project.version} + ca.uhn.hapi.fhir hapi-fhir-structures-hl7org-dstu2 diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/any/AnyListResource.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/any/AnyListResource.java index c2636f3d88a..9ba10073f52 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/any/AnyListResource.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/any/AnyListResource.java @@ -43,6 +43,8 @@ public class AnyListResource { return new AnyListResource(new org.hl7.fhir.dstu3.model.ListResource()); case R4: return new AnyListResource(new org.hl7.fhir.r4.model.ListResource()); + case R5: + return new AnyListResource(new org.hl7.fhir.r5.model.ListResource()); default: throw new UnsupportedOperationException(version + " not supported"); } @@ -63,6 +65,11 @@ public class AnyListResource { myListResource = theListResourceR4; } + public AnyListResource(org.hl7.fhir.r5.model.ListResource theListResourceR5) { + myFhirVersion = FhirVersionEnum.R5; + myListResource = theListResourceR5; + } + public static AnyListResource fromResource(IBaseResource theListResource) { if (theListResource instanceof ca.uhn.fhir.model.dstu2.resource.ListResource) { return new AnyListResource((ca.uhn.fhir.model.dstu2.resource.ListResource) theListResource); @@ -70,6 +77,8 @@ public class AnyListResource { return new AnyListResource((org.hl7.fhir.dstu3.model.ListResource) theListResource); } else if (theListResource instanceof org.hl7.fhir.r4.model.ListResource) { return new AnyListResource((org.hl7.fhir.r4.model.ListResource) theListResource); + } else if (theListResource instanceof org.hl7.fhir.r5.model.ListResource) { + return new AnyListResource((org.hl7.fhir.r5.model.ListResource) theListResource); } else { throw new UnsupportedOperationException("Cannot convert " + theListResource.getClass().getName() + " to AnyList"); } @@ -94,6 +103,11 @@ public class AnyListResource { return (org.hl7.fhir.r4.model.ListResource) get(); } + public org.hl7.fhir.r5.model.ListResource getR5() { + Validate.isTrue(myFhirVersion == FhirVersionEnum.R5); + return (org.hl7.fhir.r5.model.ListResource) get(); + } + public FhirVersionEnum getFhirVersion() { return myFhirVersion; } @@ -106,6 +120,9 @@ public class AnyListResource { case R4: getR4().getCode().addCoding().setSystem(theSystem).setCode(theCode); break; + case R5: + getR5().getCode().addCoding().setSystem(theSystem).setCode(theCode); + break; default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -119,6 +136,9 @@ public class AnyListResource { case R4: getR4().getIdentifier().add(new org.hl7.fhir.r4.model.Identifier().setSystem(theSystem).setValue(theValue)); break; + case R5: + getR5().getIdentifier().add(new org.hl7.fhir.r5.model.Identifier().setSystem(theSystem).setValue(theValue)); + break; default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -132,6 +152,9 @@ public class AnyListResource { case R4: getR4().addExtension().setUrl(theUrl).setValue(new org.hl7.fhir.r4.model.StringType(theValue)); break; + case R5: + getR5().addExtension().setUrl(theUrl).setValue(new org.hl7.fhir.r5.model.StringType(theValue)); + break; default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -143,6 +166,8 @@ public class AnyListResource { return getStringExtensionValueOrNullDstu3(theUrl); case R4: return getStringExtensionValueOrNullR4(theUrl); + case R5: + return getStringExtensionValueOrNullR5(theUrl); default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -166,6 +191,15 @@ public class AnyListResource { return targetType.getValue(); } + private String getStringExtensionValueOrNullR5(String theUrl) { + List targetTypes = getR5().getExtensionsByUrl(theUrl); + if (targetTypes.size() < 1) { + return null; + } + org.hl7.fhir.r5.model.StringType targetType = (org.hl7.fhir.r5.model.StringType) targetTypes.get(0).getValue(); + return targetType.getValue(); + } + public void addReference(IBaseReference theReference) { switch (myFhirVersion) { case DSTU3: @@ -174,6 +208,9 @@ public class AnyListResource { case R4: getR4().addEntry().setItem((org.hl7.fhir.r4.model.Reference) theReference); break; + case R5: + getR5().addEntry().setItem((org.hl7.fhir.r5.model.Reference) theReference); + break; default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -187,6 +224,9 @@ public class AnyListResource { case R4: getR4().addEntry().setItem(new org.hl7.fhir.r4.model.Reference(theReferenceId)); break; + case R5: + getR5().addEntry().setItem(new org.hl7.fhir.r5.model.Reference(theReferenceId)); + break; default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -202,6 +242,10 @@ public class AnyListResource { return getR4().getEntry().stream() .map(entry -> entry.getItem().getReference()) .map(reference -> new org.hl7.fhir.r4.model.IdType(reference).toUnqualifiedVersionless().getValue()); + case R5: + return getR5().getEntry().stream() + .map(entry -> entry.getItem().getReference()) + .map(reference -> new org.hl7.fhir.r5.model.IdType(reference).toUnqualifiedVersionless().getValue()); default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -213,6 +257,8 @@ public class AnyListResource { return removeItemDstu3(theReferenceId); case R4: return removeItemR4(theReferenceId); + case R5: + return removeItemR5(theReferenceId); default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } @@ -250,6 +296,22 @@ public class AnyListResource { return removed; } + private boolean removeItemR5(String theReferenceId) { + boolean removed = false; + for (org.hl7.fhir.r5.model.ListResource.ListEntryComponent entry : getR5().getEntry()) { + if (theReferenceId.equals(entry.getItem().getReference()) && !entry.getDeleted()) { + entry.setDeleted(true); + removed = true; + break; + } + } + + if (removed) { + getR5().getEntry().removeIf(entry -> entry.getDeleted()); + } + return removed; + } + public TokenParam getCodeFirstRep() { switch (myFhirVersion) { case DSTU3: @@ -258,17 +320,40 @@ public class AnyListResource { case R4: org.hl7.fhir.r4.model.Coding codingR4 = getR4().getCode().getCodingFirstRep(); return new TokenParam(codingR4.getSystem(), codingR4.getCode()); + case R5: + org.hl7.fhir.r5.model.Coding codingR5 = getR5().getCode().getCodingFirstRep(); + return new TokenParam(codingR5.getSystem(), codingR5.getCode()); default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } } + public TokenParam getIdentifierirstRep() { + switch (myFhirVersion) { + case DSTU3: + org.hl7.fhir.dstu3.model.Identifier identDstu3 = getDstu3().getIdentifierFirstRep(); + return new TokenParam(identDstu3.getSystem(), identDstu3.getValue()); + case R4: + org.hl7.fhir.r4.model.Identifier identR4 = getR4().getIdentifierFirstRep(); + return new TokenParam(identR4.getSystem(), identR4.getValue()); + case R5: + org.hl7.fhir.r5.model.Identifier identR5 = getR5().getIdentifierFirstRep(); + return new TokenParam(identR5.getSystem(), identR5.getValue()); + default: + throw new UnsupportedOperationException(myFhirVersion + " not supported"); + } + } + + + public boolean isEmpty() { switch (myFhirVersion) { case DSTU3: return getDstu3().getEntry().isEmpty(); case R4: return getR4().getEntry().isEmpty(); + case R5: + return getR5().getEntry().isEmpty(); default: throw new UnsupportedOperationException(myFhirVersion + " not supported"); } diff --git a/hapi-fhir-jpaserver-model/src/test/java/ca/uhn/fhir/jpa/model/any/AnyListResourceTest.java b/hapi-fhir-jpaserver-model/src/test/java/ca/uhn/fhir/jpa/model/any/AnyListResourceTest.java new file mode 100644 index 00000000000..46b71084710 --- /dev/null +++ b/hapi-fhir-jpaserver-model/src/test/java/ca/uhn/fhir/jpa/model/any/AnyListResourceTest.java @@ -0,0 +1,24 @@ +package ca.uhn.fhir.jpa.model.any; + +import org.hl7.fhir.r5.model.ListResource; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class AnyListResourceTest { + @Test + public void getCodeFirstRep() { + AnyListResource listResource = AnyListResource.fromResource(new ListResource()); + listResource.addCode("foo", "bar"); + assertEquals("foo", listResource.getCodeFirstRep().getSystem()); + assertEquals("bar", listResource.getCodeFirstRep().getValue()); + } + + @Test + public void getIdentifierFirstRep() { + AnyListResource listResource = AnyListResource.fromResource(new ListResource()); + listResource.addIdentifier("foo", "bar"); + assertEquals("foo", listResource.getIdentifierirstRep().getSystem()); + assertEquals("bar", listResource.getIdentifierirstRep().getValue()); + } +} diff --git a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java index bd771e1c766..d2dfd73e5a2 100644 --- a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java @@ -389,6 +389,11 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander throw new UnsupportedOperationException(); } + @Override + public String getLinkForUrl(String theS, String theS1) { + return null; + } + @Override public List getTypeNames() { throw new UnsupportedOperationException(); diff --git a/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java index a3ebec0ce7e..b3268072dce 100644 --- a/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java @@ -353,6 +353,11 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander throw new UnsupportedOperationException(); } + @Override + public String getLinkForUrl(String theS, String theS1) { + return null; + } + @Override public boolean isNoTerminologyServer() { return false; From e6d866cbf1cd95efb38ff7b38268ac4b751ba0c2 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Tue, 3 Sep 2019 15:14:27 -0400 Subject: [PATCH 12/16] Revert: premature hapiworkercontext changes --- .../java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java | 5 ----- .../java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java | 5 ----- 2 files changed, 10 deletions(-) diff --git a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java index d2dfd73e5a2..bd771e1c766 100644 --- a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/HapiWorkerContext.java @@ -389,11 +389,6 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander throw new UnsupportedOperationException(); } - @Override - public String getLinkForUrl(String theS, String theS1) { - return null; - } - @Override public List getTypeNames() { throw new UnsupportedOperationException(); diff --git a/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java b/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java index b3268072dce..a3ebec0ce7e 100644 --- a/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java +++ b/hapi-fhir-structures-r5/src/main/java/org/hl7/fhir/r5/hapi/ctx/HapiWorkerContext.java @@ -353,11 +353,6 @@ public final class HapiWorkerContext implements IWorkerContext, ValueSetExpander throw new UnsupportedOperationException(); } - @Override - public String getLinkForUrl(String theS, String theS1) { - return null; - } - @Override public boolean isNoTerminologyServer() { return false; From 0a5c52122e7d3cbf2cdc3d1294d6280c8c4ce950 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Wed, 4 Sep 2019 08:49:18 -0400 Subject: [PATCH 13/16] added list stresstest --- .../fhir/jpa/stresstest/StressTestR4Test.java | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java index 15d9764c213..88fc163c44d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java @@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.test.utilities.UnregisterScheduledProcessor; @@ -31,10 +32,8 @@ import org.hl7.fhir.r4.hapi.validation.FhirInstanceValidator; import org.hl7.fhir.r4.model.*; import org.hl7.fhir.r4.model.Bundle.BundleType; import org.hl7.fhir.r4.model.Bundle.HTTPVerb; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; +import org.hl7.fhir.r4.model.codesystems.HttpVerb; +import org.junit.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; import org.springframework.test.util.AopTestUtils; @@ -337,6 +336,45 @@ public class StressTestR4Test extends BaseResourceProviderR4Test { assertEquals(1202, resultsAndIncludes.size()); } + @Ignore + @Test + public void testUpdateListWithLargeNumberOfEntries() { + int numPatients = 3000; + + ListResource lr = new ListResource(); + lr.setId(IdType.newRandomUuid()); + + { + Bundle bundle = new Bundle(); + for (int i = 0; i < numPatients; ++i) { + Patient patient = new Patient(); + patient.setId(IdType.newRandomUuid()); + bundle.addEntry().setFullUrl(patient.getId()).setResource(patient).getRequest().setMethod(HTTPVerb.POST).setUrl("Patient"); + lr.addEntry().setItem(new Reference(patient.getId())); + } + bundle.addEntry().setFullUrl(lr.getId()).setResource(lr).getRequest().setMethod(HTTPVerb.POST).setUrl("List"); + + StopWatch sw = new StopWatch(); + ourLog.info("Saving list with {} entries", lr.getEntry().size()); + mySystemDao.transaction(null, bundle); + ourLog.info("Saved {} resources in {}", bundle.getEntry().size(), sw); + } + + { + Bundle bundle = new Bundle(); + + Patient newPatient = new Patient(); + newPatient.setId(IdType.newRandomUuid()); + bundle.addEntry().setFullUrl(newPatient.getId()).setResource(newPatient).getRequest().setMethod(HTTPVerb.POST).setUrl("Patient"); + lr.addEntry().setItem(new Reference(newPatient.getId())); + bundle.addEntry().setFullUrl(lr.getId()).setResource(lr).getRequest().setMethod(HTTPVerb.PUT).setUrl(lr.getIdElement().toUnqualifiedVersionless().getValue()); + + StopWatch sw = new StopWatch(); + ourLog.info("Updating list with {} entries", lr.getEntry().size()); + mySystemDao.transaction(null, bundle); + ourLog.info("Updated {} resources in {}", bundle.getEntry().size(), sw); + } + } @Test public void testMultithreadedSearch() throws Exception { From 14fadd30045a932fc10ac820a2b25f4cbbc65c7f Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Thu, 5 Sep 2019 15:51:38 -0400 Subject: [PATCH 14/16] fix drop foreign key task --- .../ca/uhn/fhir/jpa/migrate/JdbcUtils.java | 6 +++++- .../migrate/taskdef/DropForeignKeyTask.java | 10 ++++++++-- .../tasks/HapiFhirJpaMigrationTasks.java | 4 ++-- .../migrate/tasks/api/BaseMigrationTasks.java | 8 +++++++- .../fhir/jpa/migrate/taskdef/AddIndexTest.java | 4 ++-- .../taskdef/DropForeignKeyTaskTest.java | 18 +++++++++--------- 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java index 014de578bd4..32810675705 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java @@ -185,7 +185,11 @@ public class JdbcUtils { DatabaseMetaData metadata; try { metadata = connection.getMetaData(); - ResultSet indexes = metadata.getCrossReference(connection.getCatalog(), connection.getSchema(), massageIdentifier(metadata, theTableName), connection.getCatalog(), connection.getSchema(), massageIdentifier(metadata, theForeignTable)); + String catalog = connection.getCatalog(); + String schema = connection.getSchema(); + String parentTable = massageIdentifier(metadata, theTableName); + String foreignTable = massageIdentifier(metadata, theForeignTable); + ResultSet indexes = metadata.getCrossReference(catalog, schema, parentTable, catalog, schema, foreignTable); Set columnNames = new HashSet<>(); while (indexes.next()) { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java index 3d777691df6..bea3f581096 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java @@ -30,26 +30,32 @@ import java.util.Set; import static org.apache.commons.lang3.StringUtils.isNotBlank; -public class DropForeignKeyTask extends BaseTableColumnTask { +public class DropForeignKeyTask extends BaseTableTask { private static final Logger ourLog = LoggerFactory.getLogger(DropForeignKeyTask.class); private String myConstraintName; + private String myForeignTableName; public void setConstraintName(String theConstraintName) { myConstraintName = theConstraintName; } + public void setForeignTableName(String theForeignTableName) { + myForeignTableName = theForeignTableName; + } + @Override public void validate() { super.validate(); Validate.isTrue(isNotBlank(myConstraintName)); + Validate.isTrue(isNotBlank(myForeignTableName)); } @Override public void execute() throws SQLException { - Set existing = JdbcUtils.getForeignKeys(getConnectionProperties(), null, null); + Set existing = JdbcUtils.getForeignKeys(getConnectionProperties(), getTableName(), myForeignTableName); if (!existing.contains(myConstraintName)) { ourLog.info("Don't have constraint named {} - No action performed", myConstraintName); return; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index a2d2efce7b4..6973e2cb268 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -88,8 +88,8 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { .toColumn("VALUESET_PID") .references("TRM_VALUESET", "PID"); // Drop HFJ_SEARCH_RESULT foreign keys - version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_RES"); - version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_SEARCH"); + version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_RES", "HFJ_RESOURCE"); + version.onTable("HFJ_SEARCH_RESULT").dropForeignKey("FK_SEARCHRES_SEARCH", "HFJ_SEARCH"); // TermValueSet version.startSectionWithMessage("Processing table: TRM_VALUESET"); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java index d536578fd8b..890f573ff6b 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java @@ -241,9 +241,15 @@ public class BaseMigrationTasks { return this; } - public void dropForeignKey(String theFkName) { + /** + * + * @param theFkName the name of the foreign key + * @param theForeignTableName the name of the table that imports the foreign key (I know it's a confusing name, but it's what java.sql.DatabaseMetaData calls it) + */ + public void dropForeignKey(String theFkName, String theForeignTableName) { DropForeignKeyTask task = new DropForeignKeyTask(); task.setConstraintName(theFkName); + task.setForeignTableName(theForeignTableName); task.setTableName(getTableName()); addTask(task); } diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/AddIndexTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/AddIndexTest.java index 6e21947a2e7..0f78c9e1ad7 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/AddIndexTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/AddIndexTest.java @@ -1,12 +1,12 @@ package ca.uhn.fhir.jpa.migrate.taskdef; import ca.uhn.fhir.jpa.migrate.JdbcUtils; -import com.google.common.collect.Lists; import org.junit.Test; import java.sql.SQLException; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; public class AddIndexTest extends BaseTest { diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java index 4150d24c30e..ae6d3b17f02 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTaskTest.java @@ -1,33 +1,33 @@ package ca.uhn.fhir.jpa.migrate.taskdef; import ca.uhn.fhir.jpa.migrate.JdbcUtils; -import org.hamcrest.Matchers; import org.junit.Test; import java.sql.SQLException; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.junit.Assert.assertThat; public class DropForeignKeyTaskTest extends BaseTest { @Test public void testDropForeignKey() throws SQLException { - executeSql("create table HOME (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); - executeSql("create table FOREIGNTBL (PID bigint not null, HOMEREF bigint)"); - executeSql("alter table HOME add foreign key FK_FOO (PID) references FOREIGNTABLE(HOMEREF)"); + executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); - assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "HOME", "FOREIGNTBL"), empty()); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); DropForeignKeyTask task = new DropForeignKeyTask(); - task.setTableName("FOREIGNTBL"); - task.setColumnName("HOMEREF"); - task.setConstraintName("FK_FOO"); + task.setTableName("PARENT"); + task.setForeignTableName("CHILD"); + task.setConstraintName("FK_MOM"); getMigrator().addTask(task); getMigrator().migrate(); - assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "HOME", "FOREIGNTBL"), empty()); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), empty()); // Make sure additional calls don't crash getMigrator().migrate(); From 2a08808935f1f08035f8e91ebfc5484e5cb2cc37 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Thu, 5 Sep 2019 16:11:58 -0400 Subject: [PATCH 15/16] final PR cleanup --- .../java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java | 3 +-- .../main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java | 4 ---- .../src/main/java/ca/uhn/fhir/jpa/entity/Search.java | 5 +---- .../src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java | 1 - .../ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java | 1 - .../ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java | 1 - .../jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java | 4 +++- .../ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java | 3 --- .../src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java | 4 ++-- .../src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java | 4 +--- .../fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java | 2 +- .../jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java | 1 - .../java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java | 2 -- .../ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java | 2 +- .../fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java | 1 - .../uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java | 2 +- 16 files changed, 11 insertions(+), 29 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java index 21a76447194..0dc0681ba9a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchIncludeDao.java @@ -20,13 +20,12 @@ package ca.uhn.fhir.jpa.dao.data; * #L% */ +import ca.uhn.fhir.jpa.entity.SearchInclude; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import ca.uhn.fhir.jpa.entity.SearchInclude; - public interface ISearchIncludeDao extends JpaRepository { @Modifying diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java index db7b620c7cc..107537ed3c9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java @@ -1,8 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchResult; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; @@ -10,9 +8,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import java.util.Iterator; import java.util.List; -import java.util.Set; /* * #%L diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index 928e4b0d48c..10908546890 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -1,10 +1,10 @@ package ca.uhn.fhir.jpa.entity; -import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import org.apache.commons.lang3.SerializationUtils; import org.hibernate.annotations.OptimisticLock; @@ -80,9 +80,6 @@ public class Search implements ICachedSearchDetails, Serializable { private Long myResourceId; @Column(name = "RESOURCE_TYPE", length = 200, nullable = true) private String myResourceType; - // FIXME JA: delete this if we can - I don't want to imply that the results are a part of the search, they link to it but they don't need to be loaded just because we're loading the search - // @OneToMany(mappedBy = "mySearch", fetch = FetchType.LAZY) - // private Collection myResults; @NotNull @Temporal(TemporalType.TIMESTAMP) @Column(name = "SEARCH_LAST_RETURNED", nullable = false, updatable = false) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java index ab8a53eed53..62536ccb5d7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchResult.java @@ -20,7 +20,6 @@ package ca.uhn.fhir.jpa.entity; * #L% */ -import ca.uhn.fhir.jpa.model.entity.ResourceTable; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.ToStringBuilder; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java index b380816ae9b..8d3f97c2912 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaBundleProvider.java @@ -31,7 +31,6 @@ import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.BaseHasResource; import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; -import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster; import ca.uhn.fhir.model.primitive.InstantDt; import ca.uhn.fhir.rest.api.server.*; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java index ccc6bc0fac4..5f41ca45230 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java @@ -22,7 +22,6 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; -import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.transaction.annotation.Propagation; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index 7684c268ee0..572416dc952 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -11,7 +11,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Pageable; import javax.transaction.Transactional; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import static ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.toPage; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java index 352c1a7e407..13d2f255724 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchResultCacheSvc.java @@ -2,10 +2,7 @@ package ca.uhn.fhir.jpa.search.cache; import ca.uhn.fhir.jpa.entity.Search; -import java.util.Collection; -import java.util.Date; import java.util.List; -import java.util.Optional; public interface ISearchResultCacheSvc { /** diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java index 8b7583f180d..248c7cd7435 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java @@ -5,6 +5,7 @@ import ca.uhn.fhir.interceptor.api.IInterceptorService; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.interceptor.executor.InterceptorService; import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.SystemProviderDstu2Test; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; @@ -16,14 +17,13 @@ import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; import ca.uhn.fhir.jpa.term.VersionIndependentConcept; import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.jpa.util.ExpungeOptions; -import ca.uhn.fhir.jpa.model.util.JpaConstants; -import ca.uhn.fhir.test.utilities.LoggingRule; import ca.uhn.fhir.model.dstu2.resource.Bundle; import ca.uhn.fhir.model.dstu2.resource.Bundle.Entry; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.test.utilities.LoggingRule; import ca.uhn.fhir.util.BundleUtil; import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.TestUtil; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java index fd771372875..d872b64fc6e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.interceptor.api.IInterceptorService; +import ca.uhn.fhir.jpa.binstore.BinaryAccessProvider; import ca.uhn.fhir.jpa.binstore.BinaryStorageInterceptor; import ca.uhn.fhir.jpa.config.TestR4Config; import ca.uhn.fhir.jpa.dao.*; @@ -11,7 +12,6 @@ import ca.uhn.fhir.jpa.interceptor.PerformanceTracingLoggingInterceptor; import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString; import ca.uhn.fhir.jpa.model.entity.ResourceTable; -import ca.uhn.fhir.jpa.binstore.BinaryAccessProvider; import ca.uhn.fhir.jpa.provider.r4.JpaSystemProviderR4; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc; @@ -21,7 +21,6 @@ import ca.uhn.fhir.jpa.search.warm.ICacheWarmingSvc; import ca.uhn.fhir.jpa.searchparam.registry.BaseSearchParamRegistry; import ca.uhn.fhir.jpa.subscription.module.cache.SubscriptionRegistry; import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvcImpl; -import ca.uhn.fhir.jpa.term.IHapiTerminologySvc; import ca.uhn.fhir.jpa.term.IHapiTerminologySvcR4; import ca.uhn.fhir.jpa.util.ResourceCountCache; import ca.uhn.fhir.jpa.util.ResourceProviderFactory; @@ -57,7 +56,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.util.AopTestUtils; import org.springframework.transaction.PlatformTransactionManager; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java index 826a387cd6f..d38c66269b5 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoHashesTest.java @@ -1,7 +1,7 @@ package ca.uhn.fhir.jpa.dao.r4; -import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java index 75f7d4c6f1f..c14e457e9d6 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java @@ -5,7 +5,6 @@ import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl; -import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.TestUtil; import ca.uhn.fhir.rest.api.server.IBundleProvider; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java index 3d3f422b2df..64e942ed78f 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4Test.java @@ -8,7 +8,6 @@ import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; -import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.searchparam.SearchParamConstants; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.TestUtil; @@ -45,7 +44,6 @@ import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; import org.hl7.fhir.r4.model.OperationOutcome.IssueType; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; import org.junit.*; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index ef8e70e40fe..44da70e8c45 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -5,8 +5,8 @@ import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.data.ISearchDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable; -import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.util.TestUtil; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.model.primitive.InstantDt; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java index 33282d2a386..f2429ccf050 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java @@ -10,7 +10,6 @@ import ca.uhn.fhir.jpa.entity.SearchTypeEnum; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl; -import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.rest.gclient.IClientExecutable; import ca.uhn.fhir.rest.gclient.IQuery; import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java index 890f573ff6b..0057645dc07 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/BaseMigrationTasks.java @@ -244,7 +244,7 @@ public class BaseMigrationTasks { /** * * @param theFkName the name of the foreign key - * @param theForeignTableName the name of the table that imports the foreign key (I know it's a confusing name, but it's what java.sql.DatabaseMetaData calls it) + * @param theForeignTableName the name of the table that imports the foreign key (I know it's a confusing name, but that's what java.sql.DatabaseMetaData calls it) */ public void dropForeignKey(String theFkName, String theForeignTableName) { DropForeignKeyTask task = new DropForeignKeyTask(); From b1421b1d60bd73aa23f309813280f5d640379c1b Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Thu, 5 Sep 2019 16:20:24 -0400 Subject: [PATCH 16/16] @NotNull (hope I picked the right one!) --- .../java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java | 2 +- .../ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java | 2 +- .../ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java | 2 +- .../java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java | 2 +- .../jpa/subscription/resthook/RestHookTestDstu2Test.java | 5 ++--- .../jpa/subscription/resthook/RestHookTestDstu3Test.java | 5 ++--- .../java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java | 1 - 7 files changed, 8 insertions(+), 11 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index f638c26be59..82994c5e145 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -54,7 +54,6 @@ import ca.uhn.fhir.validation.*; import org.apache.commons.lang3.Validate; import org.hl7.fhir.instance.model.api.*; import org.hl7.fhir.r4.model.InstantType; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -72,6 +71,7 @@ import javax.annotation.PostConstruct; import javax.persistence.NoResultException; import javax.persistence.TypedQuery; import javax.servlet.http.HttpServletResponse; +import javax.validation.constraints.NotNull; import java.io.IOException; import java.util.*; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 2ea494daf1d..763e1f18251 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -58,7 +58,6 @@ import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.AbstractPageRequest; import org.springframework.data.domain.Pageable; @@ -79,6 +78,7 @@ import org.springframework.transaction.support.TransactionTemplate; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.persistence.EntityManager; +import javax.validation.constraints.NotNull; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index cdba0d04b2d..2511af14770 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -61,7 +61,6 @@ import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.r4.model.*; import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -87,6 +86,7 @@ import javax.persistence.PersistenceContext; import javax.persistence.PersistenceContextType; import javax.persistence.TypedQuery; import javax.persistence.criteria.*; +import javax.validation.constraints.NotNull; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java index f719c98b37d..31bff96384a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcImpl.java @@ -34,9 +34,9 @@ import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.ConceptMap; import org.hl7.fhir.r4.model.Enumerations; import org.hl7.fhir.r4.model.ValueSet; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; +import javax.validation.constraints.NotNull; import java.io.*; import java.util.*; import java.util.Map.Entry; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu2Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu2Test.java index ffe45928795..cf9591cf8cb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu2Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu2Test.java @@ -20,16 +20,17 @@ import ca.uhn.fhir.rest.api.MethodOutcome; import ca.uhn.fhir.rest.server.IResourceProvider; import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.test.utilities.JettyUtil; import com.google.common.collect.Lists; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.jetbrains.annotations.NotNull; import org.junit.*; import org.springframework.beans.factory.annotation.Autowired; +import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -38,8 +39,6 @@ import static ca.uhn.fhir.jpa.subscription.resthook.RestHookTestDstu3Test.logAll import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; -import ca.uhn.fhir.test.utilities.JettyUtil; - /** * Test the rest-hook subscriptions */ diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu3Test.java index 07b6b479d42..02fde230ef3 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/subscription/resthook/RestHookTestDstu3Test.java @@ -17,6 +17,7 @@ import ca.uhn.fhir.rest.api.MethodOutcome; import ca.uhn.fhir.rest.server.IResourceProvider; import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; +import ca.uhn.fhir.test.utilities.JettyUtil; import com.google.common.collect.Lists; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -24,11 +25,11 @@ import org.eclipse.jetty.servlet.ServletHolder; import org.hl7.fhir.dstu3.model.*; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.jetbrains.annotations.NotNull; import org.junit.*; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletRequest; +import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -40,8 +41,6 @@ import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.*; -import ca.uhn.fhir.test.utilities.JettyUtil; - /** * Test the rest-hook subscriptions */ diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java index 3f68e64de3a..e53908edf9a 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java @@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.migrate.taskdef; */ import ca.uhn.fhir.jpa.migrate.JdbcUtils; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory;